View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.lang.ref.WeakReference;
22  import java.nio.ByteBuffer;
23  import java.util.EnumMap;
24  import java.util.Iterator;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.PriorityQueue;
28  import java.util.SortedSet;
29  import java.util.TreeSet;
30  import java.util.concurrent.ConcurrentHashMap;
31  import java.util.concurrent.Executors;
32  import java.util.concurrent.ScheduledExecutorService;
33  import java.util.concurrent.TimeUnit;
34  import java.util.concurrent.atomic.AtomicLong;
35  import java.util.concurrent.locks.ReentrantLock;
36  
37  import com.google.common.base.Objects;
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.hadoop.hbase.classification.InterfaceAudience;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.hbase.io.HeapSize;
43  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
44  import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.ClassSize;
47  import org.apache.hadoop.hbase.util.HasThread;
48  import org.apache.hadoop.util.StringUtils;
49  import org.codehaus.jackson.annotate.JsonIgnoreProperties;
50  
51  import com.google.common.annotations.VisibleForTesting;
52  import com.google.common.util.concurrent.ThreadFactoryBuilder;
53  
54  /**
55   * A block cache implementation that is memory-aware using {@link HeapSize},
56   * memory-bound using an LRU eviction algorithm, and concurrent: backed by a
57   * {@link ConcurrentHashMap} and with a non-blocking eviction thread giving
58   * constant-time {@link #cacheBlock} and {@link #getBlock} operations.<p>
59   *
60   * Contains three levels of block priority to allow for
61   * scan-resistance and in-memory families 
62   * {@link org.apache.hadoop.hbase.HColumnDescriptor#setInMemory(boolean)} (An
63   * in-memory column family is a column family that should be served from memory if possible):
64   * single-access, multiple-accesses, and in-memory priority.
65   * A block is added with an in-memory priority flag if
66   * {@link org.apache.hadoop.hbase.HColumnDescriptor#isInMemory()}, 
67   * otherwise a block becomes a single access
68   * priority the first time it is read into this block cache.  If a block is accessed again while
69   * in cache, it is marked as a multiple access priority block.  This delineation of blocks is used
70   * to prevent scans from thrashing the cache adding a least-frequently-used
71   * element to the eviction algorithm.<p>
72   *
73   * Each priority is given its own chunk of the total cache to ensure
74   * fairness during eviction.  Each priority will retain close to its maximum
75   * size, however, if any priority is not using its entire chunk the others
76   * are able to grow beyond their chunk size.<p>
77   *
78   * Instantiated at a minimum with the total size and average block size.
79   * All sizes are in bytes.  The block size is not especially important as this
80   * cache is fully dynamic in its sizing of blocks.  It is only used for
81   * pre-allocating data structures and in initial heap estimation of the map.<p>
82   *
83   * The detailed constructor defines the sizes for the three priorities (they
84   * should total to the <code>maximum size</code> defined).  It also sets the levels that
85   * trigger and control the eviction thread.<p>
86   *
87   * The <code>acceptable size</code> is the cache size level which triggers the eviction
88   * process to start.  It evicts enough blocks to get the size below the
89   * minimum size specified.<p>
90   *
91   * Eviction happens in a separate thread and involves a single full-scan
92   * of the map.  It determines how many bytes must be freed to reach the minimum
93   * size, and then while scanning determines the fewest least-recently-used
94   * blocks necessary from each of the three priorities (would be 3 times bytes
95   * to free).  It then uses the priority chunk sizes to evict fairly according
96   * to the relative sizes and usage.
97   */
98  @InterfaceAudience.Private
99  @JsonIgnoreProperties({"encodingCountsForTest"})
100 public class LruBlockCache implements ResizableBlockCache, HeapSize {
101 
102   static final Log LOG = LogFactory.getLog(LruBlockCache.class);
103 
104   /**
105    * Percentage of total size that eviction will evict until; e.g. if set to .8, then we will keep
106    * evicting during an eviction run till the cache size is down to 80% of the total.
107    */
108   static final String LRU_MIN_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.min.factor";
109 
110   /**
111    * Acceptable size of cache (no evictions if size < acceptable)
112    */
113   static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.acceptable.factor";
114 
115   static final String LRU_SINGLE_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.single.percentage";
116   static final String LRU_MULTI_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.multi.percentage";
117   static final String LRU_MEMORY_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.memory.percentage";
118 
119   /**
120    * Configuration key to force data-block always (except in-memory are too much)
121    * cached in memory for in-memory hfile, unlike inMemory, which is a column-family
122    * configuration, inMemoryForceMode is a cluster-wide configuration
123    */
124   static final String LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME = "hbase.lru.rs.inmemoryforcemode";
125 
126   /** Default Configuration Parameters*/
127 
128   /** Backing Concurrent Map Configuration */
129   static final float DEFAULT_LOAD_FACTOR = 0.75f;
130   static final int DEFAULT_CONCURRENCY_LEVEL = 16;
131 
132   /** Eviction thresholds */
133   static final float DEFAULT_MIN_FACTOR = 0.95f;
134   static final float DEFAULT_ACCEPTABLE_FACTOR = 0.99f;
135 
136   /** Priority buckets */
137   static final float DEFAULT_SINGLE_FACTOR = 0.25f;
138   static final float DEFAULT_MULTI_FACTOR = 0.50f;
139   static final float DEFAULT_MEMORY_FACTOR = 0.25f;
140 
141   static final boolean DEFAULT_IN_MEMORY_FORCE_MODE = false;
142 
143   /** Statistics thread */
144   static final int statThreadPeriod = 60 * 5;
145 
146   /** Concurrent map (the cache) */
147   private final Map<BlockCacheKey,LruCachedBlock> map;
148 
149   /** Eviction lock (locked when eviction in process) */
150   private final ReentrantLock evictionLock = new ReentrantLock(true);
151 
152   /** Volatile boolean to track if we are in an eviction process or not */
153   private volatile boolean evictionInProgress = false;
154 
155   /** Eviction thread */
156   private final EvictionThread evictionThread;
157 
158   /** Statistics thread schedule pool (for heavy debugging, could remove) */
159   private final ScheduledExecutorService scheduleThreadPool = Executors.newScheduledThreadPool(1,
160     new ThreadFactoryBuilder().setNameFormat("LruBlockCacheStatsExecutor").setDaemon(true).build());
161 
162   /** Current size of cache */
163   private final AtomicLong size;
164 
165   /** Current number of cached elements */
166   private final AtomicLong elements;
167 
168   /** Cache access count (sequential ID) */
169   private final AtomicLong count;
170 
171   /** Cache statistics */
172   private final CacheStats stats;
173 
174   /** Maximum allowable size of cache (block put if size > max, evict) */
175   private long maxSize;
176 
177   /** Approximate block size */
178   private long blockSize;
179 
180   /** Acceptable size of cache (no evictions if size < acceptable) */
181   private float acceptableFactor;
182 
183   /** Minimum threshold of cache (when evicting, evict until size < min) */
184   private float minFactor;
185 
186   /** Single access bucket size */
187   private float singleFactor;
188 
189   /** Multiple access bucket size */
190   private float multiFactor;
191 
192   /** In-memory bucket size */
193   private float memoryFactor;
194 
195   /** Overhead of the structure itself */
196   private long overhead;
197 
198   /** Whether in-memory hfile's data block has higher priority when evicting */
199   private boolean forceInMemory;
200 
201   /** Where to send victims (blocks evicted/missing from the cache) */
202   // TODO: Fix it so this is not explicit reference to a particular BlockCache implementation.
203   private BucketCache victimHandler = null;
204 
205   /**
206    * Default constructor.  Specify maximum size and expected average block
207    * size (approximation is fine).
208    *
209    * <p>All other factors will be calculated based on defaults specified in
210    * this class.
211    * @param maxSize maximum size of cache, in bytes
212    * @param blockSize approximate size of each block, in bytes
213    */
214   public LruBlockCache(long maxSize, long blockSize) {
215     this(maxSize, blockSize, true);
216   }
217 
218   /**
219    * Constructor used for testing.  Allows disabling of the eviction thread.
220    */
221   public LruBlockCache(long maxSize, long blockSize, boolean evictionThread) {
222     this(maxSize, blockSize, evictionThread,
223         (int)Math.ceil(1.2*maxSize/blockSize),
224         DEFAULT_LOAD_FACTOR, DEFAULT_CONCURRENCY_LEVEL,
225         DEFAULT_MIN_FACTOR, DEFAULT_ACCEPTABLE_FACTOR,
226         DEFAULT_SINGLE_FACTOR,
227         DEFAULT_MULTI_FACTOR,
228         DEFAULT_MEMORY_FACTOR,
229         false
230         );
231   }
232 
233   public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, Configuration conf) {
234     this(maxSize, blockSize, evictionThread,
235         (int)Math.ceil(1.2*maxSize/blockSize),
236         DEFAULT_LOAD_FACTOR,
237         DEFAULT_CONCURRENCY_LEVEL,
238         conf.getFloat(LRU_MIN_FACTOR_CONFIG_NAME, DEFAULT_MIN_FACTOR),
239         conf.getFloat(LRU_ACCEPTABLE_FACTOR_CONFIG_NAME, DEFAULT_ACCEPTABLE_FACTOR),
240         conf.getFloat(LRU_SINGLE_PERCENTAGE_CONFIG_NAME, DEFAULT_SINGLE_FACTOR),
241         conf.getFloat(LRU_MULTI_PERCENTAGE_CONFIG_NAME, DEFAULT_MULTI_FACTOR),
242         conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, DEFAULT_MEMORY_FACTOR),
243         conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, DEFAULT_IN_MEMORY_FORCE_MODE)
244         );
245   }
246 
247   public LruBlockCache(long maxSize, long blockSize, Configuration conf) {
248     this(maxSize, blockSize, true, conf);
249   }
250 
251   /**
252    * Configurable constructor.  Use this constructor if not using defaults.
253    * @param maxSize maximum size of this cache, in bytes
254    * @param blockSize expected average size of blocks, in bytes
255    * @param evictionThread whether to run evictions in a bg thread or not
256    * @param mapInitialSize initial size of backing ConcurrentHashMap
257    * @param mapLoadFactor initial load factor of backing ConcurrentHashMap
258    * @param mapConcurrencyLevel initial concurrency factor for backing CHM
259    * @param minFactor percentage of total size that eviction will evict until
260    * @param acceptableFactor percentage of total size that triggers eviction
261    * @param singleFactor percentage of total size for single-access blocks
262    * @param multiFactor percentage of total size for multiple-access blocks
263    * @param memoryFactor percentage of total size for in-memory blocks
264    */
265   public LruBlockCache(long maxSize, long blockSize, boolean evictionThread,
266       int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel,
267       float minFactor, float acceptableFactor, float singleFactor,
268       float multiFactor, float memoryFactor, boolean forceInMemory) {
269     if(singleFactor + multiFactor + memoryFactor != 1 ||
270         singleFactor < 0 || multiFactor < 0 || memoryFactor < 0) {
271       throw new IllegalArgumentException("Single, multi, and memory factors " +
272           " should be non-negative and total 1.0");
273     }
274     if(minFactor >= acceptableFactor) {
275       throw new IllegalArgumentException("minFactor must be smaller than acceptableFactor");
276     }
277     if(minFactor >= 1.0f || acceptableFactor >= 1.0f) {
278       throw new IllegalArgumentException("all factors must be < 1");
279     }
280     this.maxSize = maxSize;
281     this.blockSize = blockSize;
282     this.forceInMemory = forceInMemory;
283     map = new ConcurrentHashMap<BlockCacheKey,LruCachedBlock>(mapInitialSize,
284         mapLoadFactor, mapConcurrencyLevel);
285     this.minFactor = minFactor;
286     this.acceptableFactor = acceptableFactor;
287     this.singleFactor = singleFactor;
288     this.multiFactor = multiFactor;
289     this.memoryFactor = memoryFactor;
290     this.stats = new CacheStats(this.getClass().getSimpleName());
291     this.count = new AtomicLong(0);
292     this.elements = new AtomicLong(0);
293     this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel);
294     this.size = new AtomicLong(this.overhead);
295     if(evictionThread) {
296       this.evictionThread = new EvictionThread(this);
297       this.evictionThread.start(); // FindBugs SC_START_IN_CTOR
298     } else {
299       this.evictionThread = null;
300     }
301     // TODO: Add means of turning this off.  Bit obnoxious running thread just to make a log
302     // every five minutes.
303     this.scheduleThreadPool.scheduleAtFixedRate(new StatisticsThread(this),
304         statThreadPeriod, statThreadPeriod, TimeUnit.SECONDS);
305   }
306 
307   @Override
308   public void setMaxSize(long maxSize) {
309     this.maxSize = maxSize;
310     if(this.size.get() > acceptableSize() && !evictionInProgress) {
311       runEviction();
312     }
313   }
314 
315   // BlockCache implementation
316 
317   /**
318    * Cache the block with the specified name and buffer.
319    * <p>
320    * It is assumed this will NOT be called on an already cached block. In rare cases (HBASE-8547)
321    * this can happen, for which we compare the buffer contents.
322    * @param cacheKey block's cache key
323    * @param buf block buffer
324    * @param inMemory if block is in-memory
325    * @param cacheDataInL1
326    */
327   @Override
328   public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory,
329       final boolean cacheDataInL1) {
330     LruCachedBlock cb = map.get(cacheKey);
331     if (cb != null) {
332       // compare the contents, if they are not equal, we are in big trouble
333       if (compare(buf, cb.getBuffer()) != 0) {
334         throw new RuntimeException("Cached block contents differ, which should not have happened."
335           + "cacheKey:" + cacheKey);
336       }
337       String msg = "Cached an already cached block: " + cacheKey + " cb:" + cb.getCacheKey();
338       msg += ". This is harmless and can happen in rare cases (see HBASE-8547)";
339       LOG.warn(msg);
340       return;
341     }
342     cb = new LruCachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory);
343     long newSize = updateSizeMetrics(cb, false);
344     map.put(cacheKey, cb);
345     long val = elements.incrementAndGet();
346     if (LOG.isTraceEnabled()) {
347       long size = map.size();
348       assertCounterSanity(size, val);
349     }
350     if (newSize > acceptableSize() && !evictionInProgress) {
351       runEviction();
352     }
353   }
354 
355   /**
356    * Sanity-checking for parity between actual block cache content and metrics.
357    * Intended only for use with TRACE level logging and -ea JVM.
358    */
359   private static void assertCounterSanity(long mapSize, long counterVal) {
360     if (counterVal < 0) {
361       LOG.trace("counterVal overflow. Assertions unreliable. counterVal=" + counterVal +
362         ", mapSize=" + mapSize);
363       return;
364     }
365     if (mapSize < Integer.MAX_VALUE) {
366       double pct_diff = Math.abs((((double) counterVal) / ((double) mapSize)) - 1.);
367       if (pct_diff > 0.05) {
368         LOG.trace("delta between reported and actual size > 5%. counterVal=" + counterVal +
369           ", mapSize=" + mapSize);
370       }
371     }
372   }
373 
374   private int compare(Cacheable left, Cacheable right) {
375     ByteBuffer l = ByteBuffer.allocate(left.getSerializedLength());
376     left.serialize(l);
377     ByteBuffer r = ByteBuffer.allocate(right.getSerializedLength());
378     right.serialize(r);
379     return Bytes.compareTo(l.array(), l.arrayOffset(), l.limit(),
380       r.array(), r.arrayOffset(), r.limit());
381   }
382 
383   /**
384    * Cache the block with the specified name and buffer.
385    * <p>
386    * @param cacheKey block's cache key
387    * @param buf block buffer
388    */
389   public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
390     cacheBlock(cacheKey, buf, false, false);
391   }
392 
393   /**
394    * Helper function that updates the local size counter and also updates any
395    * per-cf or per-blocktype metrics it can discern from given
396    * {@link LruCachedBlock}
397    *
398    * @param cb
399    * @param evict
400    */
401   protected long updateSizeMetrics(LruCachedBlock cb, boolean evict) {
402     long heapsize = cb.heapSize();
403     if (evict) {
404       heapsize *= -1;
405     }
406     return size.addAndGet(heapsize);
407   }
408 
409   /**
410    * Get the buffer of the block with the specified name.
411    * @param cacheKey block's cache key
412    * @param caching true if the caller caches blocks on cache misses
413    * @param repeat Whether this is a repeat lookup for the same block
414    *        (used to avoid double counting cache misses when doing double-check locking)
415    * @param updateCacheMetrics Whether to update cache metrics or not
416    * @return buffer of specified cache key, or null if not in cache
417    */
418   @Override
419   public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat,
420       boolean updateCacheMetrics) {
421     LruCachedBlock cb = map.get(cacheKey);
422     if (cb == null) {
423       if (!repeat && updateCacheMetrics) stats.miss(caching);
424       if (victimHandler != null) {
425         return victimHandler.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
426       }
427       return null;
428     }
429     if (updateCacheMetrics) stats.hit(caching);
430     cb.access(count.incrementAndGet());
431     return cb.getBuffer();
432   }
433 
434   /**
435    * Whether the cache contains block with specified cacheKey
436    * @param cacheKey
437    * @return true if contains the block
438    */
439   public boolean containsBlock(BlockCacheKey cacheKey) {
440     return map.containsKey(cacheKey);
441   }
442 
443   @Override
444   public boolean evictBlock(BlockCacheKey cacheKey) {
445     LruCachedBlock cb = map.get(cacheKey);
446     if (cb == null) return false;
447     evictBlock(cb, false);
448     return true;
449   }
450 
451   /**
452    * Evicts all blocks for a specific HFile. This is an
453    * expensive operation implemented as a linear-time search through all blocks
454    * in the cache. Ideally this should be a search in a log-access-time map.
455    *
456    * <p>
457    * This is used for evict-on-close to remove all blocks of a specific HFile.
458    *
459    * @return the number of blocks evicted
460    */
461   @Override
462   public int evictBlocksByHfileName(String hfileName) {
463     int numEvicted = 0;
464     for (BlockCacheKey key : map.keySet()) {
465       if (key.getHfileName().equals(hfileName)) {
466         if (evictBlock(key))
467           ++numEvicted;
468       }
469     }
470     if (victimHandler != null) {
471       numEvicted += victimHandler.evictBlocksByHfileName(hfileName);
472     }
473     return numEvicted;
474   }
475 
476   /**
477    * Evict the block, and it will be cached by the victim handler if exists &&
478    * block may be read again later
479    * @param block
480    * @param evictedByEvictionProcess true if the given block is evicted by
481    *          EvictionThread
482    * @return the heap size of evicted block
483    */
484   protected long evictBlock(LruCachedBlock block, boolean evictedByEvictionProcess) {
485     map.remove(block.getCacheKey());
486     updateSizeMetrics(block, true);
487     long val = elements.decrementAndGet();
488     if (LOG.isTraceEnabled()) {
489       long size = map.size();
490       assertCounterSanity(size, val);
491     }
492     stats.evicted(block.getCachedTime());
493     if (evictedByEvictionProcess && victimHandler != null) {
494       boolean wait = getCurrentSize() < acceptableSize();
495       boolean inMemory = block.getPriority() == BlockPriority.MEMORY;
496       victimHandler.cacheBlockWithWait(block.getCacheKey(), block.getBuffer(),
497           inMemory, wait);
498     }
499     return block.heapSize();
500   }
501 
502   /**
503    * Multi-threaded call to run the eviction process.
504    */
505   private void runEviction() {
506     if(evictionThread == null) {
507       evict();
508     } else {
509       evictionThread.evict();
510     }
511   }
512 
513   /**
514    * Eviction method.
515    */
516   void evict() {
517 
518     // Ensure only one eviction at a time
519     if(!evictionLock.tryLock()) return;
520 
521     try {
522       evictionInProgress = true;
523       long currentSize = this.size.get();
524       long bytesToFree = currentSize - minSize();
525 
526       if (LOG.isTraceEnabled()) {
527         LOG.trace("Block cache LRU eviction started; Attempting to free " +
528           StringUtils.byteDesc(bytesToFree) + " of total=" +
529           StringUtils.byteDesc(currentSize));
530       }
531 
532       if(bytesToFree <= 0) return;
533 
534       // Instantiate priority buckets
535       BlockBucket bucketSingle = new BlockBucket("single", bytesToFree, blockSize,
536           singleSize());
537       BlockBucket bucketMulti = new BlockBucket("multi", bytesToFree, blockSize,
538           multiSize());
539       BlockBucket bucketMemory = new BlockBucket("memory", bytesToFree, blockSize,
540           memorySize());
541 
542       // Scan entire map putting into appropriate buckets
543       for(LruCachedBlock cachedBlock : map.values()) {
544         switch(cachedBlock.getPriority()) {
545           case SINGLE: {
546             bucketSingle.add(cachedBlock);
547             break;
548           }
549           case MULTI: {
550             bucketMulti.add(cachedBlock);
551             break;
552           }
553           case MEMORY: {
554             bucketMemory.add(cachedBlock);
555             break;
556           }
557         }
558       }
559 
560       long bytesFreed = 0;
561       if (forceInMemory || memoryFactor > 0.999f) {
562         long s = bucketSingle.totalSize();
563         long m = bucketMulti.totalSize();
564         if (bytesToFree > (s + m)) {
565           // this means we need to evict blocks in memory bucket to make room,
566           // so the single and multi buckets will be emptied
567           bytesFreed = bucketSingle.free(s);
568           bytesFreed += bucketMulti.free(m);
569           if (LOG.isTraceEnabled()) {
570             LOG.trace("freed " + StringUtils.byteDesc(bytesFreed) +
571               " from single and multi buckets");
572           }
573           bytesFreed += bucketMemory.free(bytesToFree - bytesFreed);
574           if (LOG.isTraceEnabled()) {
575             LOG.trace("freed " + StringUtils.byteDesc(bytesFreed) +
576               " total from all three buckets ");
577           }
578         } else {
579           // this means no need to evict block in memory bucket,
580           // and we try best to make the ratio between single-bucket and
581           // multi-bucket is 1:2
582           long bytesRemain = s + m - bytesToFree;
583           if (3 * s <= bytesRemain) {
584             // single-bucket is small enough that no eviction happens for it
585             // hence all eviction goes from multi-bucket
586             bytesFreed = bucketMulti.free(bytesToFree);
587           } else if (3 * m <= 2 * bytesRemain) {
588             // multi-bucket is small enough that no eviction happens for it
589             // hence all eviction goes from single-bucket
590             bytesFreed = bucketSingle.free(bytesToFree);
591           } else {
592             // both buckets need to evict some blocks
593             bytesFreed = bucketSingle.free(s - bytesRemain / 3);
594             if (bytesFreed < bytesToFree) {
595               bytesFreed += bucketMulti.free(bytesToFree - bytesFreed);
596             }
597           }
598         }
599       } else {
600         PriorityQueue<BlockBucket> bucketQueue =
601           new PriorityQueue<BlockBucket>(3);
602 
603         bucketQueue.add(bucketSingle);
604         bucketQueue.add(bucketMulti);
605         bucketQueue.add(bucketMemory);
606 
607         int remainingBuckets = 3;
608 
609         BlockBucket bucket;
610         while((bucket = bucketQueue.poll()) != null) {
611           long overflow = bucket.overflow();
612           if(overflow > 0) {
613             long bucketBytesToFree = Math.min(overflow,
614                 (bytesToFree - bytesFreed) / remainingBuckets);
615             bytesFreed += bucket.free(bucketBytesToFree);
616           }
617           remainingBuckets--;
618         }
619       }
620 
621       if (LOG.isTraceEnabled()) {
622         long single = bucketSingle.totalSize();
623         long multi = bucketMulti.totalSize();
624         long memory = bucketMemory.totalSize();
625         LOG.trace("Block cache LRU eviction completed; " +
626           "freed=" + StringUtils.byteDesc(bytesFreed) + ", " +
627           "total=" + StringUtils.byteDesc(this.size.get()) + ", " +
628           "single=" + StringUtils.byteDesc(single) + ", " +
629           "multi=" + StringUtils.byteDesc(multi) + ", " +
630           "memory=" + StringUtils.byteDesc(memory));
631       }
632     } finally {
633       stats.evict();
634       evictionInProgress = false;
635       evictionLock.unlock();
636     }
637   }
638 
639   @Override
640   public String toString() {
641     return Objects.toStringHelper(this)
642       .add("blockCount", getBlockCount())
643       .add("currentSize", getCurrentSize())
644       .add("freeSize", getFreeSize())
645       .add("maxSize", getMaxSize())
646       .add("heapSize", heapSize())
647       .add("minSize", minSize())
648       .add("minFactor", minFactor)
649       .add("multiSize", multiSize())
650       .add("multiFactor", multiFactor)
651       .add("singleSize", singleSize())
652       .add("singleFactor", singleFactor)
653       .toString();
654   }
655 
656   /**
657    * Used to group blocks into priority buckets.  There will be a BlockBucket
658    * for each priority (single, multi, memory).  Once bucketed, the eviction
659    * algorithm takes the appropriate number of elements out of each according
660    * to configuration parameters and their relatives sizes.
661    */
662   private class BlockBucket implements Comparable<BlockBucket> {
663 
664     private final String name;
665     private LruCachedBlockQueue queue;
666     private long totalSize = 0;
667     private long bucketSize;
668 
669     public BlockBucket(String name, long bytesToFree, long blockSize, long bucketSize) {
670       this.name = name;
671       this.bucketSize = bucketSize;
672       queue = new LruCachedBlockQueue(bytesToFree, blockSize);
673       totalSize = 0;
674     }
675 
676     public void add(LruCachedBlock block) {
677       totalSize += block.heapSize();
678       queue.add(block);
679     }
680 
681     public long free(long toFree) {
682       if (LOG.isTraceEnabled()) {
683         LOG.trace("freeing " + StringUtils.byteDesc(toFree) + " from " + this);
684       }
685       LruCachedBlock cb;
686       long freedBytes = 0;
687       while ((cb = queue.pollLast()) != null) {
688         freedBytes += evictBlock(cb, true);
689         if (freedBytes >= toFree) {
690           return freedBytes;
691         }
692       }
693       if (LOG.isTraceEnabled()) {
694         LOG.trace("freed " + StringUtils.byteDesc(freedBytes) + " from " + this);
695       }
696       return freedBytes;
697     }
698 
699     public long overflow() {
700       return totalSize - bucketSize;
701     }
702 
703     public long totalSize() {
704       return totalSize;
705     }
706 
707     public int compareTo(BlockBucket that) {
708       if(this.overflow() == that.overflow()) return 0;
709       return this.overflow() > that.overflow() ? 1 : -1;
710     }
711 
712     @Override
713     public boolean equals(Object that) {
714       if (that == null || !(that instanceof BlockBucket)){
715         return false;
716       }
717       return compareTo((BlockBucket)that) == 0;
718     }
719 
720     @Override
721     public int hashCode() {
722       return Objects.hashCode(name, bucketSize, queue, totalSize);
723     }
724 
725     @Override
726     public String toString() {
727       return Objects.toStringHelper(this)
728         .add("name", name)
729         .add("totalSize", StringUtils.byteDesc(totalSize))
730         .add("bucketSize", StringUtils.byteDesc(bucketSize))
731         .toString();
732     }
733   }
734 
735   /**
736    * Get the maximum size of this cache.
737    * @return max size in bytes
738    */
739   public long getMaxSize() {
740     return this.maxSize;
741   }
742 
743   @Override
744   public long getCurrentSize() {
745     return this.size.get();
746   }
747 
748   @Override
749   public long getFreeSize() {
750     return getMaxSize() - getCurrentSize();
751   }
752 
753   @Override
754   public long size() {
755     return getMaxSize();
756   }
757 
758   @Override
759   public long getBlockCount() {
760     return this.elements.get();
761   }
762 
763   EvictionThread getEvictionThread() {
764     return this.evictionThread;
765   }
766 
767   /*
768    * Eviction thread.  Sits in waiting state until an eviction is triggered
769    * when the cache size grows above the acceptable level.<p>
770    *
771    * Thread is triggered into action by {@link LruBlockCache#runEviction()}
772    */
773   static class EvictionThread extends HasThread {
774     private WeakReference<LruBlockCache> cache;
775     private volatile boolean go = true;
776     // flag set after enter the run method, used for test
777     private boolean enteringRun = false;
778 
779     public EvictionThread(LruBlockCache cache) {
780       super(Thread.currentThread().getName() + ".LruBlockCache.EvictionThread");
781       setDaemon(true);
782       this.cache = new WeakReference<LruBlockCache>(cache);
783     }
784 
785     @Override
786     public void run() {
787       enteringRun = true;
788       while (this.go) {
789         synchronized(this) {
790           try {
791             this.wait(1000 * 10/*Don't wait for ever*/);
792           } catch(InterruptedException e) {}
793         }
794         LruBlockCache cache = this.cache.get();
795         if (cache == null) break;
796         cache.evict();
797       }
798     }
799 
800     @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NN_NAKED_NOTIFY",
801         justification="This is what we want")
802     public void evict() {
803       synchronized(this) {
804         this.notifyAll();
805       }
806     }
807 
808     synchronized void shutdown() {
809       this.go = false;
810       this.notifyAll();
811     }
812 
813     /**
814      * Used for the test.
815      */
816     boolean isEnteringRun() {
817       return this.enteringRun;
818     }
819   }
820 
821   /*
822    * Statistics thread.  Periodically prints the cache statistics to the log.
823    */
824   static class StatisticsThread extends Thread {
825     private final LruBlockCache lru;
826 
827     public StatisticsThread(LruBlockCache lru) {
828       super("LruBlockCacheStats");
829       setDaemon(true);
830       this.lru = lru;
831     }
832 
833     @Override
834     public void run() {
835       lru.logStats();
836     }
837   }
838 
839   public void logStats() {
840     // Log size
841     long totalSize = heapSize();
842     long freeSize = maxSize - totalSize;
843     LruBlockCache.LOG.info("totalSize=" + StringUtils.byteDesc(totalSize) + ", " +
844         "freeSize=" + StringUtils.byteDesc(freeSize) + ", " +
845         "max=" + StringUtils.byteDesc(this.maxSize) + ", " +
846         "blockCount=" + getBlockCount() + ", " +
847         "accesses=" + stats.getRequestCount() + ", " +
848         "hits=" + stats.getHitCount() + ", " +
849         "hitRatio=" + (stats.getHitCount() == 0 ?
850           "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) + ", " +
851         "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
852         "cachingHits=" + stats.getHitCachingCount() + ", " +
853         "cachingHitsRatio=" + (stats.getHitCachingCount() == 0 ?
854           "0,": (StringUtils.formatPercent(stats.getHitCachingRatio(), 2) + ", ")) +
855         "evictions=" + stats.getEvictionCount() + ", " +
856         "evicted=" + stats.getEvictedCount() + ", " +
857         "evictedPerRun=" + stats.evictedPerEviction());
858   }
859 
860   /**
861    * Get counter statistics for this cache.
862    *
863    * <p>Includes: total accesses, hits, misses, evicted blocks, and runs
864    * of the eviction processes.
865    */
866   public CacheStats getStats() {
867     return this.stats;
868   }
869 
870   public final static long CACHE_FIXED_OVERHEAD = ClassSize.align(
871       (3 * Bytes.SIZEOF_LONG) + (9 * ClassSize.REFERENCE) +
872       (5 * Bytes.SIZEOF_FLOAT) + Bytes.SIZEOF_BOOLEAN
873       + ClassSize.OBJECT);
874 
875   @Override
876   public long heapSize() {
877     return getCurrentSize();
878   }
879 
880   public static long calculateOverhead(long maxSize, long blockSize, int concurrency){
881     // FindBugs ICAST_INTEGER_MULTIPLY_CAST_TO_LONG
882     return CACHE_FIXED_OVERHEAD + ClassSize.CONCURRENT_HASHMAP +
883         ((long)Math.ceil(maxSize*1.2/blockSize)
884             * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
885         ((long)concurrency * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
886   }
887 
888   @Override
889   public Iterator<CachedBlock> iterator() {
890     final Iterator<LruCachedBlock> iterator = map.values().iterator();
891 
892     return new Iterator<CachedBlock>() {
893       private final long now = System.nanoTime();
894 
895       @Override
896       public boolean hasNext() {
897         return iterator.hasNext();
898       }
899 
900       @Override
901       public CachedBlock next() {
902         final LruCachedBlock b = iterator.next();
903         return new CachedBlock() {
904           @Override
905           public String toString() {
906             return BlockCacheUtil.toString(this, now);
907           }
908 
909           @Override
910           public BlockPriority getBlockPriority() {
911             return b.getPriority();
912           }
913 
914           @Override
915           public BlockType getBlockType() {
916             return b.getBuffer().getBlockType();
917           }
918 
919           @Override
920           public long getOffset() {
921             return b.getCacheKey().getOffset();
922           }
923 
924           @Override
925           public long getSize() {
926             return b.getBuffer().heapSize();
927           }
928 
929           @Override
930           public long getCachedTime() {
931             return b.getCachedTime();
932           }
933 
934           @Override
935           public String getFilename() {
936             return b.getCacheKey().getHfileName();
937           }
938 
939           @Override
940           public int compareTo(CachedBlock other) {
941             int diff = this.getFilename().compareTo(other.getFilename());
942             if (diff != 0) return diff;
943             diff = (int)(this.getOffset() - other.getOffset());
944             if (diff != 0) return diff;
945             if (other.getCachedTime() < 0 || this.getCachedTime() < 0) {
946               throw new IllegalStateException("" + this.getCachedTime() + ", " +
947                 other.getCachedTime());
948             }
949             return (int)(other.getCachedTime() - this.getCachedTime());
950           }
951 
952           @Override
953           public int hashCode() {
954             return b.hashCode();
955           }
956 
957           @Override
958           public boolean equals(Object obj) {
959             if (obj instanceof CachedBlock) {
960               CachedBlock cb = (CachedBlock)obj;
961               return compareTo(cb) == 0;
962             } else {
963               return false;
964             }
965           }
966         };
967       }
968 
969       @Override
970       public void remove() {
971         throw new UnsupportedOperationException();
972       }
973     };
974   }
975 
976   // Simple calculators of sizes given factors and maxSize
977 
978   long acceptableSize() {
979     return (long)Math.floor(this.maxSize * this.acceptableFactor);
980   }
981   private long minSize() {
982     return (long)Math.floor(this.maxSize * this.minFactor);
983   }
984   private long singleSize() {
985     return (long)Math.floor(this.maxSize * this.singleFactor * this.minFactor);
986   }
987   private long multiSize() {
988     return (long)Math.floor(this.maxSize * this.multiFactor * this.minFactor);
989   }
990   private long memorySize() {
991     return (long)Math.floor(this.maxSize * this.memoryFactor * this.minFactor);
992   }
993 
994   public void shutdown() {
995     if (victimHandler != null)
996       victimHandler.shutdown();
997     this.scheduleThreadPool.shutdown();
998     for (int i = 0; i < 10; i++) {
999       if (!this.scheduleThreadPool.isShutdown()) {
1000         try {
1001           Thread.sleep(10);
1002         } catch (InterruptedException e) {
1003           LOG.warn("Interrupted while sleeping");
1004           Thread.currentThread().interrupt();
1005           break;
1006         }
1007       }
1008     }
1009 
1010     if (!this.scheduleThreadPool.isShutdown()) {
1011       List<Runnable> runnables = this.scheduleThreadPool.shutdownNow();
1012       LOG.debug("Still running " + runnables);
1013     }
1014     this.evictionThread.shutdown();
1015   }
1016 
1017   /** Clears the cache. Used in tests. */
1018   @VisibleForTesting
1019   public void clearCache() {
1020     this.map.clear();
1021     this.elements.set(0);
1022   }
1023 
1024   /**
1025    * Used in testing. May be very inefficient.
1026    * @return the set of cached file names
1027    */
1028   @VisibleForTesting
1029   SortedSet<String> getCachedFileNamesForTest() {
1030     SortedSet<String> fileNames = new TreeSet<String>();
1031     for (BlockCacheKey cacheKey : map.keySet()) {
1032       fileNames.add(cacheKey.getHfileName());
1033     }
1034     return fileNames;
1035   }
1036 
1037   @VisibleForTesting
1038   Map<BlockType, Integer> getBlockTypeCountsForTest() {
1039     Map<BlockType, Integer> counts =
1040         new EnumMap<BlockType, Integer>(BlockType.class);
1041     for (LruCachedBlock cb : map.values()) {
1042       BlockType blockType = ((Cacheable)cb.getBuffer()).getBlockType();
1043       Integer count = counts.get(blockType);
1044       counts.put(blockType, (count == null ? 0 : count) + 1);
1045     }
1046     return counts;
1047   }
1048 
1049   @VisibleForTesting
1050   public Map<DataBlockEncoding, Integer> getEncodingCountsForTest() {
1051     Map<DataBlockEncoding, Integer> counts =
1052         new EnumMap<DataBlockEncoding, Integer>(DataBlockEncoding.class);
1053     for (LruCachedBlock block : map.values()) {
1054       DataBlockEncoding encoding =
1055               ((HFileBlock) block.getBuffer()).getDataBlockEncoding();
1056       Integer count = counts.get(encoding);
1057       counts.put(encoding, (count == null ? 0 : count) + 1);
1058     }
1059     return counts;
1060   }
1061 
1062   public void setVictimCache(BucketCache handler) {
1063     assert victimHandler == null;
1064     victimHandler = handler;
1065   }
1066 
1067   @VisibleForTesting
1068   Map<BlockCacheKey, LruCachedBlock> getMapForTests() {
1069     return map;
1070   }
1071 
1072   BucketCache getVictimHandler() {
1073     return this.victimHandler;
1074   }
1075 
1076   @Override
1077   public BlockCache[] getBlockCaches() {
1078     return null;
1079   }
1080 }