1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.IOException;
22 import java.lang.ref.WeakReference;
23 import java.util.ArrayList;
24 import java.util.Collections;
25 import java.util.EnumMap;
26 import java.util.HashMap;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.PriorityQueue;
30 import java.util.SortedSet;
31 import java.util.TreeSet;
32 import java.util.concurrent.ConcurrentHashMap;
33 import java.util.concurrent.Executors;
34 import java.util.concurrent.ScheduledExecutorService;
35 import java.util.concurrent.TimeUnit;
36 import java.util.concurrent.atomic.AtomicLong;
37 import java.util.concurrent.locks.ReentrantLock;
38
39 import org.apache.commons.logging.Log;
40 import org.apache.commons.logging.LogFactory;
41 import org.apache.hadoop.classification.InterfaceAudience;
42 import org.apache.hadoop.conf.Configuration;
43 import org.apache.hadoop.fs.FileSystem;
44 import org.apache.hadoop.fs.Path;
45 import org.apache.hadoop.hbase.io.HeapSize;
46 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
47 import org.apache.hadoop.hbase.io.hfile.CachedBlock.BlockPriority;
48 import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.ClassSize;
51 import org.apache.hadoop.hbase.util.FSUtils;
52 import org.apache.hadoop.hbase.util.HasThread;
53 import org.apache.hadoop.hbase.util.Threads;
54 import org.apache.hadoop.util.StringUtils;
55
56 import com.google.common.util.concurrent.ThreadFactoryBuilder;
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96 @InterfaceAudience.Private
97 public class LruBlockCache implements BlockCache, HeapSize {
98
99 static final Log LOG = LogFactory.getLog(LruBlockCache.class);
100
101 static final String LRU_MIN_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.min.factor";
102 static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.acceptable.factor";
103
104
105
106
107 static final float DEFAULT_LOAD_FACTOR = 0.75f;
108 static final int DEFAULT_CONCURRENCY_LEVEL = 16;
109
110
111 static final float DEFAULT_MIN_FACTOR = 0.95f;
112 static final float DEFAULT_ACCEPTABLE_FACTOR = 0.99f;
113
114
115 static final float DEFAULT_SINGLE_FACTOR = 0.25f;
116 static final float DEFAULT_MULTI_FACTOR = 0.50f;
117 static final float DEFAULT_MEMORY_FACTOR = 0.25f;
118
119
120 static final int statThreadPeriod = 60 * 5;
121
122
123 private final ConcurrentHashMap<BlockCacheKey,CachedBlock> map;
124
125
126 private final ReentrantLock evictionLock = new ReentrantLock(true);
127
128
129 private volatile boolean evictionInProgress = false;
130
131
132 private final EvictionThread evictionThread;
133
134
135 private final ScheduledExecutorService scheduleThreadPool =
136 Executors.newScheduledThreadPool(1,
137 new ThreadFactoryBuilder()
138 .setNameFormat("LRU Statistics #%d")
139 .setDaemon(true)
140 .build());
141
142
143 private final AtomicLong size;
144
145
146 private final AtomicLong elements;
147
148
149 private final AtomicLong count;
150
151
152 private final CacheStats stats;
153
154
155 private long maxSize;
156
157
158 private long blockSize;
159
160
161 private float acceptableFactor;
162
163
164 private float minFactor;
165
166
167 private float singleFactor;
168
169
170 private float multiFactor;
171
172
173 private float memoryFactor;
174
175
176 private long overhead;
177
178
179 private BucketCache victimHandler = null;
180
181
182
183
184
185
186
187
188
189
190 public LruBlockCache(long maxSize, long blockSize) {
191 this(maxSize, blockSize, true);
192 }
193
194
195
196
197 public LruBlockCache(long maxSize, long blockSize, boolean evictionThread) {
198 this(maxSize, blockSize, evictionThread,
199 (int)Math.ceil(1.2*maxSize/blockSize),
200 DEFAULT_LOAD_FACTOR, DEFAULT_CONCURRENCY_LEVEL,
201 DEFAULT_MIN_FACTOR, DEFAULT_ACCEPTABLE_FACTOR,
202 DEFAULT_SINGLE_FACTOR, DEFAULT_MULTI_FACTOR,
203 DEFAULT_MEMORY_FACTOR);
204 }
205
206 public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, Configuration conf) {
207 this(maxSize, blockSize, evictionThread,
208 (int)Math.ceil(1.2*maxSize/blockSize),
209 DEFAULT_LOAD_FACTOR,
210 DEFAULT_CONCURRENCY_LEVEL,
211 conf.getFloat(LRU_MIN_FACTOR_CONFIG_NAME, DEFAULT_MIN_FACTOR),
212 conf.getFloat(LRU_ACCEPTABLE_FACTOR_CONFIG_NAME, DEFAULT_ACCEPTABLE_FACTOR),
213 DEFAULT_SINGLE_FACTOR,
214 DEFAULT_MULTI_FACTOR,
215 DEFAULT_MEMORY_FACTOR);
216 }
217
218 public LruBlockCache(long maxSize, long blockSize, Configuration conf) {
219 this(maxSize, blockSize, true, conf);
220 }
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236 public LruBlockCache(long maxSize, long blockSize, boolean evictionThread,
237 int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel,
238 float minFactor, float acceptableFactor,
239 float singleFactor, float multiFactor, float memoryFactor) {
240 if(singleFactor + multiFactor + memoryFactor != 1) {
241 throw new IllegalArgumentException("Single, multi, and memory factors " +
242 " should total 1.0");
243 }
244 if(minFactor >= acceptableFactor) {
245 throw new IllegalArgumentException("minFactor must be smaller than acceptableFactor");
246 }
247 if(minFactor >= 1.0f || acceptableFactor >= 1.0f) {
248 throw new IllegalArgumentException("all factors must be < 1");
249 }
250 this.maxSize = maxSize;
251 this.blockSize = blockSize;
252 map = new ConcurrentHashMap<BlockCacheKey,CachedBlock>(mapInitialSize,
253 mapLoadFactor, mapConcurrencyLevel);
254 this.minFactor = minFactor;
255 this.acceptableFactor = acceptableFactor;
256 this.singleFactor = singleFactor;
257 this.multiFactor = multiFactor;
258 this.memoryFactor = memoryFactor;
259 this.stats = new CacheStats();
260 this.count = new AtomicLong(0);
261 this.elements = new AtomicLong(0);
262 this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel);
263 this.size = new AtomicLong(this.overhead);
264 if(evictionThread) {
265 this.evictionThread = new EvictionThread(this);
266 this.evictionThread.start();
267 } else {
268 this.evictionThread = null;
269 }
270 this.scheduleThreadPool.scheduleAtFixedRate(new StatisticsThread(this),
271 statThreadPeriod, statThreadPeriod, TimeUnit.SECONDS);
272 }
273
274 public void setMaxSize(long maxSize) {
275 this.maxSize = maxSize;
276 if(this.size.get() > acceptableSize() && !evictionInProgress) {
277 runEviction();
278 }
279 }
280
281
282
283
284
285
286
287
288
289
290
291
292 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
293 CachedBlock cb = map.get(cacheKey);
294 if(cb != null) {
295 throw new RuntimeException("Cached an already cached block");
296 }
297 cb = new CachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory);
298 long newSize = updateSizeMetrics(cb, false);
299 map.put(cacheKey, cb);
300 elements.incrementAndGet();
301 if(newSize > acceptableSize() && !evictionInProgress) {
302 runEviction();
303 }
304 }
305
306
307
308
309
310
311
312
313
314
315
316 public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
317 cacheBlock(cacheKey, buf, false);
318 }
319
320
321
322
323
324
325
326
327
328 protected long updateSizeMetrics(CachedBlock cb, boolean evict) {
329 long heapsize = cb.heapSize();
330 if (evict) {
331 heapsize *= -1;
332 }
333 return size.addAndGet(heapsize);
334 }
335
336
337
338
339
340
341
342
343
344
345 @Override
346 public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) {
347 CachedBlock cb = map.get(cacheKey);
348 if(cb == null) {
349 if (!repeat) stats.miss(caching);
350 if (victimHandler != null)
351 return victimHandler.getBlock(cacheKey, caching, repeat);
352 return null;
353 }
354 stats.hit(caching);
355 cb.access(count.incrementAndGet());
356 return cb.getBuffer();
357 }
358
359
360
361
362
363
364 public boolean containsBlock(BlockCacheKey cacheKey) {
365 return map.containsKey(cacheKey);
366 }
367
368 @Override
369 public boolean evictBlock(BlockCacheKey cacheKey) {
370 CachedBlock cb = map.get(cacheKey);
371 if (cb == null) return false;
372 evictBlock(cb, false);
373 return true;
374 }
375
376
377
378
379
380
381
382
383
384
385
386 @Override
387 public int evictBlocksByHfileName(String hfileName) {
388 int numEvicted = 0;
389 for (BlockCacheKey key : map.keySet()) {
390 if (key.getHfileName().equals(hfileName)) {
391 if (evictBlock(key))
392 ++numEvicted;
393 }
394 }
395 if (victimHandler != null) {
396 numEvicted += victimHandler.evictBlocksByHfileName(hfileName);
397 }
398 return numEvicted;
399 }
400
401
402
403
404
405
406
407
408
409 protected long evictBlock(CachedBlock block, boolean evictedByEvictionProcess) {
410 map.remove(block.getCacheKey());
411 updateSizeMetrics(block, true);
412 elements.decrementAndGet();
413 stats.evicted();
414 if (evictedByEvictionProcess && victimHandler != null) {
415 boolean wait = getCurrentSize() < acceptableSize();
416 boolean inMemory = block.getPriority() == BlockPriority.MEMORY;
417 victimHandler.cacheBlockWithWait(block.getCacheKey(), block.getBuffer(),
418 inMemory, wait);
419 }
420 return block.heapSize();
421 }
422
423
424
425
426 private void runEviction() {
427 if(evictionThread == null) {
428 evict();
429 } else {
430 evictionThread.evict();
431 }
432 }
433
434
435
436
437 void evict() {
438
439
440 if(!evictionLock.tryLock()) return;
441
442 try {
443 evictionInProgress = true;
444 long currentSize = this.size.get();
445 long bytesToFree = currentSize - minSize();
446
447 if (LOG.isDebugEnabled()) {
448 LOG.debug("Block cache LRU eviction started; Attempting to free " +
449 StringUtils.byteDesc(bytesToFree) + " of total=" +
450 StringUtils.byteDesc(currentSize));
451 }
452
453 if(bytesToFree <= 0) return;
454
455
456 BlockBucket bucketSingle = new BlockBucket(bytesToFree, blockSize,
457 singleSize());
458 BlockBucket bucketMulti = new BlockBucket(bytesToFree, blockSize,
459 multiSize());
460 BlockBucket bucketMemory = new BlockBucket(bytesToFree, blockSize,
461 memorySize());
462
463
464 for(CachedBlock cachedBlock : map.values()) {
465 switch(cachedBlock.getPriority()) {
466 case SINGLE: {
467 bucketSingle.add(cachedBlock);
468 break;
469 }
470 case MULTI: {
471 bucketMulti.add(cachedBlock);
472 break;
473 }
474 case MEMORY: {
475 bucketMemory.add(cachedBlock);
476 break;
477 }
478 }
479 }
480
481 PriorityQueue<BlockBucket> bucketQueue =
482 new PriorityQueue<BlockBucket>(3);
483
484 bucketQueue.add(bucketSingle);
485 bucketQueue.add(bucketMulti);
486 bucketQueue.add(bucketMemory);
487
488 int remainingBuckets = 3;
489 long bytesFreed = 0;
490
491 BlockBucket bucket;
492 while((bucket = bucketQueue.poll()) != null) {
493 long overflow = bucket.overflow();
494 if(overflow > 0) {
495 long bucketBytesToFree = Math.min(overflow,
496 (bytesToFree - bytesFreed) / remainingBuckets);
497 bytesFreed += bucket.free(bucketBytesToFree);
498 }
499 remainingBuckets--;
500 }
501
502 if (LOG.isDebugEnabled()) {
503 long single = bucketSingle.totalSize();
504 long multi = bucketMulti.totalSize();
505 long memory = bucketMemory.totalSize();
506 LOG.debug("Block cache LRU eviction completed; " +
507 "freed=" + StringUtils.byteDesc(bytesFreed) + ", " +
508 "total=" + StringUtils.byteDesc(this.size.get()) + ", " +
509 "single=" + StringUtils.byteDesc(single) + ", " +
510 "multi=" + StringUtils.byteDesc(multi) + ", " +
511 "memory=" + StringUtils.byteDesc(memory));
512 }
513 } finally {
514 stats.evict();
515 evictionInProgress = false;
516 evictionLock.unlock();
517 }
518 }
519
520
521
522
523
524
525
526 private class BlockBucket implements Comparable<BlockBucket> {
527
528 private CachedBlockQueue queue;
529 private long totalSize = 0;
530 private long bucketSize;
531
532 public BlockBucket(long bytesToFree, long blockSize, long bucketSize) {
533 this.bucketSize = bucketSize;
534 queue = new CachedBlockQueue(bytesToFree, blockSize);
535 totalSize = 0;
536 }
537
538 public void add(CachedBlock block) {
539 totalSize += block.heapSize();
540 queue.add(block);
541 }
542
543 public long free(long toFree) {
544 CachedBlock cb;
545 long freedBytes = 0;
546 while ((cb = queue.pollLast()) != null) {
547 freedBytes += evictBlock(cb, true);
548 if (freedBytes >= toFree) {
549 return freedBytes;
550 }
551 }
552 return freedBytes;
553 }
554
555 public long overflow() {
556 return totalSize - bucketSize;
557 }
558
559 public long totalSize() {
560 return totalSize;
561 }
562
563 public int compareTo(BlockBucket that) {
564 if(this.overflow() == that.overflow()) return 0;
565 return this.overflow() > that.overflow() ? 1 : -1;
566 }
567
568 @Override
569 public boolean equals(Object that) {
570 if (that == null || !(that instanceof BlockBucket)){
571 return false;
572 }
573
574 return compareTo(( BlockBucket)that) == 0;
575 }
576
577 }
578
579
580
581
582
583 public long getMaxSize() {
584 return this.maxSize;
585 }
586
587
588
589
590
591 public long getCurrentSize() {
592 return this.size.get();
593 }
594
595
596
597
598
599 public long getFreeSize() {
600 return getMaxSize() - getCurrentSize();
601 }
602
603
604
605
606
607 public long size() {
608 return this.elements.get();
609 }
610
611 @Override
612 public long getBlockCount() {
613 return this.elements.get();
614 }
615
616
617
618
619 public long getEvictionCount() {
620 return this.stats.getEvictionCount();
621 }
622
623
624
625
626
627 public long getEvictedCount() {
628 return this.stats.getEvictedCount();
629 }
630
631 EvictionThread getEvictionThread() {
632 return this.evictionThread;
633 }
634
635
636
637
638
639
640
641 static class EvictionThread extends HasThread {
642 private WeakReference<LruBlockCache> cache;
643 private boolean go = true;
644
645 private boolean enteringRun = false;
646
647 public EvictionThread(LruBlockCache cache) {
648 super(Thread.currentThread().getName() + ".LruBlockCache.EvictionThread");
649 setDaemon(true);
650 this.cache = new WeakReference<LruBlockCache>(cache);
651 }
652
653 @Override
654 public void run() {
655 enteringRun = true;
656 while (this.go) {
657 synchronized(this) {
658 try {
659 this.wait();
660 } catch(InterruptedException e) {}
661 }
662 LruBlockCache cache = this.cache.get();
663 if(cache == null) break;
664 cache.evict();
665 }
666 }
667
668 public void evict() {
669 synchronized(this) {
670 this.notifyAll();
671 }
672 }
673
674 synchronized void shutdown() {
675 this.go = false;
676 this.notifyAll();
677 }
678
679
680
681
682 boolean isEnteringRun() {
683 return this.enteringRun;
684 }
685 }
686
687
688
689
690 static class StatisticsThread extends Thread {
691 LruBlockCache lru;
692
693 public StatisticsThread(LruBlockCache lru) {
694 super("LruBlockCache.StatisticsThread");
695 setDaemon(true);
696 this.lru = lru;
697 }
698 @Override
699 public void run() {
700 lru.logStats();
701 }
702 }
703
704 public void logStats() {
705 if (!LOG.isDebugEnabled()) return;
706
707 long totalSize = heapSize();
708 long freeSize = maxSize - totalSize;
709 LruBlockCache.LOG.debug("Stats: " +
710 "total=" + StringUtils.byteDesc(totalSize) + ", " +
711 "free=" + StringUtils.byteDesc(freeSize) + ", " +
712 "max=" + StringUtils.byteDesc(this.maxSize) + ", " +
713 "blocks=" + size() +", " +
714 "accesses=" + stats.getRequestCount() + ", " +
715 "hits=" + stats.getHitCount() + ", " +
716 "hitRatio=" +
717 (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitRatio(), 2)+ ", ")) + ", " +
718 "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
719 "cachingHits=" + stats.getHitCachingCount() + ", " +
720 "cachingHitsRatio=" +
721 (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent(stats.getHitCachingRatio(), 2)+ ", ")) + ", " +
722 "evictions=" + stats.getEvictionCount() + ", " +
723 "evicted=" + stats.getEvictedCount() + ", " +
724 "evictedPerRun=" + stats.evictedPerEviction());
725 }
726
727
728
729
730
731
732
733 public CacheStats getStats() {
734 return this.stats;
735 }
736
737 public final static long CACHE_FIXED_OVERHEAD = ClassSize.align(
738 (3 * Bytes.SIZEOF_LONG) + (9 * ClassSize.REFERENCE) +
739 (5 * Bytes.SIZEOF_FLOAT) + Bytes.SIZEOF_BOOLEAN
740 + ClassSize.OBJECT);
741
742
743 public long heapSize() {
744 return getCurrentSize();
745 }
746
747 public static long calculateOverhead(long maxSize, long blockSize, int concurrency){
748
749 return CACHE_FIXED_OVERHEAD + ClassSize.CONCURRENT_HASHMAP +
750 ((long)Math.ceil(maxSize*1.2/blockSize)
751 * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
752 ((long)concurrency * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
753 }
754
755 @Override
756 public List<BlockCacheColumnFamilySummary> getBlockCacheColumnFamilySummaries(Configuration conf) throws IOException {
757
758 Map<String, Path> sfMap = FSUtils.getTableStoreFilePathMap(
759 FileSystem.get(conf),
760 FSUtils.getRootDir(conf));
761
762
763
764 Map<BlockCacheColumnFamilySummary, BlockCacheColumnFamilySummary> bcs =
765 new HashMap<BlockCacheColumnFamilySummary, BlockCacheColumnFamilySummary>();
766
767 for (CachedBlock cb : map.values()) {
768 String sf = cb.getCacheKey().getHfileName();
769 Path path = sfMap.get(sf);
770 if ( path != null) {
771 BlockCacheColumnFamilySummary lookup =
772 BlockCacheColumnFamilySummary.createFromStoreFilePath(path);
773 BlockCacheColumnFamilySummary bcse = bcs.get(lookup);
774 if (bcse == null) {
775 bcse = BlockCacheColumnFamilySummary.create(lookup);
776 bcs.put(lookup,bcse);
777 }
778 bcse.incrementBlocks();
779 bcse.incrementHeapSize(cb.heapSize());
780 }
781 }
782 List<BlockCacheColumnFamilySummary> list =
783 new ArrayList<BlockCacheColumnFamilySummary>(bcs.values());
784 Collections.sort( list );
785 return list;
786 }
787
788
789
790 private long acceptableSize() {
791 return (long)Math.floor(this.maxSize * this.acceptableFactor);
792 }
793 private long minSize() {
794 return (long)Math.floor(this.maxSize * this.minFactor);
795 }
796 private long singleSize() {
797 return (long)Math.floor(this.maxSize * this.singleFactor * this.minFactor);
798 }
799 private long multiSize() {
800 return (long)Math.floor(this.maxSize * this.multiFactor * this.minFactor);
801 }
802 private long memorySize() {
803 return (long)Math.floor(this.maxSize * this.memoryFactor * this.minFactor);
804 }
805
806 public void shutdown() {
807 if (victimHandler != null)
808 victimHandler.shutdown();
809 this.scheduleThreadPool.shutdown();
810 for (int i = 0; i < 10; i++) {
811 if (!this.scheduleThreadPool.isShutdown()) Threads.sleep(10);
812 }
813 if (!this.scheduleThreadPool.isShutdown()) {
814 List<Runnable> runnables = this.scheduleThreadPool.shutdownNow();
815 LOG.debug("Still running " + runnables);
816 }
817 this.evictionThread.shutdown();
818 }
819
820
821 public void clearCache() {
822 map.clear();
823 }
824
825
826
827
828
829 SortedSet<String> getCachedFileNamesForTest() {
830 SortedSet<String> fileNames = new TreeSet<String>();
831 for (BlockCacheKey cacheKey : map.keySet()) {
832 fileNames.add(cacheKey.getHfileName());
833 }
834 return fileNames;
835 }
836
837 Map<BlockType, Integer> getBlockTypeCountsForTest() {
838 Map<BlockType, Integer> counts =
839 new EnumMap<BlockType, Integer>(BlockType.class);
840 for (CachedBlock cb : map.values()) {
841 BlockType blockType = ((HFileBlock) cb.getBuffer()).getBlockType();
842 Integer count = counts.get(blockType);
843 counts.put(blockType, (count == null ? 0 : count) + 1);
844 }
845 return counts;
846 }
847
848 public Map<DataBlockEncoding, Integer> getEncodingCountsForTest() {
849 Map<DataBlockEncoding, Integer> counts =
850 new EnumMap<DataBlockEncoding, Integer>(DataBlockEncoding.class);
851 for (BlockCacheKey cacheKey : map.keySet()) {
852 DataBlockEncoding encoding = cacheKey.getDataBlockEncoding();
853 Integer count = counts.get(encoding);
854 counts.put(encoding, (count == null ? 0 : count) + 1);
855 }
856 return counts;
857 }
858
859 public void setVictimCache(BucketCache handler) {
860 assert victimHandler == null;
861 victimHandler = handler;
862 }
863
864 }