1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23
24 import java.nio.ByteBuffer;
25 import java.util.Collection;
26 import java.util.Map;
27 import java.util.Random;
28
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.SmallTests;
31 import org.apache.hadoop.hbase.io.HeapSize;
32 import org.apache.hadoop.hbase.io.hfile.LruBlockCache.EvictionThread;
33 import org.apache.hadoop.hbase.util.ClassSize;
34 import org.junit.After;
35 import org.junit.Before;
36 import org.junit.Test;
37 import org.junit.experimental.categories.Category;
38 import org.junit.runner.RunWith;
39 import org.junit.runners.Parameterized;
40 import org.junit.runners.Parameterized.Parameters;
41
42
43
44
45
46
47
48
49 @Category(SmallTests.class)
50 public class TestLruBlockCache {
51
52
53 @Test
54 public void testBackgroundEvictionThread() throws Exception {
55 long maxSize = 100000;
56 long blockSize = calculateBlockSizeDefault(maxSize, 9);
57
58 LruBlockCache cache = new LruBlockCache(maxSize,blockSize);
59
60 CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
61
62 EvictionThread evictionThread = cache.getEvictionThread();
63 assertTrue(evictionThread != null);
64
65
66 while (!evictionThread.isEnteringRun()) {
67 Thread.sleep(1);
68 }
69
70
71
72 for (CachedItem block : blocks) {
73 cache.cacheBlock(block.cacheKey, block);
74 }
75
76
77 int n = 0;
78 while(cache.getEvictionCount() == 0) {
79 Thread.sleep(200);
80 assertTrue(n++ < 20);
81 }
82 System.out.println("Background Evictions run: " + cache.getEvictionCount());
83
84
85 assertEquals(cache.getEvictionCount(), 1);
86 }
87
88 @Test
89 public void testCacheSimple() throws Exception {
90
91 long maxSize = 1000000;
92 long blockSize = calculateBlockSizeDefault(maxSize, 101);
93
94 LruBlockCache cache = new LruBlockCache(maxSize, blockSize);
95
96 CachedItem [] blocks = generateRandomBlocks(100, blockSize);
97
98 long expectedCacheSize = cache.heapSize();
99
100
101 for (CachedItem block : blocks) {
102 assertTrue(cache.getBlock(block.cacheKey, true, false) == null);
103 }
104
105
106 for (CachedItem block : blocks) {
107 cache.cacheBlock(block.cacheKey, block);
108 expectedCacheSize += block.cacheBlockHeapSize();
109 }
110
111
112 assertEquals(expectedCacheSize, cache.heapSize());
113
114
115 for (CachedItem block : blocks) {
116 HeapSize buf = cache.getBlock(block.cacheKey, true, false);
117 assertTrue(buf != null);
118 assertEquals(buf.heapSize(), block.heapSize());
119 }
120
121
122 assertEquals(expectedCacheSize, cache.heapSize());
123
124
125 for (CachedItem block : blocks) {
126 HeapSize buf = cache.getBlock(block.cacheKey, true, false);
127 assertTrue(buf != null);
128 assertEquals(buf.heapSize(), block.heapSize());
129 }
130
131
132 assertEquals(0, cache.getEvictionCount());
133 Thread t = new LruBlockCache.StatisticsThread(cache);
134 t.start();
135 t.join();
136 }
137
138 @Test
139 public void testCacheEvictionSimple() throws Exception {
140
141 long maxSize = 100000;
142 long blockSize = calculateBlockSizeDefault(maxSize, 10);
143
144 LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false);
145
146 CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
147
148 long expectedCacheSize = cache.heapSize();
149
150
151 for (CachedItem block : blocks) {
152 cache.cacheBlock(block.cacheKey, block);
153 expectedCacheSize += block.cacheBlockHeapSize();
154 }
155
156
157 assertEquals(1, cache.getEvictionCount());
158
159
160 assertTrue(expectedCacheSize >
161 (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
162
163
164 assertTrue(cache.heapSize() < maxSize);
165
166
167 assertTrue(cache.heapSize() <
168 (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
169
170
171 assertTrue(cache.getBlock(blocks[0].cacheKey, true, false) == null);
172 for(int i=1;i<blocks.length;i++) {
173 assertEquals(cache.getBlock(blocks[i].cacheKey, true, false),
174 blocks[i]);
175 }
176 }
177
178 @Test
179 public void testCacheEvictionTwoPriorities() throws Exception {
180
181 long maxSize = 100000;
182 long blockSize = calculateBlockSizeDefault(maxSize, 10);
183
184 LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false);
185
186 CachedItem [] singleBlocks = generateFixedBlocks(5, 10000, "single");
187 CachedItem [] multiBlocks = generateFixedBlocks(5, 10000, "multi");
188
189 long expectedCacheSize = cache.heapSize();
190
191
192 for (CachedItem block : multiBlocks) {
193 cache.cacheBlock(block.cacheKey, block);
194 expectedCacheSize += block.cacheBlockHeapSize();
195 assertEquals(cache.getBlock(block.cacheKey, true, false), block);
196 }
197
198
199 for (CachedItem block : singleBlocks) {
200 cache.cacheBlock(block.cacheKey, block);
201 expectedCacheSize += block.heapSize();
202 }
203
204
205 assertEquals(cache.getEvictionCount(), 1);
206
207
208 assertEquals(cache.getEvictedCount(), 2);
209
210
211 assertTrue(expectedCacheSize >
212 (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
213
214
215 assertTrue(cache.heapSize() <= maxSize);
216
217
218 assertTrue(cache.heapSize() <=
219 (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
220
221
222
223
224
225 assertTrue(cache.getBlock(singleBlocks[0].cacheKey, true, false) == null);
226 assertTrue(cache.getBlock(multiBlocks[0].cacheKey, true, false) == null);
227
228
229 for(int i=1;i<4;i++) {
230 assertEquals(cache.getBlock(singleBlocks[i].cacheKey, true, false),
231 singleBlocks[i]);
232 assertEquals(cache.getBlock(multiBlocks[i].cacheKey, true, false),
233 multiBlocks[i]);
234 }
235 }
236
237 @Test
238 public void testCacheEvictionThreePriorities() throws Exception {
239
240 long maxSize = 100000;
241 long blockSize = calculateBlockSize(maxSize, 10);
242
243 LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false,
244 (int)Math.ceil(1.2*maxSize/blockSize),
245 LruBlockCache.DEFAULT_LOAD_FACTOR,
246 LruBlockCache.DEFAULT_CONCURRENCY_LEVEL,
247 0.98f,
248 0.99f,
249 0.33f,
250 0.33f,
251 0.34f);
252
253
254 CachedItem [] singleBlocks = generateFixedBlocks(5, blockSize, "single");
255 CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
256 CachedItem [] memoryBlocks = generateFixedBlocks(5, blockSize, "memory");
257
258 long expectedCacheSize = cache.heapSize();
259
260
261 for(int i=0;i<3;i++) {
262
263
264 cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
265 expectedCacheSize += singleBlocks[i].cacheBlockHeapSize();
266
267
268 cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
269 expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
270 cache.getBlock(multiBlocks[i].cacheKey, true, false);
271
272
273 cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
274 expectedCacheSize += memoryBlocks[i].cacheBlockHeapSize();
275
276 }
277
278
279 assertEquals(0, cache.getEvictionCount());
280
281
282 assertEquals(expectedCacheSize, cache.heapSize());
283
284
285 cache.cacheBlock(singleBlocks[3].cacheKey, singleBlocks[3]);
286
287
288 assertEquals(1, cache.getEvictionCount());
289 assertEquals(1, cache.getEvictedCount());
290
291
292 assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false));
293
294
295 cache.getBlock(singleBlocks[1].cacheKey, true, false);
296
297
298 cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]);
299
300
301 assertEquals(2, cache.getEvictionCount());
302 assertEquals(2, cache.getEvictedCount());
303
304
305 assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false));
306
307
308 cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true);
309
310
311 assertEquals(3, cache.getEvictionCount());
312 assertEquals(3, cache.getEvictedCount());
313
314
315 assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false));
316
317
318 CachedItem [] bigBlocks = generateFixedBlocks(3, blockSize*3, "big");
319 cache.cacheBlock(bigBlocks[0].cacheKey, bigBlocks[0]);
320
321
322 assertEquals(4, cache.getEvictionCount());
323 assertEquals(6, cache.getEvictedCount());
324
325
326 assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false));
327 assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false));
328 assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false));
329
330
331 cache.getBlock(bigBlocks[0].cacheKey, true, false);
332
333
334 cache.cacheBlock(bigBlocks[1].cacheKey, bigBlocks[1]);
335
336
337 assertEquals(5, cache.getEvictionCount());
338 assertEquals(9, cache.getEvictedCount());
339
340
341 assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false));
342 assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false));
343 assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false));
344
345
346 cache.cacheBlock(bigBlocks[2].cacheKey, bigBlocks[2], true);
347
348
349 assertEquals(6, cache.getEvictionCount());
350 assertEquals(12, cache.getEvictedCount());
351
352
353 assertEquals(null, cache.getBlock(memoryBlocks[1].cacheKey, true, false));
354 assertEquals(null, cache.getBlock(memoryBlocks[2].cacheKey, true, false));
355 assertEquals(null, cache.getBlock(memoryBlocks[3].cacheKey, true, false));
356
357
358 }
359
360
361 @Test
362 public void testScanResistance() throws Exception {
363
364 long maxSize = 100000;
365 long blockSize = calculateBlockSize(maxSize, 10);
366
367 LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false,
368 (int)Math.ceil(1.2*maxSize/blockSize),
369 LruBlockCache.DEFAULT_LOAD_FACTOR,
370 LruBlockCache.DEFAULT_CONCURRENCY_LEVEL,
371 0.66f,
372 0.99f,
373 0.33f,
374 0.33f,
375 0.34f);
376
377 CachedItem [] singleBlocks = generateFixedBlocks(20, blockSize, "single");
378 CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
379
380
381 for (CachedItem block : multiBlocks) {
382 cache.cacheBlock(block.cacheKey, block);
383 cache.getBlock(block.cacheKey, true, false);
384 }
385
386
387 for(int i=0;i<5;i++) {
388 cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
389 }
390
391
392 assertEquals(1, cache.getEvictionCount());
393
394
395 assertEquals(4, cache.getEvictedCount());
396
397
398 assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false));
399 assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false));
400 assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false));
401 assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false));
402
403
404
405
406
407
408
409
410 for(int i=5;i<18;i++) {
411 cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
412 }
413
414
415 assertEquals(4, cache.getEvictionCount());
416 assertEquals(16, cache.getEvictedCount());
417
418
419 assertEquals(7, cache.size());
420
421 }
422
423
424 @Test
425 public void testResizeBlockCache() throws Exception {
426
427 long maxSize = 300000;
428 long blockSize = calculateBlockSize(maxSize, 31);
429
430 LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false,
431 (int)Math.ceil(1.2*maxSize/blockSize),
432 LruBlockCache.DEFAULT_LOAD_FACTOR,
433 LruBlockCache.DEFAULT_CONCURRENCY_LEVEL,
434 0.98f,
435 0.99f,
436 0.33f,
437 0.33f,
438 0.34f);
439
440 CachedItem [] singleBlocks = generateFixedBlocks(10, blockSize, "single");
441 CachedItem [] multiBlocks = generateFixedBlocks(10, blockSize, "multi");
442 CachedItem [] memoryBlocks = generateFixedBlocks(10, blockSize, "memory");
443
444
445 for(int i=0;i<10;i++) {
446
447
448 cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
449
450
451 cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
452 cache.getBlock(multiBlocks[i].cacheKey, true, false);
453
454
455 cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
456 }
457
458
459 assertEquals(0, cache.getEvictionCount());
460
461
462 cache.setMaxSize((long)(maxSize * 0.5f));
463
464
465 assertEquals(1, cache.getEvictionCount());
466
467
468 assertEquals(15, cache.getEvictedCount());
469
470
471 for(int i=0;i<5;i++) {
472 assertEquals(null, cache.getBlock(singleBlocks[i].cacheKey, true, false));
473 assertEquals(null, cache.getBlock(multiBlocks[i].cacheKey, true, false));
474 assertEquals(null, cache.getBlock(memoryBlocks[i].cacheKey, true, false));
475 }
476
477
478 for(int i=5;i<10;i++) {
479 assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].cacheKey, true, false));
480 assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].cacheKey, true, false));
481 assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].cacheKey, true, false));
482 }
483 }
484
485
486 @Test
487 public void testPastNPeriodsMetrics() throws Exception {
488 double delta = 0.01;
489
490
491 CacheStats stats = new CacheStats(3);
492
493
494 stats.rollMetricsPeriod();
495 assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta);
496 assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
497
498
499
500 stats.hit(false);
501 stats.hit(true);
502 stats.miss(false);
503 stats.miss(false);
504 stats.rollMetricsPeriod();
505 assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
506 assertEquals(1.0, stats.getHitCachingRatioPastNPeriods(), delta);
507
508
509
510 stats.miss(true);
511 stats.miss(false);
512 stats.miss(false);
513 stats.miss(false);
514 stats.rollMetricsPeriod();
515 assertEquals(0.25, stats.getHitRatioPastNPeriods(), delta);
516 assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta);
517
518
519
520 stats.hit(false);
521 stats.hit(true);
522 stats.hit(false);
523 stats.hit(true);
524 stats.rollMetricsPeriod();
525 assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
526 assertEquals(0.75, stats.getHitCachingRatioPastNPeriods(), delta);
527
528
529
530 stats.miss(true);
531 stats.miss(true);
532 stats.rollMetricsPeriod();
533 assertEquals(0.4, stats.getHitRatioPastNPeriods(), delta);
534 assertEquals(0.4, stats.getHitCachingRatioPastNPeriods(), delta);
535
536
537
538 stats.miss(true);
539 stats.miss(true);
540 stats.hit(false);
541 stats.hit(false);
542 stats.rollMetricsPeriod();
543 assertEquals(0.6, stats.getHitRatioPastNPeriods(), delta);
544 assertEquals((double)1/3, stats.getHitCachingRatioPastNPeriods(), delta);
545
546
547
548 stats.rollMetricsPeriod();
549 assertEquals((double)1/3, stats.getHitRatioPastNPeriods(), delta);
550 assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
551
552
553
554 stats.rollMetricsPeriod();
555 assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
556 assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
557
558
559
560 stats.rollMetricsPeriod();
561 assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta);
562 assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
563
564
565
566 stats.miss(true);
567 stats.miss(false);
568 stats.hit(true);
569 stats.hit(false);
570 stats.rollMetricsPeriod();
571 assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
572 assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta);
573 }
574
575 private CachedItem [] generateFixedBlocks(int numBlocks, int size, String pfx) {
576 CachedItem [] blocks = new CachedItem[numBlocks];
577 for(int i=0;i<numBlocks;i++) {
578 blocks[i] = new CachedItem(pfx + i, size);
579 }
580 return blocks;
581 }
582
583 private CachedItem [] generateFixedBlocks(int numBlocks, long size, String pfx) {
584 return generateFixedBlocks(numBlocks, (int)size, pfx);
585 }
586
587 private CachedItem [] generateRandomBlocks(int numBlocks, long maxSize) {
588 CachedItem [] blocks = new CachedItem[numBlocks];
589 Random r = new Random();
590 for(int i=0;i<numBlocks;i++) {
591 blocks[i] = new CachedItem("block" + i, r.nextInt((int)maxSize)+1);
592 }
593 return blocks;
594 }
595
596 private long calculateBlockSize(long maxSize, int numBlocks) {
597 long roughBlockSize = maxSize / numBlocks;
598 int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
599 long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
600 ClassSize.CONCURRENT_HASHMAP +
601 (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
602 (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
603 long negateBlockSize = (long)(totalOverhead/numEntries);
604 negateBlockSize += CachedBlock.PER_BLOCK_OVERHEAD;
605 return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*0.99f));
606 }
607
608 private long calculateBlockSizeDefault(long maxSize, int numBlocks) {
609 long roughBlockSize = maxSize / numBlocks;
610 int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
611 long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
612 ClassSize.CONCURRENT_HASHMAP +
613 (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
614 (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
615 long negateBlockSize = totalOverhead / numEntries;
616 negateBlockSize += CachedBlock.PER_BLOCK_OVERHEAD;
617 return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*
618 LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
619 }
620
621 private static class CachedItem implements Cacheable {
622 BlockCacheKey cacheKey;
623 int size;
624
625 CachedItem(String blockName, int size) {
626 this.cacheKey = new BlockCacheKey(blockName, 0);
627 this.size = size;
628 }
629
630
631 @Override
632 public long heapSize() {
633 return ClassSize.align(size);
634 }
635
636
637 public long cacheBlockHeapSize() {
638 return CachedBlock.PER_BLOCK_OVERHEAD
639 + ClassSize.align(cacheKey.heapSize())
640 + ClassSize.align(size);
641 }
642
643 @Override
644 public int getSerializedLength() {
645 return 0;
646 }
647
648 @Override
649 public CacheableDeserializer<Cacheable> getDeserializer() {
650 return null;
651 }
652
653 @Override
654 public void serialize(ByteBuffer destination) {
655 }
656
657 @Override
658 public BlockType getBlockType() {
659 return BlockType.DATA;
660 }
661
662 }
663
664 }
665