View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertNotNull;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.nio.ByteBuffer;
27  import java.util.Random;
28  
29  import org.apache.hadoop.hbase.io.HeapSize;
30  import org.apache.hadoop.hbase.io.hfile.LruBlockCache.EvictionThread;
31  import org.apache.hadoop.hbase.testclassification.SmallTests;
32  import org.apache.hadoop.hbase.util.ClassSize;
33  import org.junit.Test;
34  import org.junit.experimental.categories.Category;
35  
36  /**
37   * Tests the concurrent LruBlockCache.<p>
38   *
39   * Tests will ensure it grows and shrinks in size properly,
40   * evictions run when they're supposed to and do what they should,
41   * and that cached blocks are accessible when expected to be.
42   */
43  @Category(SmallTests.class)
44  public class TestLruBlockCache {
45  
46  
47    @Test
48    public void testBackgroundEvictionThread() throws Exception {
49      long maxSize = 100000;
50      int numBlocks = 9;
51      long blockSize = calculateBlockSizeDefault(maxSize, numBlocks);
52      assertTrue("calculateBlockSize appears broken.", blockSize * numBlocks <= maxSize);
53  
54      LruBlockCache cache = new LruBlockCache(maxSize,blockSize);
55      EvictionThread evictionThread = cache.getEvictionThread();
56      assertTrue(evictionThread != null);
57  
58      CachedItem[] blocks = generateFixedBlocks(numBlocks + 1, blockSize, "block");
59  
60      // Make sure eviction thread has entered run method
61      while (!evictionThread.isEnteringRun()) {
62        Thread.sleep(1);
63      }
64  
65      // Add all the blocks
66      for (CachedItem block : blocks) {
67        cache.cacheBlock(block.cacheKey, block);
68      }
69  
70      // wait until at least one eviction has run
71      int n = 0;
72      while(cache.getStats().getEvictionCount() == 0) {
73        Thread.sleep(200);
74        assertTrue("Eviction never happened.", n++ < 20);
75      }
76  
77      // let cache stabilize
78      // On some systems, the cache will run multiple evictions before it attains
79      // steady-state. For instance, after populating the cache with 10 blocks,
80      // the first eviction evicts a single block and then a second eviction
81      // evicts another. I think this is due to the delta between minSize and
82      // acceptableSize, combined with variance between object overhead on
83      // different environments.
84      n = 0;
85      for (long prevCnt = 0 /* < number of blocks added */,
86                curCnt = cache.getBlockCount();
87          prevCnt != curCnt; prevCnt = curCnt, curCnt = cache.getBlockCount()) {
88        Thread.sleep(200);
89        assertTrue("Cache never stabilized.", n++ < 20);
90      }
91  
92      long evictionCount = cache.getStats().getEvictionCount();
93      assertTrue(evictionCount >= 1);
94      System.out.println("Background Evictions run: " + evictionCount);
95    }
96  
97    @Test
98    public void testCacheSimple() throws Exception {
99  
100     long maxSize = 1000000;
101     long blockSize = calculateBlockSizeDefault(maxSize, 101);
102 
103     LruBlockCache cache = new LruBlockCache(maxSize, blockSize);
104 
105     CachedItem [] blocks = generateRandomBlocks(100, blockSize);
106 
107     long expectedCacheSize = cache.heapSize();
108 
109     // Confirm empty
110     for (CachedItem block : blocks) {
111       assertTrue(cache.getBlock(block.cacheKey, true, false, true) == null);
112     }
113 
114     // Add blocks
115     for (CachedItem block : blocks) {
116       cache.cacheBlock(block.cacheKey, block);
117       expectedCacheSize += block.cacheBlockHeapSize();
118     }
119 
120     // Verify correctly calculated cache heap size
121     assertEquals(expectedCacheSize, cache.heapSize());
122 
123     // Check if all blocks are properly cached and retrieved
124     for (CachedItem block : blocks) {
125       HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
126       assertTrue(buf != null);
127       assertEquals(buf.heapSize(), block.heapSize());
128     }
129 
130     // Verify correctly calculated cache heap size
131     assertEquals(expectedCacheSize, cache.heapSize());
132 
133     // Check if all blocks are properly cached and retrieved
134     for (CachedItem block : blocks) {
135       HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
136       assertTrue(buf != null);
137       assertEquals(buf.heapSize(), block.heapSize());
138     }
139 
140     // Expect no evictions
141     assertEquals(0, cache.getStats().getEvictionCount());
142     Thread t = new LruBlockCache.StatisticsThread(cache);
143     t.start();
144     t.join();
145   }
146 
147   @Test
148   public void testCacheEvictionSimple() throws Exception {
149 
150     long maxSize = 100000;
151     long blockSize = calculateBlockSizeDefault(maxSize, 10);
152 
153     LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false);
154 
155     CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
156 
157     long expectedCacheSize = cache.heapSize();
158 
159     // Add all the blocks
160     for (CachedItem block : blocks) {
161       cache.cacheBlock(block.cacheKey, block);
162       expectedCacheSize += block.cacheBlockHeapSize();
163     }
164 
165     // A single eviction run should have occurred
166     assertEquals(1, cache.getStats().getEvictionCount());
167 
168     // Our expected size overruns acceptable limit
169     assertTrue(expectedCacheSize >
170       (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
171 
172     // But the cache did not grow beyond max
173     assertTrue(cache.heapSize() < maxSize);
174 
175     // And is still below the acceptable limit
176     assertTrue(cache.heapSize() <
177         (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
178 
179     // All blocks except block 0  should be in the cache
180     assertTrue(cache.getBlock(blocks[0].cacheKey, true, false, true) == null);
181     for(int i=1;i<blocks.length;i++) {
182       assertEquals(cache.getBlock(blocks[i].cacheKey, true, false, true),
183           blocks[i]);
184     }
185   }
186 
187   @Test
188   public void testCacheEvictionTwoPriorities() throws Exception {
189 
190     long maxSize = 100000;
191     long blockSize = calculateBlockSizeDefault(maxSize, 10);
192 
193     LruBlockCache cache = new LruBlockCache(maxSize,blockSize,false);
194 
195     CachedItem [] singleBlocks = generateFixedBlocks(5, 10000, "single");
196     CachedItem [] multiBlocks = generateFixedBlocks(5, 10000, "multi");
197 
198     long expectedCacheSize = cache.heapSize();
199 
200     // Add and get the multi blocks
201     for (CachedItem block : multiBlocks) {
202       cache.cacheBlock(block.cacheKey, block);
203       expectedCacheSize += block.cacheBlockHeapSize();
204       assertEquals(cache.getBlock(block.cacheKey, true, false, true), block);
205     }
206 
207     // Add the single blocks (no get)
208     for (CachedItem block : singleBlocks) {
209       cache.cacheBlock(block.cacheKey, block);
210       expectedCacheSize += block.heapSize();
211     }
212 
213     // A single eviction run should have occurred
214     assertEquals(cache.getStats().getEvictionCount(), 1);
215 
216     // We expect two entries evicted
217     assertEquals(cache.getStats().getEvictedCount(), 2);
218 
219     // Our expected size overruns acceptable limit
220     assertTrue(expectedCacheSize >
221       (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
222 
223     // But the cache did not grow beyond max
224     assertTrue(cache.heapSize() <= maxSize);
225 
226     // And is now below the acceptable limit
227     assertTrue(cache.heapSize() <=
228         (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
229 
230     // We expect fairness across the two priorities.
231     // This test makes multi go barely over its limit, in-memory
232     // empty, and the rest in single.  Two single evictions and
233     // one multi eviction expected.
234     assertTrue(cache.getBlock(singleBlocks[0].cacheKey, true, false, true) == null);
235     assertTrue(cache.getBlock(multiBlocks[0].cacheKey, true, false, true) == null);
236 
237     // And all others to be cached
238     for(int i=1;i<4;i++) {
239       assertEquals(cache.getBlock(singleBlocks[i].cacheKey, true, false, true),
240           singleBlocks[i]);
241       assertEquals(cache.getBlock(multiBlocks[i].cacheKey, true, false, true),
242           multiBlocks[i]);
243     }
244   }
245 
246   @Test
247   public void testCacheEvictionThreePriorities() throws Exception {
248 
249     long maxSize = 100000;
250     long blockSize = calculateBlockSize(maxSize, 10);
251 
252     LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false,
253         (int)Math.ceil(1.2*maxSize/blockSize),
254         LruBlockCache.DEFAULT_LOAD_FACTOR,
255         LruBlockCache.DEFAULT_CONCURRENCY_LEVEL,
256         0.98f, // min
257         0.99f, // acceptable
258         0.33f, // single
259         0.33f, // multi
260         0.34f, // memory
261         false,
262         16 * 1024 * 1024);
263 
264     CachedItem [] singleBlocks = generateFixedBlocks(5, blockSize, "single");
265     CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
266     CachedItem [] memoryBlocks = generateFixedBlocks(5, blockSize, "memory");
267 
268     long expectedCacheSize = cache.heapSize();
269 
270     // Add 3 blocks from each priority
271     for(int i=0;i<3;i++) {
272 
273       // Just add single blocks
274       cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
275       expectedCacheSize += singleBlocks[i].cacheBlockHeapSize();
276 
277       // Add and get multi blocks
278       cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
279       expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
280       cache.getBlock(multiBlocks[i].cacheKey, true, false, true);
281 
282       // Add memory blocks as such
283       cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
284       expectedCacheSize += memoryBlocks[i].cacheBlockHeapSize();
285 
286     }
287 
288     // Do not expect any evictions yet
289     assertEquals(0, cache.getStats().getEvictionCount());
290 
291     // Verify cache size
292     assertEquals(expectedCacheSize, cache.heapSize());
293 
294     // Insert a single block, oldest single should be evicted
295     cache.cacheBlock(singleBlocks[3].cacheKey, singleBlocks[3]);
296 
297     // Single eviction, one thing evicted
298     assertEquals(1, cache.getStats().getEvictionCount());
299     assertEquals(1, cache.getStats().getEvictedCount());
300 
301     // Verify oldest single block is the one evicted
302     assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true));
303 
304     // Change the oldest remaining single block to a multi
305     cache.getBlock(singleBlocks[1].cacheKey, true, false, true);
306 
307     // Insert another single block
308     cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]);
309 
310     // Two evictions, two evicted.
311     assertEquals(2, cache.getStats().getEvictionCount());
312     assertEquals(2, cache.getStats().getEvictedCount());
313 
314     // Oldest multi block should be evicted now
315     assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true));
316 
317     // Insert another memory block
318     cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true);
319 
320     // Three evictions, three evicted.
321     assertEquals(3, cache.getStats().getEvictionCount());
322     assertEquals(3, cache.getStats().getEvictedCount());
323 
324     // Oldest memory block should be evicted now
325     assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false, true));
326 
327     // Add a block that is twice as big (should force two evictions)
328     CachedItem [] bigBlocks = generateFixedBlocks(3, blockSize*3, "big");
329     cache.cacheBlock(bigBlocks[0].cacheKey, bigBlocks[0]);
330 
331     // Four evictions, six evicted (inserted block 3X size, expect +3 evicted)
332     assertEquals(4, cache.getStats().getEvictionCount());
333     assertEquals(6, cache.getStats().getEvictedCount());
334 
335     // Expect three remaining singles to be evicted
336     assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false, true));
337     assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false, true));
338     assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false, true));
339 
340     // Make the big block a multi block
341     cache.getBlock(bigBlocks[0].cacheKey, true, false, true);
342 
343     // Cache another single big block
344     cache.cacheBlock(bigBlocks[1].cacheKey, bigBlocks[1]);
345 
346     // Five evictions, nine evicted (3 new)
347     assertEquals(5, cache.getStats().getEvictionCount());
348     assertEquals(9, cache.getStats().getEvictedCount());
349 
350     // Expect three remaining multis to be evicted
351     assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true));
352     assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true));
353     assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false, true));
354 
355     // Cache a big memory block
356     cache.cacheBlock(bigBlocks[2].cacheKey, bigBlocks[2], true);
357 
358     // Six evictions, twelve evicted (3 new)
359     assertEquals(6, cache.getStats().getEvictionCount());
360     assertEquals(12, cache.getStats().getEvictedCount());
361 
362     // Expect three remaining in-memory to be evicted
363     assertEquals(null, cache.getBlock(memoryBlocks[1].cacheKey, true, false, true));
364     assertEquals(null, cache.getBlock(memoryBlocks[2].cacheKey, true, false, true));
365     assertEquals(null, cache.getBlock(memoryBlocks[3].cacheKey, true, false, true));
366   }
367 
368   @Test
369   public void testCacheEvictionInMemoryForceMode() throws Exception {
370     long maxSize = 100000;
371     long blockSize = calculateBlockSize(maxSize, 10);
372 
373     LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false,
374         (int)Math.ceil(1.2*maxSize/blockSize),
375         LruBlockCache.DEFAULT_LOAD_FACTOR,
376         LruBlockCache.DEFAULT_CONCURRENCY_LEVEL,
377         0.98f, // min
378         0.99f, // acceptable
379         0.2f, // single
380         0.3f, // multi
381         0.5f, // memory
382         true,
383         16 * 1024 * 1024);
384 
385     CachedItem [] singleBlocks = generateFixedBlocks(10, blockSize, "single");
386     CachedItem [] multiBlocks = generateFixedBlocks(10, blockSize, "multi");
387     CachedItem [] memoryBlocks = generateFixedBlocks(10, blockSize, "memory");
388 
389     long expectedCacheSize = cache.heapSize();
390 
391     // 0. Add 5 single blocks and 4 multi blocks to make cache full, si:mu:me = 5:4:0
392     for(int i = 0; i < 4; i++) {
393       // Just add single blocks
394       cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
395       expectedCacheSize += singleBlocks[i].cacheBlockHeapSize();
396       // Add and get multi blocks
397       cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
398       expectedCacheSize += multiBlocks[i].cacheBlockHeapSize();
399       cache.getBlock(multiBlocks[i].cacheKey, true, false, true);
400     }
401     // 5th single block
402     cache.cacheBlock(singleBlocks[4].cacheKey, singleBlocks[4]);
403     expectedCacheSize += singleBlocks[4].cacheBlockHeapSize();
404     // Do not expect any evictions yet
405     assertEquals(0, cache.getStats().getEvictionCount());
406     // Verify cache size
407     assertEquals(expectedCacheSize, cache.heapSize());
408 
409     // 1. Insert a memory block, oldest single should be evicted, si:mu:me = 4:4:1
410     cache.cacheBlock(memoryBlocks[0].cacheKey, memoryBlocks[0], true);
411     // Single eviction, one block evicted
412     assertEquals(1, cache.getStats().getEvictionCount());
413     assertEquals(1, cache.getStats().getEvictedCount());
414     // Verify oldest single block (index = 0) is the one evicted
415     assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true));
416 
417     // 2. Insert another memory block, another single evicted, si:mu:me = 3:4:2
418     cache.cacheBlock(memoryBlocks[1].cacheKey, memoryBlocks[1], true);
419     // Two evictions, two evicted.
420     assertEquals(2, cache.getStats().getEvictionCount());
421     assertEquals(2, cache.getStats().getEvictedCount());
422     // Current oldest single block (index = 1) should be evicted now
423     assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true));
424 
425     // 3. Insert 4 memory blocks, 2 single and 2 multi evicted, si:mu:me = 1:2:6
426     cache.cacheBlock(memoryBlocks[2].cacheKey, memoryBlocks[2], true);
427     cache.cacheBlock(memoryBlocks[3].cacheKey, memoryBlocks[3], true);
428     cache.cacheBlock(memoryBlocks[4].cacheKey, memoryBlocks[4], true);
429     cache.cacheBlock(memoryBlocks[5].cacheKey, memoryBlocks[5], true);
430     // Three evictions, three evicted.
431     assertEquals(6, cache.getStats().getEvictionCount());
432     assertEquals(6, cache.getStats().getEvictedCount());
433     // two oldest single blocks and two oldest multi blocks evicted
434     assertEquals(null, cache.getBlock(singleBlocks[2].cacheKey, true, false, true));
435     assertEquals(null, cache.getBlock(singleBlocks[3].cacheKey, true, false, true));
436     assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true));
437     assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true));
438 
439     // 4. Insert 3 memory blocks, the remaining 1 single and 2 multi evicted
440     // si:mu:me = 0:0:9
441     cache.cacheBlock(memoryBlocks[6].cacheKey, memoryBlocks[6], true);
442     cache.cacheBlock(memoryBlocks[7].cacheKey, memoryBlocks[7], true);
443     cache.cacheBlock(memoryBlocks[8].cacheKey, memoryBlocks[8], true);
444     // Three evictions, three evicted.
445     assertEquals(9, cache.getStats().getEvictionCount());
446     assertEquals(9, cache.getStats().getEvictedCount());
447     // one oldest single block and two oldest multi blocks evicted
448     assertEquals(null, cache.getBlock(singleBlocks[4].cacheKey, true, false, true));
449     assertEquals(null, cache.getBlock(multiBlocks[2].cacheKey, true, false, true));
450     assertEquals(null, cache.getBlock(multiBlocks[3].cacheKey, true, false, true));
451 
452     // 5. Insert one memory block, the oldest memory evicted
453     // si:mu:me = 0:0:9
454     cache.cacheBlock(memoryBlocks[9].cacheKey, memoryBlocks[9], true);
455     // one eviction, one evicted.
456     assertEquals(10, cache.getStats().getEvictionCount());
457     assertEquals(10, cache.getStats().getEvictedCount());
458     // oldest memory block evicted
459     assertEquals(null, cache.getBlock(memoryBlocks[0].cacheKey, true, false, true));
460 
461     // 6. Insert one new single block, itself evicted immediately since
462     //    all blocks in cache are memory-type which have higher priority
463     // si:mu:me = 0:0:9 (no change)
464     cache.cacheBlock(singleBlocks[9].cacheKey, singleBlocks[9]);
465     // one eviction, one evicted.
466     assertEquals(11, cache.getStats().getEvictionCount());
467     assertEquals(11, cache.getStats().getEvictedCount());
468     // the single block just cached now evicted (can't evict memory)
469     assertEquals(null, cache.getBlock(singleBlocks[9].cacheKey, true, false, true));
470   }
471 
472   // test scan resistance
473   @Test
474   public void testScanResistance() throws Exception {
475 
476     long maxSize = 100000;
477     long blockSize = calculateBlockSize(maxSize, 10);
478 
479     LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false,
480         (int)Math.ceil(1.2*maxSize/blockSize),
481         LruBlockCache.DEFAULT_LOAD_FACTOR,
482         LruBlockCache.DEFAULT_CONCURRENCY_LEVEL,
483         0.66f, // min
484         0.99f, // acceptable
485         0.33f, // single
486         0.33f, // multi
487         0.34f, // memory
488         false,
489         16 * 1024 * 1024);
490 
491     CachedItem [] singleBlocks = generateFixedBlocks(20, blockSize, "single");
492     CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
493 
494     // Add 5 multi blocks
495     for (CachedItem block : multiBlocks) {
496       cache.cacheBlock(block.cacheKey, block);
497       cache.getBlock(block.cacheKey, true, false, true);
498     }
499 
500     // Add 5 single blocks
501     for(int i=0;i<5;i++) {
502       cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
503     }
504 
505     // An eviction ran
506     assertEquals(1, cache.getStats().getEvictionCount());
507 
508     // To drop down to 2/3 capacity, we'll need to evict 4 blocks
509     assertEquals(4, cache.getStats().getEvictedCount());
510 
511     // Should have been taken off equally from single and multi
512     assertEquals(null, cache.getBlock(singleBlocks[0].cacheKey, true, false, true));
513     assertEquals(null, cache.getBlock(singleBlocks[1].cacheKey, true, false, true));
514     assertEquals(null, cache.getBlock(multiBlocks[0].cacheKey, true, false, true));
515     assertEquals(null, cache.getBlock(multiBlocks[1].cacheKey, true, false, true));
516 
517     // Let's keep "scanning" by adding single blocks.  From here on we only
518     // expect evictions from the single bucket.
519 
520     // Every time we reach 10 total blocks (every 4 inserts) we get 4 single
521     // blocks evicted.  Inserting 13 blocks should yield 3 more evictions and
522     // 12 more evicted.
523 
524     for(int i=5;i<18;i++) {
525       cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
526     }
527 
528     // 4 total evictions, 16 total evicted
529     assertEquals(4, cache.getStats().getEvictionCount());
530     assertEquals(16, cache.getStats().getEvictedCount());
531 
532     // Should now have 7 total blocks
533     assertEquals(7, cache.getBlockCount());
534 
535   }
536 
537   @Test
538   public void testMaxBlockSize() throws Exception {
539     long maxSize = 100000;
540     long blockSize = calculateBlockSize(maxSize, 10);
541 
542     LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false,
543         (int)Math.ceil(1.2*maxSize/blockSize),
544         LruBlockCache.DEFAULT_LOAD_FACTOR,
545         LruBlockCache.DEFAULT_CONCURRENCY_LEVEL,
546         0.66f, // min
547         0.99f, // acceptable
548         0.33f, // single
549         0.33f, // multi
550         0.34f, // memory
551         false,
552         1024);
553     CachedItem [] tooLong = generateFixedBlocks(10, 1024+5, "long");
554     CachedItem [] small = generateFixedBlocks(15, 600, "small");
555 
556 
557     for (CachedItem i:tooLong) {
558       cache.cacheBlock(i.cacheKey, i);
559     }
560     for (CachedItem i:small) {
561       cache.cacheBlock(i.cacheKey, i);
562     }
563     assertEquals(15,cache.getBlockCount());
564     for (CachedItem i:small) {
565       assertNotNull(cache.getBlock(i.cacheKey, true, false, false));
566     }
567     for (CachedItem i:tooLong) {
568       assertNull(cache.getBlock(i.cacheKey, true, false, false));
569     }
570 
571     assertEquals(10, cache.getStats().getFailedInserts());
572   }
573 
574   // test setMaxSize
575   @Test
576   public void testResizeBlockCache() throws Exception {
577 
578     long maxSize = 300000;
579     long blockSize = calculateBlockSize(maxSize, 31);
580 
581     LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false,
582         (int)Math.ceil(1.2*maxSize/blockSize),
583         LruBlockCache.DEFAULT_LOAD_FACTOR,
584         LruBlockCache.DEFAULT_CONCURRENCY_LEVEL,
585         0.98f, // min
586         0.99f, // acceptable
587         0.33f, // single
588         0.33f, // multi
589         0.34f, // memory
590         false,
591         16 * 1024 * 1024);
592 
593     CachedItem [] singleBlocks = generateFixedBlocks(10, blockSize, "single");
594     CachedItem [] multiBlocks = generateFixedBlocks(10, blockSize, "multi");
595     CachedItem [] memoryBlocks = generateFixedBlocks(10, blockSize, "memory");
596 
597     // Add all blocks from all priorities
598     for(int i=0;i<10;i++) {
599 
600       // Just add single blocks
601       cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
602 
603       // Add and get multi blocks
604       cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
605       cache.getBlock(multiBlocks[i].cacheKey, true, false, true);
606 
607       // Add memory blocks as such
608       cache.cacheBlock(memoryBlocks[i].cacheKey, memoryBlocks[i], true);
609     }
610 
611     // Do not expect any evictions yet
612     assertEquals(0, cache.getStats().getEvictionCount());
613 
614     // Resize to half capacity plus an extra block (otherwise we evict an extra)
615     cache.setMaxSize((long)(maxSize * 0.5f));
616 
617     // Should have run a single eviction
618     assertEquals(1, cache.getStats().getEvictionCount());
619 
620     // And we expect 1/2 of the blocks to be evicted
621     assertEquals(15, cache.getStats().getEvictedCount());
622 
623     // And the oldest 5 blocks from each category should be gone
624     for(int i=0;i<5;i++) {
625       assertEquals(null, cache.getBlock(singleBlocks[i].cacheKey, true, false, true));
626       assertEquals(null, cache.getBlock(multiBlocks[i].cacheKey, true, false, true));
627       assertEquals(null, cache.getBlock(memoryBlocks[i].cacheKey, true, false, true));
628     }
629 
630     // And the newest 5 blocks should still be accessible
631     for(int i=5;i<10;i++) {
632       assertEquals(singleBlocks[i], cache.getBlock(singleBlocks[i].cacheKey, true, false, true));
633       assertEquals(multiBlocks[i], cache.getBlock(multiBlocks[i].cacheKey, true, false, true));
634       assertEquals(memoryBlocks[i], cache.getBlock(memoryBlocks[i].cacheKey, true, false, true));
635     }
636   }
637 
638   // test metricsPastNPeriods
639   @Test
640   public void testPastNPeriodsMetrics() throws Exception {
641    double delta = 0.01;
642 
643     // 3 total periods
644     CacheStats stats = new CacheStats(3);
645 
646     // No accesses, should be 0
647     stats.rollMetricsPeriod();
648     assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta);
649     assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
650 
651     // period 1, 1 hit caching, 1 hit non-caching, 2 miss non-caching
652     // should be (2/4)=0.5 and (1/1)=1
653     stats.hit(false);
654     stats.hit(true);
655     stats.miss(false);
656     stats.miss(false);
657     stats.rollMetricsPeriod();
658     assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
659     assertEquals(1.0, stats.getHitCachingRatioPastNPeriods(), delta);
660 
661     // period 2, 1 miss caching, 3 miss non-caching
662     // should be (2/8)=0.25 and (1/2)=0.5
663     stats.miss(true);
664     stats.miss(false);
665     stats.miss(false);
666     stats.miss(false);
667     stats.rollMetricsPeriod();
668     assertEquals(0.25, stats.getHitRatioPastNPeriods(), delta);
669     assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta);
670 
671     // period 3, 2 hits of each type
672     // should be (6/12)=0.5 and (3/4)=0.75
673     stats.hit(false);
674     stats.hit(true);
675     stats.hit(false);
676     stats.hit(true);
677     stats.rollMetricsPeriod();
678     assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
679     assertEquals(0.75, stats.getHitCachingRatioPastNPeriods(), delta);
680 
681     // period 4, evict period 1, two caching misses
682     // should be (4/10)=0.4 and (2/5)=0.4
683     stats.miss(true);
684     stats.miss(true);
685     stats.rollMetricsPeriod();
686     assertEquals(0.4, stats.getHitRatioPastNPeriods(), delta);
687     assertEquals(0.4, stats.getHitCachingRatioPastNPeriods(), delta);
688 
689     // period 5, evict period 2, 2 caching misses, 2 non-caching hit
690     // should be (6/10)=0.6 and (2/6)=1/3
691     stats.miss(true);
692     stats.miss(true);
693     stats.hit(false);
694     stats.hit(false);
695     stats.rollMetricsPeriod();
696     assertEquals(0.6, stats.getHitRatioPastNPeriods(), delta);
697     assertEquals((double)1/3, stats.getHitCachingRatioPastNPeriods(), delta);
698 
699     // period 6, evict period 3
700     // should be (2/6)=1/3 and (0/4)=0
701     stats.rollMetricsPeriod();
702     assertEquals((double)1/3, stats.getHitRatioPastNPeriods(), delta);
703     assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
704 
705     // period 7, evict period 4
706     // should be (2/4)=0.5 and (0/2)=0
707     stats.rollMetricsPeriod();
708     assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
709     assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
710 
711     // period 8, evict period 5
712     // should be 0 and 0
713     stats.rollMetricsPeriod();
714     assertEquals(0.0, stats.getHitRatioPastNPeriods(), delta);
715     assertEquals(0.0, stats.getHitCachingRatioPastNPeriods(), delta);
716 
717     // period 9, one of each
718     // should be (2/4)=0.5 and (1/2)=0.5
719     stats.miss(true);
720     stats.miss(false);
721     stats.hit(true);
722     stats.hit(false);
723     stats.rollMetricsPeriod();
724     assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta);
725     assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta);
726   }
727 
728   private CachedItem [] generateFixedBlocks(int numBlocks, int size, String pfx) {
729     CachedItem [] blocks = new CachedItem[numBlocks];
730     for(int i=0;i<numBlocks;i++) {
731       blocks[i] = new CachedItem(pfx + i, size);
732     }
733     return blocks;
734   }
735 
736   private CachedItem [] generateFixedBlocks(int numBlocks, long size, String pfx) {
737     return generateFixedBlocks(numBlocks, (int)size, pfx);
738   }
739 
740   private CachedItem [] generateRandomBlocks(int numBlocks, long maxSize) {
741     CachedItem [] blocks = new CachedItem[numBlocks];
742     Random r = new Random();
743     for(int i=0;i<numBlocks;i++) {
744       blocks[i] = new CachedItem("block" + i, r.nextInt((int)maxSize)+1);
745     }
746     return blocks;
747   }
748 
749   private long calculateBlockSize(long maxSize, int numBlocks) {
750     long roughBlockSize = maxSize / numBlocks;
751     int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
752     long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
753         ClassSize.CONCURRENT_HASHMAP +
754         (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
755         (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
756     long negateBlockSize = (long)(totalOverhead/numEntries);
757     negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
758     return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*0.99f));
759   }
760 
761   private long calculateBlockSizeDefault(long maxSize, int numBlocks) {
762     long roughBlockSize = maxSize / numBlocks;
763     int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
764     long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
765         ClassSize.CONCURRENT_HASHMAP +
766         (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
767         (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
768     long negateBlockSize = totalOverhead / numEntries;
769     negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
770     return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*
771         LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
772   }
773 
774   private static class CachedItem implements Cacheable {
775     BlockCacheKey cacheKey;
776     int size;
777 
778     CachedItem(String blockName, int size) {
779       this.cacheKey = new BlockCacheKey(blockName, 0);
780       this.size = size;
781     }
782 
783     /** The size of this item reported to the block cache layer */
784     @Override
785     public long heapSize() {
786       return ClassSize.align(size);
787     }
788 
789     /** Size of the cache block holding this item. Used for verification. */
790     public long cacheBlockHeapSize() {
791       return LruCachedBlock.PER_BLOCK_OVERHEAD
792           + ClassSize.align(cacheKey.heapSize())
793           + ClassSize.align(size);
794     }
795 
796     @Override
797     public int getSerializedLength() {
798       return 0;
799     }
800 
801     @Override
802     public CacheableDeserializer<Cacheable> getDeserializer() {
803       return null;
804     }
805 
806     @Override
807     public void serialize(ByteBuffer destination) {
808     }
809     
810     @Override
811     public BlockType getBlockType() {
812       return BlockType.DATA;
813     }
814 
815   }
816 
817 }
818