View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one or more
5    * contributor license agreements. See the NOTICE file distributed with this
6    * work for additional information regarding copyright ownership. The ASF
7    * licenses this file to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance with the License.
9    * You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16   * License for the specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile.bucket;
20  
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.FileNotFoundException;
24  import java.io.IOException;
25  import java.util.ArrayList;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.SmallTests;
30  import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
31  import org.apache.hadoop.hbase.io.hfile.CacheTestUtils;
32  import org.apache.hadoop.hbase.io.hfile.Cacheable;
33  import org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.BucketSizeInfo;
34  import org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.IndexStatistics;
35  import org.junit.After;
36  import org.junit.Before;
37  import org.junit.Test;
38  import org.junit.experimental.categories.Category;
39  
40  /**
41   * Basic test of BucketCache.Puts and gets.
42   * <p>
43   * Tests will ensure that blocks' data correctness under several threads
44   * concurrency
45   */
46  @Category(SmallTests.class)
47  public class TestBucketCache {
48    static final Log LOG = LogFactory.getLog(TestBucketCache.class);
49    BucketCache cache;
50    final int CACHE_SIZE = 1000000;
51    final int NUM_BLOCKS = 100;
52    final int BLOCK_SIZE = CACHE_SIZE / NUM_BLOCKS;
53    final int NUM_THREADS = 1000;
54    final int NUM_QUERIES = 10000;
55    
56    final long capacitySize = 32 * 1024 * 1024;
57    final int writeThreads = BucketCache.DEFAULT_WRITER_THREADS;
58    final int writerQLen = BucketCache.DEFAULT_WRITER_QUEUE_ITEMS;
59    String ioEngineName = "heap";
60    String persistencePath = null;
61  
62    private class MockedBucketCache extends BucketCache {
63  
64      public MockedBucketCache(String ioEngineName, long capacity,
65          int writerThreads,
66          int writerQLen, String persistencePath) throws FileNotFoundException,
67          IOException {
68        super(ioEngineName, capacity, 8192, writerThreads, writerQLen, persistencePath);
69        super.wait_when_cache = true;
70      }
71  
72      @Override
73      public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf,
74          boolean inMemory) {
75        if (super.getBlock(cacheKey, true, false, true) != null) {
76          throw new RuntimeException("Cached an already cached block");
77        }
78        super.cacheBlock(cacheKey, buf, inMemory);
79      }
80  
81      @Override
82      public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
83        if (super.getBlock(cacheKey, true, false, true) != null) {
84          throw new RuntimeException("Cached an already cached block");
85        }
86        super.cacheBlock(cacheKey, buf);
87      }
88    }
89  
90    @Before
91    public void setup() throws FileNotFoundException, IOException {
92      cache = new MockedBucketCache(ioEngineName, capacitySize, writeThreads,
93          writerQLen, persistencePath);
94    }
95  
96    @After
97    public void tearDown() {
98      cache.shutdown();
99    }
100 
101   @Test
102   public void testBucketAllocator() throws BucketAllocatorException {
103     BucketAllocator mAllocator = cache.getAllocator();
104     /*
105      * Test the allocator first
106      */
107     int[] blockSizes = new int[2];
108     blockSizes[0] = 4 * 1024;
109     blockSizes[1] = 8 * 1024;
110     boolean full = false;
111     int i = 0;
112     ArrayList<Long> allocations = new ArrayList<Long>();
113     // Fill the allocated extents
114     while (!full) {
115       try {
116         allocations.add(new Long(mAllocator.allocateBlock(blockSizes[i
117             % blockSizes.length])));
118         ++i;
119       } catch (CacheFullException cfe) {
120         full = true;
121       }
122     }
123 
124     for (i = 0; i < blockSizes.length; i++) {
125       BucketSizeInfo bucketSizeInfo = mAllocator
126           .roundUpToBucketSizeInfo(blockSizes[0]);
127       IndexStatistics indexStatistics = bucketSizeInfo.statistics();
128       assertTrue(indexStatistics.freeCount() == 0);
129     }
130 
131     for (long offset : allocations) {
132       assertTrue(mAllocator.sizeOfAllocation(offset) == mAllocator
133           .freeBlock(offset));
134     }
135     assertTrue(mAllocator.getUsedSize() == 0);
136   }
137 
138   @Test
139   public void testCacheSimple() throws Exception {
140     CacheTestUtils.testCacheSimple(cache, BLOCK_SIZE, NUM_QUERIES);
141   }
142 
143   @Test
144   public void testCacheMultiThreadedSingleKey() throws Exception {
145     CacheTestUtils.hammerSingleKey(cache, BLOCK_SIZE, NUM_THREADS, NUM_QUERIES);
146   }
147 
148   @Test
149   public void testHeapSizeChanges() throws Exception {
150     cache.stopWriterThreads();
151     CacheTestUtils.testHeapSizeChanges(cache, BLOCK_SIZE);
152   }
153 
154 }