1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import static org.junit.Assert.*;
23  
24  import java.util.List;
25  import java.util.Map;
26  import java.util.Random;
27  import java.util.concurrent.atomic.AtomicInteger;
28  
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.MultithreadedTestUtil;
31  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
32  import org.apache.hadoop.hbase.SmallTests;
33  import org.apache.hadoop.hbase.regionserver.MemStoreLAB.Allocation;
34  import org.junit.Test;
35  
36  import com.google.common.collect.Iterables;
37  import com.google.common.collect.Lists;
38  import com.google.common.collect.Maps;
39  import com.google.common.primitives.Ints;
40  import org.junit.experimental.categories.Category;
41  
42  @Category(SmallTests.class)
43  public class TestMemStoreLAB {
44  
45    /**
46     * Test a bunch of random allocations
47     */
48    @Test
49    public void testLABRandomAllocation() {
50      Random rand = new Random();
51      MemStoreLAB mslab = new MemStoreLAB();
52      int expectedOff = 0;
53      byte[] lastBuffer = null;
54      // 100K iterations by 0-1K alloc -> 50MB expected
55      // should be reasonable for unit test and also cover wraparound
56      // behavior
57      for (int i = 0; i < 100000; i++) {
58        int size = rand.nextInt(1000);
59        Allocation alloc = mslab.allocateBytes(size);
60        
61        if (alloc.getData() != lastBuffer) {
62          expectedOff = 0;
63          lastBuffer = alloc.getData();
64        }
65        assertEquals(expectedOff, alloc.getOffset());
66        assertTrue("Allocation " + alloc + " overruns buffer",
67            alloc.getOffset() + size <= alloc.getData().length);
68        expectedOff += size;
69      }
70    }
71  
72    @Test
73    public void testLABLargeAllocation() {
74      MemStoreLAB mslab = new MemStoreLAB();
75      Allocation alloc = mslab.allocateBytes(2*1024*1024);
76      assertNull("2MB allocation shouldn't be satisfied by LAB.",
77        alloc);
78    } 
79  
80    /**
81     * Test allocation from lots of threads, making sure the results don't
82     * overlap in any way
83     */
84    @Test
85    public void testLABThreading() throws Exception {
86      Configuration conf = new Configuration();
87      MultithreadedTestUtil.TestContext ctx =
88        new MultithreadedTestUtil.TestContext(conf);
89      
90      final AtomicInteger totalAllocated = new AtomicInteger();
91      
92      final MemStoreLAB mslab = new MemStoreLAB();
93      List<List<AllocRecord>> allocations = Lists.newArrayList();
94      
95      for (int i = 0; i < 10; i++) {
96        final List<AllocRecord> allocsByThisThread = Lists.newLinkedList();
97        allocations.add(allocsByThisThread);
98        
99        TestThread t = new MultithreadedTestUtil.RepeatingTestThread(ctx) {
100         private Random r = new Random();
101         @Override
102         public void doAnAction() throws Exception {
103           int size = r.nextInt(1000);
104           Allocation alloc = mslab.allocateBytes(size);
105           totalAllocated.addAndGet(size);
106           allocsByThisThread.add(new AllocRecord(alloc, size));
107         }
108       };
109       ctx.addThread(t);
110     }
111     
112     ctx.startThreads();
113     while (totalAllocated.get() < 50*1024*1024 && ctx.shouldRun()) {
114       Thread.sleep(10);
115     }
116     ctx.stop();
117     
118     // Partition the allocations by the actual byte[] they point into,
119     // make sure offsets are unique for each chunk
120     Map<byte[], Map<Integer, AllocRecord>> mapsByChunk =
121       Maps.newHashMap();
122     
123     int sizeCounted = 0;
124     for (AllocRecord rec : Iterables.concat(allocations)) {
125       sizeCounted += rec.size;
126       if (rec.size == 0) continue;
127       
128       Map<Integer, AllocRecord> mapForThisByteArray =
129         mapsByChunk.get(rec.alloc.getData());
130       if (mapForThisByteArray == null) {
131         mapForThisByteArray = Maps.newTreeMap();
132         mapsByChunk.put(rec.alloc.getData(), mapForThisByteArray);
133       }
134       AllocRecord oldVal = mapForThisByteArray.put(rec.alloc.getOffset(), rec);
135       assertNull("Already had an entry " + oldVal + " for allocation " + rec,
136           oldVal);
137     }
138     assertEquals("Sanity check test", sizeCounted, totalAllocated.get());
139     
140     // Now check each byte array to make sure allocations don't overlap
141     for (Map<Integer, AllocRecord> allocsInChunk : mapsByChunk.values()) {
142       int expectedOff = 0;
143       for (AllocRecord alloc : allocsInChunk.values()) {
144         assertEquals(expectedOff, alloc.alloc.getOffset());
145         assertTrue("Allocation " + alloc + " overruns buffer",
146             alloc.alloc.getOffset() + alloc.size <= alloc.alloc.getData().length);
147         expectedOff += alloc.size;
148       }
149     }
150 
151   }
152   
153   private static class AllocRecord implements Comparable<AllocRecord>{
154     private final Allocation alloc;
155     private final int size;
156     public AllocRecord(Allocation alloc, int size) {
157       super();
158       this.alloc = alloc;
159       this.size = size;
160     }
161 
162     @Override
163     public int compareTo(AllocRecord e) {
164       if (alloc.getData() != e.alloc.getData()) {
165         throw new RuntimeException("Can only compare within a particular array");
166       }
167       return Ints.compare(alloc.getOffset(), e.alloc.getOffset());
168     }
169     
170     @Override
171     public String toString() {
172       return "AllocRecord(alloc=" + alloc + ", size=" + size + ")";
173     }
174     
175   }
176 
177   @org.junit.Rule
178   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
179     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
180 }
181