1   /*
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.io;
22  
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.ArrayList;
26  import java.util.TreeMap;
27  import java.util.concurrent.ConcurrentHashMap;
28  import java.util.concurrent.ConcurrentSkipListMap;
29  import java.util.concurrent.CopyOnWriteArrayList;
30  import java.util.concurrent.CopyOnWriteArraySet;
31  import java.util.concurrent.atomic.AtomicBoolean;
32  import java.util.concurrent.atomic.AtomicInteger;
33  import java.util.concurrent.atomic.AtomicLong;
34  import java.util.concurrent.locks.ReentrantReadWriteLock;
35  
36  import junit.framework.TestCase;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.hadoop.hbase.KeyValue;
41  import org.apache.hadoop.hbase.client.Put;
42  import org.apache.hadoop.hbase.io.hfile.CachedBlock;
43  import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
44  import org.apache.hadoop.hbase.regionserver.HRegion;
45  import org.apache.hadoop.hbase.regionserver.MemStore;
46  import org.apache.hadoop.hbase.regionserver.Store;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.ClassSize;
49  
50  /**
51   * Testing the sizing that HeapSize offers and compares to the size given by
52   * ClassSize.
53   */
54  public class TestHeapSize extends TestCase {
55    static final Log LOG = LogFactory.getLog(TestHeapSize.class);
56    // List of classes implementing HeapSize
57    // BatchOperation, BatchUpdate, BlockIndex, Entry, Entry<K,V>, HStoreKey
58    // KeyValue, LruBlockCache, LruHashMap<K,V>, Put, HLogKey
59  
60    /**
61     * Test our hard-coded sizing of native java objects
62     */
63    public void testNativeSizes() throws IOException {
64      @SuppressWarnings("rawtypes")
65      Class cl = null;
66      long expected = 0L;
67      long actual = 0L;
68  
69      // ArrayList
70      cl = ArrayList.class;
71      expected = ClassSize.estimateBase(cl, false);
72      actual = ClassSize.ARRAYLIST;
73      if(expected != actual) {
74        ClassSize.estimateBase(cl, true);
75        assertEquals(expected, actual);
76      }
77  
78      // ByteBuffer
79      cl = ByteBuffer.class;
80      expected = ClassSize.estimateBase(cl, false);
81      actual = ClassSize.BYTE_BUFFER;
82      if(expected != actual) {
83        ClassSize.estimateBase(cl, true);
84        assertEquals(expected, actual);
85      }
86  
87      // Integer
88      cl = Integer.class;
89      expected = ClassSize.estimateBase(cl, false);
90      actual = ClassSize.INTEGER;
91      if(expected != actual) {
92        ClassSize.estimateBase(cl, true);
93        assertEquals(expected, actual);
94      }
95  
96      // Map.Entry
97      // Interface is public, all others are not.  Hard to size via ClassSize
98  //    cl = Map.Entry.class;
99  //    expected = ClassSize.estimateBase(cl, false);
100 //    actual = ClassSize.MAP_ENTRY;
101 //    if(expected != actual) {
102 //      ClassSize.estimateBase(cl, true);
103 //      assertEquals(expected, actual);
104 //    }
105 
106     // Object
107     cl = Object.class;
108     expected = ClassSize.estimateBase(cl, false);
109     actual = ClassSize.OBJECT;
110     if(expected != actual) {
111       ClassSize.estimateBase(cl, true);
112       assertEquals(expected, actual);
113     }
114 
115     // TreeMap
116     cl = TreeMap.class;
117     expected = ClassSize.estimateBase(cl, false);
118     actual = ClassSize.TREEMAP;
119     if(expected != actual) {
120       ClassSize.estimateBase(cl, true);
121       assertEquals(expected, actual);
122     }
123 
124     // String
125     cl = String.class;
126     expected = ClassSize.estimateBase(cl, false);
127     actual = ClassSize.STRING;
128     if(expected != actual) {
129       ClassSize.estimateBase(cl, true);
130       assertEquals(expected, actual);
131     }
132 
133     // ConcurrentHashMap
134     cl = ConcurrentHashMap.class;
135     expected = ClassSize.estimateBase(cl, false);
136     actual = ClassSize.CONCURRENT_HASHMAP;
137     if(expected != actual) {
138       ClassSize.estimateBase(cl, true);
139       assertEquals(expected, actual);
140     }
141 
142     // ConcurrentSkipListMap
143     cl = ConcurrentSkipListMap.class;
144     expected = ClassSize.estimateBase(cl, false);
145     actual = ClassSize.CONCURRENT_SKIPLISTMAP;
146     if(expected != actual) {
147       ClassSize.estimateBase(cl, true);
148       assertEquals(expected, actual);
149     }
150 
151     // ReentrantReadWriteLock
152     cl = ReentrantReadWriteLock.class;
153     expected = ClassSize.estimateBase(cl, false);
154     actual = ClassSize.REENTRANT_LOCK;
155     if(expected != actual) {
156       ClassSize.estimateBase(cl, true);
157       assertEquals(expected, actual);
158     }
159 
160     // AtomicLong
161     cl = AtomicLong.class;
162     expected = ClassSize.estimateBase(cl, false);
163     actual = ClassSize.ATOMIC_LONG;
164     if(expected != actual) {
165       ClassSize.estimateBase(cl, true);
166       assertEquals(expected, actual);
167     }
168 
169     // AtomicInteger
170     cl = AtomicInteger.class;
171     expected = ClassSize.estimateBase(cl, false);
172     actual = ClassSize.ATOMIC_INTEGER;
173     if(expected != actual) {
174       ClassSize.estimateBase(cl, true);
175       assertEquals(expected, actual);
176     }
177 
178     // AtomicBoolean
179     cl = AtomicBoolean.class;
180     expected = ClassSize.estimateBase(cl, false);
181     actual = ClassSize.ATOMIC_BOOLEAN;
182     if(expected != actual) {
183       ClassSize.estimateBase(cl, true);
184       assertEquals(expected, actual);
185     }
186 
187     // CopyOnWriteArraySet
188     cl = CopyOnWriteArraySet.class;
189     expected = ClassSize.estimateBase(cl, false);
190     actual = ClassSize.COPYONWRITE_ARRAYSET;
191     if(expected != actual) {
192       ClassSize.estimateBase(cl, true);
193       assertEquals(expected, actual);
194     }
195 
196     // CopyOnWriteArrayList
197     cl = CopyOnWriteArrayList.class;
198     expected = ClassSize.estimateBase(cl, false);
199     actual = ClassSize.COPYONWRITE_ARRAYLIST;
200     if(expected != actual) {
201       ClassSize.estimateBase(cl, true);
202       assertEquals(expected, actual);
203     }
204 
205 
206   }
207 
208   /**
209    * Testing the classes that implements HeapSize and are a part of 0.20.
210    * Some are not tested here for example BlockIndex which is tested in
211    * TestHFile since it is a non public class
212    * @throws IOException
213    */
214   public void testSizes() throws IOException {
215     @SuppressWarnings("rawtypes")
216     Class cl = null;
217     long expected = 0L;
218     long actual = 0L;
219 
220     //KeyValue
221     cl = KeyValue.class;
222     expected = ClassSize.estimateBase(cl, false);
223     KeyValue kv = new KeyValue();
224     actual = kv.heapSize();
225     if(expected != actual) {
226       ClassSize.estimateBase(cl, true);
227       assertEquals(expected, actual);
228     }
229 
230     //Put
231     cl = Put.class;
232     expected = ClassSize.estimateBase(cl, false);
233     //The actual TreeMap is not included in the above calculation
234     expected += ClassSize.TREEMAP;
235     Put put = new Put(Bytes.toBytes(""));
236     actual = put.heapSize();
237     if(expected != actual) {
238       ClassSize.estimateBase(cl, true);
239       assertEquals(expected, actual);
240     }
241 
242     //LruBlockCache Overhead
243     cl = LruBlockCache.class;
244     actual = LruBlockCache.CACHE_FIXED_OVERHEAD;
245     expected = ClassSize.estimateBase(cl, false);
246     if(expected != actual) {
247       ClassSize.estimateBase(cl, true);
248       assertEquals(expected, actual);
249     }
250 
251     // CachedBlock Fixed Overhead
252     // We really need "deep" sizing but ClassSize does not do this.
253     // Perhaps we should do all these more in this style....
254     cl = CachedBlock.class;
255     actual = CachedBlock.PER_BLOCK_OVERHEAD;
256     expected = ClassSize.estimateBase(cl, false);
257     expected += ClassSize.estimateBase(String.class, false);
258     expected += ClassSize.estimateBase(ByteBuffer.class, false);
259     if(expected != actual) {
260       ClassSize.estimateBase(cl, true);
261       ClassSize.estimateBase(String.class, true);
262       ClassSize.estimateBase(ByteBuffer.class, true);
263       assertEquals(expected, actual);
264     }
265 
266     // MemStore Overhead
267     cl = MemStore.class;
268     actual = MemStore.FIXED_OVERHEAD;
269     expected = ClassSize.estimateBase(cl, false);
270     if(expected != actual) {
271       ClassSize.estimateBase(cl, true);
272       assertEquals(expected, actual);
273     }
274 
275     // MemStore Deep Overhead
276     actual = MemStore.DEEP_OVERHEAD;
277     expected = ClassSize.estimateBase(cl, false);
278     expected += ClassSize.estimateBase(ReentrantReadWriteLock.class, false);
279     expected += ClassSize.estimateBase(AtomicLong.class, false);
280     expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
281     expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
282     expected += ClassSize.estimateBase(CopyOnWriteArraySet.class, false);
283     expected += ClassSize.estimateBase(CopyOnWriteArrayList.class, false);
284     if(expected != actual) {
285       ClassSize.estimateBase(cl, true);
286       ClassSize.estimateBase(ReentrantReadWriteLock.class, true);
287       ClassSize.estimateBase(AtomicLong.class, true);
288       ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
289       ClassSize.estimateBase(CopyOnWriteArraySet.class, true);
290       ClassSize.estimateBase(CopyOnWriteArrayList.class, true);
291       assertEquals(expected, actual);
292     }
293 
294     // Store Overhead
295     cl = Store.class;
296     actual = Store.FIXED_OVERHEAD;
297     expected = ClassSize.estimateBase(cl, false);
298     if(expected != actual) {
299       ClassSize.estimateBase(cl, true);
300       assertEquals(expected, actual);
301     }
302 
303     // Region Overhead
304     cl = HRegion.class;
305     actual = HRegion.FIXED_OVERHEAD;
306     expected = ClassSize.estimateBase(cl, false);
307     if(expected != actual) {
308       ClassSize.estimateBase(cl, true);
309       assertEquals(expected, actual);
310     }
311 
312     // Currently NOT testing Deep Overheads of many of these classes.
313     // Deep overheads cover a vast majority of stuff, but will not be 100%
314     // accurate because it's unclear when we're referencing stuff that's already
315     // accounted for.  But we have satisfied our two core requirements.
316     // Sizing is quite accurate now, and our tests will throw errors if
317     // any of these classes are modified without updating overhead sizes.
318 
319   }
320 
321 }