View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.io.hfile;
19  
20  import java.io.IOException;
21  import java.util.NavigableMap;
22  import java.util.NavigableSet;
23  import java.util.concurrent.ConcurrentSkipListMap;
24  import java.util.concurrent.ConcurrentSkipListSet;
25  
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.conf.Configuration;
28  import org.codehaus.jackson.JsonGenerationException;
29  import org.codehaus.jackson.annotate.JsonIgnoreProperties;
30  import org.codehaus.jackson.map.JsonMappingException;
31  import org.codehaus.jackson.map.ObjectMapper;
32  import org.codehaus.jackson.map.SerializationConfig;
33  
34  import com.yammer.metrics.core.Histogram;
35  import com.yammer.metrics.core.MetricsRegistry;
36  import com.yammer.metrics.stats.Snapshot;
37  
38  /**
39   * Utilty for aggregating counts in CachedBlocks and toString/toJSON CachedBlocks and BlockCaches.
40   * No attempt has been made at making this thread safe.
41   */
42  @InterfaceAudience.Private
43  public class BlockCacheUtil {
44    /**
45     * Needed making histograms.
46     */
47    private static final MetricsRegistry METRICS = new MetricsRegistry();
48  
49    /**
50     * Needed generating JSON.
51     */
52    private static final ObjectMapper MAPPER = new ObjectMapper();
53    static {
54      MAPPER.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false);
55      MAPPER.configure(SerializationConfig.Feature.FLUSH_AFTER_WRITE_VALUE, true);
56      MAPPER.configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
57    }
58  
59    /**
60     * @param cb
61     * @return The block content as String.
62     */
63    public static String toString(final CachedBlock cb, final long now) {
64      return "filename=" + cb.getFilename() + ", " + toStringMinusFileName(cb, now);
65    }
66  
67    /**
68     * Little data structure to hold counts for a file.
69     * Used doing a toJSON.
70     */
71    static class CachedBlockCountsPerFile {
72      private int count = 0;
73      private long size = 0;
74      private int countData = 0;
75      private long sizeData = 0;
76      private final String filename;
77  
78      CachedBlockCountsPerFile(final String filename) {
79        this.filename = filename;
80      }
81  
82      public int getCount() {
83        return count;
84      }
85  
86      public long getSize() {
87        return size;
88      }
89  
90      public int getCountData() {
91        return countData;
92      }
93  
94      public long getSizeData() {
95        return sizeData;
96      }
97  
98      public String getFilename() {
99        return filename;
100     }
101   }
102 
103   /**
104    * @param filename
105    * @param blocks
106    * @return A JSON String of <code>filename</code> and counts of <code>blocks</code>
107    * @throws JsonGenerationException
108    * @throws JsonMappingException
109    * @throws IOException
110    */
111   public static String toJSON(final String filename, final NavigableSet<CachedBlock> blocks)
112   throws JsonGenerationException, JsonMappingException, IOException {
113     CachedBlockCountsPerFile counts = new CachedBlockCountsPerFile(filename);
114     for (CachedBlock cb: blocks) {
115       counts.count++;
116       counts.size += cb.getSize();
117       BlockType bt = cb.getBlockType();
118       if (bt != null && bt.isData()) {
119         counts.countData++;
120         counts.sizeData += cb.getSize();
121       }
122     }
123     return MAPPER.writeValueAsString(counts);
124   }
125 
126   /**
127    * @param cbsbf
128    * @return JSON string of <code>cbsf</code> aggregated
129    * @throws JsonGenerationException
130    * @throws JsonMappingException
131    * @throws IOException
132    */
133   public static String toJSON(final CachedBlocksByFile cbsbf)
134   throws JsonGenerationException, JsonMappingException, IOException {
135     return MAPPER.writeValueAsString(cbsbf);
136   }
137 
138   /**
139    * @param bc
140    * @return JSON string of <code>bc</code> content.
141    * @throws JsonGenerationException
142    * @throws JsonMappingException
143    * @throws IOException
144    */
145   public static String toJSON(final BlockCache bc)
146   throws JsonGenerationException, JsonMappingException, IOException {
147     return MAPPER.writeValueAsString(bc);
148   }
149 
150   /**
151    * @param cb
152    * @return The block content of <code>bc</code> as a String minus the filename.
153    */
154   public static String toStringMinusFileName(final CachedBlock cb, final long now) {
155     return "offset=" + cb.getOffset() +
156       ", size=" + cb.getSize() +
157       ", age=" + (now - cb.getCachedTime()) +
158       ", type=" + cb.getBlockType() +
159       ", priority=" + cb.getBlockPriority();
160   }
161 
162   /**
163    * Snapshot of block cache age in cache.
164    * This object is preferred because we can control how it is serialized out when JSON'ing.
165    */
166   @JsonIgnoreProperties({"ageHistogram", "snapshot"})
167   public static class AgeSnapshot {
168     private final Histogram ageHistogram;
169     private final Snapshot snapshot;
170 
171     AgeSnapshot(final Histogram ageHistogram) {
172       this.ageHistogram = ageHistogram;
173       this.snapshot = ageHistogram.getSnapshot();
174     }
175 
176     public double get75thPercentile() {
177       return snapshot.get75thPercentile();
178     }
179 
180     public double get95thPercentile() {
181       return snapshot.get95thPercentile();
182     }
183 
184     public double get98thPercentile() {
185       return snapshot.get98thPercentile();
186     }
187 
188     public double get999thPercentile() {
189       return snapshot.get999thPercentile();
190     }
191 
192     public double get99thPercentile() {
193       return snapshot.get99thPercentile();
194     }
195 
196     public double getMean() {
197       return this.ageHistogram.mean();
198     }
199 
200     public double getMax() {
201       return ageHistogram.max();
202     }
203 
204     public double getMin() {
205       return ageHistogram.min();
206     }
207 
208     public double getStdDev() {
209       return ageHistogram.stdDev();
210     }
211   }
212 
213   /**
214    * Get a {@link CachedBlocksByFile} instance and load it up by iterating content in
215    * {@link BlockCache}.
216    * @param conf Used to read configurations
217    * @param bc Block Cache to iterate.
218    * @return Laoded up instance of CachedBlocksByFile
219    */
220   public static CachedBlocksByFile getLoadedCachedBlocksByFile(final Configuration conf,
221       final BlockCache bc) {
222     CachedBlocksByFile cbsbf = new CachedBlocksByFile(conf);
223     for (CachedBlock cb: bc) {
224       if (cbsbf.update(cb)) break;
225     }
226     return cbsbf;
227   }
228 
229   /**
230    * Use one of these to keep a running account of cached blocks by file.  Throw it away when done.
231    * This is different than metrics in that it is stats on current state of a cache.
232    * @see getLoadedCachedBlocksByFile
233    */
234   @JsonIgnoreProperties({"cachedBlockStatsByFile"})
235   public static class CachedBlocksByFile {
236     private int count;
237     private int dataBlockCount;
238     private long size;
239     private long dataSize;
240     private final long now = System.nanoTime();
241     /**
242      * How many blocks to look at before we give up.
243      * There could be many millions of blocks. We don't want the
244      * ui to freeze while we run through 1B blocks... users will
245      * think hbase dead. UI displays warning in red when stats
246      * are incomplete.
247      */
248     private final int max;
249     public static final int DEFAULT_MAX = 1000000;
250 
251     CachedBlocksByFile() {
252       this(null);
253     }
254 
255     CachedBlocksByFile(final Configuration c) {
256       this.max = c == null? DEFAULT_MAX: c.getInt("hbase.ui.blockcache.by.file.max", DEFAULT_MAX);
257     }
258 
259     /**
260      * Map by filename. use concurent utils because we want our Map and contained blocks sorted.
261      */
262     private NavigableMap<String, NavigableSet<CachedBlock>> cachedBlockByFile =
263       new ConcurrentSkipListMap<String, NavigableSet<CachedBlock>>();
264     Histogram age = METRICS.newHistogram(CachedBlocksByFile.class, "age");
265 
266     /**
267      * @param cb
268      * @return True if full.... if we won't be adding any more.
269      */
270     public boolean update(final CachedBlock cb) {
271       if (isFull()) return true;
272       NavigableSet<CachedBlock> set = this.cachedBlockByFile.get(cb.getFilename());
273       if (set == null) {
274         set = new ConcurrentSkipListSet<CachedBlock>();
275         this.cachedBlockByFile.put(cb.getFilename(), set);
276       }
277       set.add(cb);
278       this.size += cb.getSize();
279       this.count++;
280       BlockType bt = cb.getBlockType();
281       if (bt != null && bt.isData()) {
282         this.dataBlockCount++;
283         this.dataSize += cb.getSize();
284       }
285       long age = this.now - cb.getCachedTime();
286       this.age.update(age);
287       return false;
288     }
289 
290     /**
291      * @return True if full; i.e. there are more items in the cache but we only loaded up
292      * the maximum set in configuration <code>hbase.ui.blockcache.by.file.max</code>
293      * (Default: DEFAULT_MAX).
294      */
295     public boolean isFull() {
296       return this.count >= this.max;
297     }
298 
299     public NavigableMap<String, NavigableSet<CachedBlock>> getCachedBlockStatsByFile() {
300       return this.cachedBlockByFile;
301     }
302 
303     /**
304      * @return count of blocks in the cache
305      */
306     public int getCount() {
307       return count;
308     }
309 
310     public int getDataCount() {
311       return dataBlockCount;
312     }
313 
314     /**
315      * @return size of blocks in the cache
316      */
317     public long getSize() {
318       return size;
319     }
320 
321     /**
322      * @return Size of data.
323      */
324     public long getDataSize() {
325       return dataSize;
326     }
327 
328     public AgeSnapshot getAgeSnapshot() {
329       return new AgeSnapshot(this.age);
330     }
331 
332     @Override
333     public String toString() {
334       Snapshot snapshot = this.age.getSnapshot();
335       return "count=" + count + ", dataBlockCount=" + this.dataBlockCount + ", size=" + size +
336           ", dataSize=" + getDataSize() +
337           ", mean age=" + this.age.mean() + ", stddev age=" + this.age.stdDev() +
338           ", min age=" + this.age.min() + ", max age=" + this.age.max() +
339           ", 95th percentile age=" + snapshot.get95thPercentile() +
340           ", 99th percentile age=" + snapshot.get99thPercentile();
341     }
342   }
343 }