View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver.metrics;
21  
22  import java.io.IOException;
23  import java.lang.management.ManagementFactory;
24  import java.lang.management.MemoryUsage;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.io.hfile.HFile;
30  import org.apache.hadoop.hbase.metrics.HBaseInfo;
31  import org.apache.hadoop.hbase.metrics.MetricsRate;
32  import org.apache.hadoop.hbase.metrics.PersistentMetricsTimeVaryingRate;
33  import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
34  import com.yammer.metrics.stats.Snapshot;
35  import org.apache.hadoop.hbase.regionserver.wal.HLog;
36  import org.apache.hadoop.hbase.util.Pair;
37  import org.apache.hadoop.hbase.util.Strings;
38  import org.apache.hadoop.metrics.ContextFactory;
39  import org.apache.hadoop.metrics.MetricsContext;
40  import org.apache.hadoop.metrics.MetricsRecord;
41  import org.apache.hadoop.metrics.MetricsUtil;
42  import org.apache.hadoop.metrics.Updater;
43  import org.apache.hadoop.metrics.jvm.JvmMetrics;
44  import org.apache.hadoop.metrics.util.MetricsIntValue;
45  import org.apache.hadoop.metrics.util.MetricsLongValue;
46  import org.apache.hadoop.metrics.util.MetricsRegistry;
47  import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
48  import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong;
49  import org.apache.hadoop.util.StringUtils;
50  
51  /**
52   * This class is for maintaining the various regionserver statistics
53   * and publishing them through the metrics interfaces.
54   * <p>
55   * This class has a number of metrics variables that are publicly accessible;
56   * these variables (objects) have methods to update their values.
57   */
58  public class RegionServerMetrics implements Updater {
59    @SuppressWarnings({"FieldCanBeLocal"})
60    private final Log LOG = LogFactory.getLog(this.getClass());
61    private final MetricsRecord metricsRecord;
62    private long lastUpdate = System.currentTimeMillis();
63    private long lastExtUpdate = System.currentTimeMillis();
64    private long extendedPeriod = 0;
65    private static final int MB = 1024*1024;
66    private MetricsRegistry registry = new MetricsRegistry();
67    private final RegionServerStatistics statistics;
68  
69    public final MetricsTimeVaryingRate atomicIncrementTime =
70        new MetricsTimeVaryingRate("atomicIncrementTime", registry);
71  
72    /**
73     * Count of regions carried by this regionserver
74     */
75    public final MetricsIntValue regions =
76      new MetricsIntValue("regions", registry);
77  
78    /**
79     * Block cache size.
80     */
81    public final MetricsLongValue blockCacheSize = 
82        new MetricsLongValue("blockCacheSize", registry);
83  
84    /**
85     * Block cache free size.
86     */
87    public final MetricsLongValue blockCacheFree = 
88        new MetricsLongValue("blockCacheFree", registry);
89  
90    /**
91     * Block cache item count.
92     */
93    public final MetricsLongValue blockCacheCount = 
94        new MetricsLongValue("blockCacheCount", registry);
95  
96    /**
97     * Block cache hit count.
98     */
99    public final MetricsLongValue blockCacheHitCount = 
100       new MetricsLongValue("blockCacheHitCount", registry);
101 
102   /**
103    * Block cache miss count.
104    */
105   public final MetricsLongValue blockCacheMissCount = 
106       new MetricsLongValue("blockCacheMissCount", registry);
107 
108   /**
109    * Block cache evict count.
110    */
111   public final MetricsLongValue blockCacheEvictedCount = 
112       new MetricsLongValue("blockCacheEvictedCount", registry);
113 
114   /**
115    * Block hit ratio.
116    */
117   public final MetricsIntValue blockCacheHitRatio = 
118       new MetricsIntValue("blockCacheHitRatio", registry);
119 
120   /**
121    * Block hit caching ratio.  This only includes the requests to the block
122    * cache where caching was turned on.  See HBASE-2253.
123    */
124   public final MetricsIntValue blockCacheHitCachingRatio = 
125       new MetricsIntValue("blockCacheHitCachingRatio", registry);
126 
127   /** Block hit ratio for past N periods. */
128   public final MetricsIntValue blockCacheHitRatioPastNPeriods = new MetricsIntValue("blockCacheHitRatioPastNPeriods", registry);
129 
130   /** Block hit caching ratio for past N periods */
131   public final MetricsIntValue blockCacheHitCachingRatioPastNPeriods = new MetricsIntValue("blockCacheHitCachingRatioPastNPeriods", registry);
132 
133   /*
134    * Count of requests to the regionservers since last call to metrics update
135    */
136   public final MetricsRate requests = new MetricsRate("requests", registry);
137 
138   /**
139    * Count of stores open on the regionserver.
140    */
141   public final MetricsIntValue stores = new MetricsIntValue("stores", registry);
142 
143   /**
144    * Count of storefiles open on the regionserver.
145    */
146   public final MetricsIntValue storefiles = 
147       new MetricsIntValue("storefiles", registry);
148 
149   /**
150    * Count of read requests
151    */
152   public final MetricsLongValue readRequestsCount = 
153       new MetricsLongValue("readRequestsCount", registry);
154 
155   /**
156    * Count of write requests
157    */
158   public final MetricsLongValue writeRequestsCount = 
159       new MetricsLongValue("writeRequestsCount", registry);
160 
161   /**
162    */
163   public final MetricsIntValue storefileIndexSizeMB =
164     new MetricsIntValue("storefileIndexSizeMB", registry);
165 
166   /** The total size of block index root levels in this regionserver in KB. */
167   public final MetricsIntValue rootIndexSizeKB =
168     new MetricsIntValue("rootIndexSizeKB", registry);
169 
170   /** Total size of all block indexes (not necessarily loaded in memory) */
171   public final MetricsIntValue totalStaticIndexSizeKB =
172     new MetricsIntValue("totalStaticIndexSizeKB", registry);
173 
174   /** Total size of all Bloom filters (not necessarily loaded in memory) */
175   public final MetricsIntValue totalStaticBloomSizeKB =
176     new MetricsIntValue("totalStaticBloomSizeKB", registry);
177 
178   /**
179    * HDFS blocks locality index
180    */
181   public final MetricsIntValue hdfsBlocksLocalityIndex =
182     new MetricsIntValue("hdfsBlocksLocalityIndex", registry);
183   
184   /**
185    * Sum of all the memstore sizes in this regionserver in MB
186    */
187   public final MetricsIntValue memstoreSizeMB =
188     new MetricsIntValue("memstoreSizeMB", registry);
189 
190   /**
191    * Number of put with WAL disabled in this regionserver in MB
192    */
193   public final MetricsLongValue numPutsWithoutWAL =
194     new MetricsLongValue("numPutsWithoutWAL", registry);
195 
196   /**
197    * Possible data loss sizes (due to put with WAL disabled) in this regionserver in MB
198    */
199   public final MetricsIntValue mbInMemoryWithoutWAL =
200     new MetricsIntValue("mbInMemoryWithoutWAL", registry);
201 
202   /**
203    * Size of the compaction queue.
204    */
205   public final MetricsIntValue compactionQueueSize =
206     new MetricsIntValue("compactionQueueSize", registry);
207   
208   /**
209    * Size of the flush queue.
210    */
211   public final MetricsIntValue flushQueueSize =
212     new MetricsIntValue("flushQueueSize", registry);
213 
214   /**
215    * filesystem sequential read latency distribution
216    */
217   public final MetricsHistogram fsReadLatencyHistogram = 
218       new MetricsHistogram("fsReadLatencyHistogram", registry);
219 
220   /**
221    * filesystem pread latency distribution
222    */
223   public final MetricsHistogram fsPreadLatencyHistogram = 
224       new MetricsHistogram("fsPreadLatencyHistogram", registry);
225 
226   /**
227    * Metrics on the distribution of filesystem write latencies (improved version of fsWriteLatency)
228    */
229   public final MetricsHistogram fsWriteLatencyHistogram = 
230       new MetricsHistogram("fsWriteLatencyHistogram", registry);
231 
232   
233   /**
234    * filesystem read latency
235    */
236   public final MetricsTimeVaryingRate fsReadLatency =
237     new MetricsTimeVaryingRate("fsReadLatency", registry);
238 
239   /**
240    * filesystem positional read latency
241    */
242   public final MetricsTimeVaryingRate fsPreadLatency =
243     new MetricsTimeVaryingRate("fsPreadLatency", registry);
244 
245   /**
246    * filesystem write latency
247    */
248   public final MetricsTimeVaryingRate fsWriteLatency =
249     new MetricsTimeVaryingRate("fsWriteLatency", registry);
250 
251   /**
252    * size (in bytes) of data in HLog append calls
253    */
254   public final MetricsTimeVaryingRate fsWriteSize =
255     new MetricsTimeVaryingRate("fsWriteSize", registry);
256 
257   /**
258    * filesystem sync latency
259    */
260   public final MetricsTimeVaryingRate fsSyncLatency =
261     new MetricsTimeVaryingRate("fsSyncLatency", registry);
262 
263   
264   /**
265    * time each scheduled compaction takes
266    */
267   protected final PersistentMetricsTimeVaryingRate compactionTime =
268     new PersistentMetricsTimeVaryingRate("compactionTime", registry);
269 
270   protected final PersistentMetricsTimeVaryingRate compactionSize =
271     new PersistentMetricsTimeVaryingRate("compactionSize", registry);
272 
273   /**
274    * time each scheduled flush takes
275    */
276   protected final PersistentMetricsTimeVaryingRate flushTime =
277     new PersistentMetricsTimeVaryingRate("flushTime", registry);
278 
279   protected final PersistentMetricsTimeVaryingRate flushSize =
280     new PersistentMetricsTimeVaryingRate("flushSize", registry);
281   
282   public final MetricsLongValue slowHLogAppendCount =
283       new MetricsLongValue("slowHLogAppendCount", registry);
284   
285   public final MetricsTimeVaryingRate slowHLogAppendTime =
286       new MetricsTimeVaryingRate("slowHLogAppendTime", registry);
287   
288   public final MetricsTimeVaryingLong regionSplitSuccessCount =
289       new MetricsTimeVaryingLong("regionSplitSuccessCount", registry);
290   
291   public final MetricsTimeVaryingLong regionSplitFailureCount =
292       new MetricsTimeVaryingLong("regionSplitFailureCount", registry);
293 
294   /**
295    * Number of times checksum verification failed.
296    */
297   public final MetricsLongValue checksumFailuresCount =
298     new MetricsLongValue("checksumFailuresCount", registry);
299 
300   /**
301    * time blocked on lack of resources
302    */
303   public final MetricsLongValue updatesBlockedSeconds = new MetricsLongValue(
304       "updatesBlockedSeconds", registry);
305 
306   /**
307    * time blocked on memstoreHW
308    */
309   public final MetricsLongValue updatesBlockedSecondsHighWater = new MetricsLongValue(
310       "updatesBlockedSecondsHighWater",registry);
311 
312   public RegionServerMetrics() {
313     MetricsContext context = MetricsUtil.getContext("hbase");
314     metricsRecord = MetricsUtil.createRecord(context, "regionserver");
315     String name = Thread.currentThread().getName();
316     metricsRecord.setTag("RegionServer", name);
317     context.registerUpdater(this);
318     // Add jvmmetrics.
319     JvmMetrics.init("RegionServer", name);
320     // Add Hbase Info metrics
321     HBaseInfo.init();
322 
323     // export for JMX
324     statistics = new RegionServerStatistics(this.registry, name);
325 
326     // get custom attributes
327     try {
328       Object m = ContextFactory.getFactory().getAttribute("hbase.extendedperiod");
329       if (m instanceof String) {
330         this.extendedPeriod = Long.parseLong((String) m)*1000;
331       }
332     } catch (IOException ioe) {
333       LOG.info("Couldn't load ContextFactory for Metrics config info");
334     }
335 
336     LOG.info("Initialized");
337   }
338 
339   public void shutdown() {
340     if (statistics != null)
341       statistics.shutdown();
342   }
343 
344   /**
345    * Since this object is a registered updater, this method will be called
346    * periodically, e.g. every 5 seconds.
347    * @param caller the metrics context that this responsible for calling us
348    */
349   public void doUpdates(MetricsContext caller) {
350     synchronized (this) {
351       this.lastUpdate = System.currentTimeMillis();
352 
353       // has the extended period for long-living stats elapsed?
354       if (this.extendedPeriod > 0 &&
355           this.lastUpdate - this.lastExtUpdate >= this.extendedPeriod) {
356         this.lastExtUpdate = this.lastUpdate;
357         this.compactionTime.resetMinMaxAvg();
358         this.compactionSize.resetMinMaxAvg();
359         this.flushTime.resetMinMaxAvg();
360         this.flushSize.resetMinMaxAvg();
361         this.resetAllMinMax();
362       }
363 
364       this.stores.pushMetric(this.metricsRecord);
365       this.storefiles.pushMetric(this.metricsRecord);
366       this.storefileIndexSizeMB.pushMetric(this.metricsRecord);
367       this.rootIndexSizeKB.pushMetric(this.metricsRecord);
368       this.totalStaticIndexSizeKB.pushMetric(this.metricsRecord);
369       this.totalStaticBloomSizeKB.pushMetric(this.metricsRecord);
370       this.memstoreSizeMB.pushMetric(this.metricsRecord);
371       this.mbInMemoryWithoutWAL.pushMetric(this.metricsRecord);
372       this.numPutsWithoutWAL.pushMetric(this.metricsRecord);
373       this.readRequestsCount.pushMetric(this.metricsRecord);
374       this.writeRequestsCount.pushMetric(this.metricsRecord);
375       this.regions.pushMetric(this.metricsRecord);
376       this.requests.pushMetric(this.metricsRecord);
377       this.compactionQueueSize.pushMetric(this.metricsRecord);
378       this.flushQueueSize.pushMetric(this.metricsRecord);
379       this.blockCacheSize.pushMetric(this.metricsRecord);
380       this.blockCacheFree.pushMetric(this.metricsRecord);
381       this.blockCacheCount.pushMetric(this.metricsRecord);
382       this.blockCacheHitCount.pushMetric(this.metricsRecord);
383       this.blockCacheMissCount.pushMetric(this.metricsRecord);
384       this.blockCacheEvictedCount.pushMetric(this.metricsRecord);
385       this.blockCacheHitRatio.pushMetric(this.metricsRecord);
386       this.blockCacheHitCachingRatio.pushMetric(this.metricsRecord);
387       this.hdfsBlocksLocalityIndex.pushMetric(this.metricsRecord);
388       this.blockCacheHitRatioPastNPeriods.pushMetric(this.metricsRecord);
389       this.blockCacheHitCachingRatioPastNPeriods.pushMetric(this.metricsRecord);
390 
391       // Mix in HFile and HLog metrics
392       // Be careful. Here is code for MTVR from up in hadoop:
393       // public synchronized void inc(final int numOps, final long time) {
394       //   currentData.numOperations += numOps;
395       //   currentData.time += time;
396       //   long timePerOps = time/numOps;
397       //    minMax.update(timePerOps);
398       // }
399       // Means you can't pass a numOps of zero or get a ArithmeticException / by zero.
400       // HLog metrics
401       addHLogMetric(HLog.getWriteTime(), this.fsWriteLatency);
402       addHLogMetric(HLog.getWriteSize(), this.fsWriteSize);
403       addHLogMetric(HLog.getSyncTime(), this.fsSyncLatency);
404       addHLogMetric(HLog.getSlowAppendTime(), this.slowHLogAppendTime);
405       this.slowHLogAppendCount.set(HLog.getSlowAppendCount());
406       // HFile metrics, sequential reads
407       int ops = HFile.getReadOps(); 
408       if (ops != 0) this.fsReadLatency.inc(ops, HFile.getReadTimeMs());
409       // HFile metrics, positional reads
410       ops = HFile.getPreadOps(); 
411       if (ops != 0) this.fsPreadLatency.inc(ops, HFile.getPreadTimeMs());
412       this.checksumFailuresCount.set(HFile.getChecksumFailuresCount());
413 
414       /* NOTE: removed HFile write latency.  2 reasons:
415        * 1) Mixing HLog latencies are far higher priority since they're 
416        *      on-demand and HFile is used in background (compact/flush)
417        * 2) HFile metrics are being handled at a higher level 
418        *      by compaction & flush metrics.
419        */
420 
421       for(Long latency : HFile.getReadLatenciesNanos()) {
422         this.fsReadLatencyHistogram.update(latency);
423       }
424       for(Long latency : HFile.getPreadLatenciesNanos()) {
425         this.fsPreadLatencyHistogram.update(latency);
426       }
427       for(Long latency : HFile.getWriteLatenciesNanos()) {
428         this.fsWriteLatencyHistogram.update(latency);
429       }
430             
431 
432       // push the result
433       this.fsPreadLatency.pushMetric(this.metricsRecord);
434       this.fsReadLatency.pushMetric(this.metricsRecord);
435       this.fsWriteLatency.pushMetric(this.metricsRecord);
436       this.fsWriteSize.pushMetric(this.metricsRecord);
437       
438       this.fsReadLatencyHistogram.pushMetric(this.metricsRecord);
439       this.fsWriteLatencyHistogram.pushMetric(this.metricsRecord);
440       this.fsPreadLatencyHistogram.pushMetric(this.metricsRecord);
441 
442       this.fsSyncLatency.pushMetric(this.metricsRecord);
443       this.compactionTime.pushMetric(this.metricsRecord);
444       this.compactionSize.pushMetric(this.metricsRecord);
445       this.flushTime.pushMetric(this.metricsRecord);
446       this.flushSize.pushMetric(this.metricsRecord);
447       this.slowHLogAppendCount.pushMetric(this.metricsRecord);
448       this.regionSplitSuccessCount.pushMetric(this.metricsRecord);
449       this.regionSplitFailureCount.pushMetric(this.metricsRecord);
450       this.checksumFailuresCount.pushMetric(this.metricsRecord);
451       this.updatesBlockedSeconds.pushMetric(this.metricsRecord);
452       this.updatesBlockedSecondsHighWater.pushMetric(this.metricsRecord);
453     }
454     this.metricsRecord.update();
455   }
456 
457   private void addHLogMetric(HLog.Metric logMetric,
458       MetricsTimeVaryingRate hadoopMetric) {
459     if (logMetric.count > 0)
460       hadoopMetric.inc(logMetric.min);
461     if (logMetric.count > 1)
462       hadoopMetric.inc(logMetric.max);
463     if (logMetric.count > 2) {
464       int ops = logMetric.count - 2;
465       hadoopMetric.inc(ops, logMetric.total - logMetric.max - logMetric.min);
466     }
467   }
468 
469   public void resetAllMinMax() {
470     this.atomicIncrementTime.resetMinMax();
471     this.fsReadLatency.resetMinMax();
472     this.fsWriteLatency.resetMinMax();
473     this.fsWriteSize.resetMinMax();
474     this.fsSyncLatency.resetMinMax();
475     this.slowHLogAppendTime.resetMinMax();
476   }
477 
478   /**
479    * @return Count of requests.
480    */
481   public float getRequests() {
482     return this.requests.getPreviousIntervalValue();
483   }
484 
485   /**
486    * @param time time that compaction took
487    * @param size bytesize of storefiles in the compaction
488    */
489   public synchronized void addCompaction(long time, long size) {
490     this.compactionTime.inc(time);
491     this.compactionSize.inc(size);
492   }
493 
494   /**
495    * @param flushes history in <time, size>
496    */
497   public synchronized void addFlush(final List<Pair<Long,Long>> flushes) {
498     for (Pair<Long,Long> f : flushes) {
499       this.flushTime.inc(f.getFirst());
500       this.flushSize.inc(f.getSecond());
501     }
502   }
503 
504   /**
505    * @param inc How much to add to requests.
506    */
507   public void incrementRequests(final int inc) {
508     this.requests.inc(inc);
509   }
510   
511   public void incrementSplitSuccessCount() {
512     this.regionSplitSuccessCount.inc();
513   }
514   
515   public void incrementSplitFailureCount() {
516     this.regionSplitFailureCount.inc();
517   }
518 
519   @Override
520   public String toString() {
521     StringBuilder sb = new StringBuilder();
522     sb = Strings.appendKeyValue(sb, "requestsPerSecond", Integer
523         .valueOf((int) this.requests.getPreviousIntervalValue()));
524     sb = Strings.appendKeyValue(sb, "numberOfOnlineRegions",
525       Integer.valueOf(this.regions.get()));
526     sb = Strings.appendKeyValue(sb, "numberOfStores",
527       Integer.valueOf(this.stores.get()));
528     sb = Strings.appendKeyValue(sb, "numberOfStorefiles",
529       Integer.valueOf(this.storefiles.get()));
530     sb = Strings.appendKeyValue(sb, this.storefileIndexSizeMB.getName(),
531       Integer.valueOf(this.storefileIndexSizeMB.get()));
532     sb = Strings.appendKeyValue(sb, "rootIndexSizeKB",
533         Integer.valueOf(this.rootIndexSizeKB.get()));
534     sb = Strings.appendKeyValue(sb, "totalStaticIndexSizeKB",
535         Integer.valueOf(this.totalStaticIndexSizeKB.get()));
536     sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB",
537         Integer.valueOf(this.totalStaticBloomSizeKB.get()));
538     sb = Strings.appendKeyValue(sb, this.memstoreSizeMB.getName(),
539       Integer.valueOf(this.memstoreSizeMB.get()));
540     sb = Strings.appendKeyValue(sb, "mbInMemoryWithoutWAL",
541       Integer.valueOf(this.mbInMemoryWithoutWAL.get()));
542     sb = Strings.appendKeyValue(sb, "numberOfPutsWithoutWAL",
543       Long.valueOf(this.numPutsWithoutWAL.get()));
544     sb = Strings.appendKeyValue(sb, "readRequestsCount",
545         Long.valueOf(this.readRequestsCount.get()));
546     sb = Strings.appendKeyValue(sb, "writeRequestsCount",
547         Long.valueOf(this.writeRequestsCount.get()));
548     sb = Strings.appendKeyValue(sb, "compactionQueueSize",
549       Integer.valueOf(this.compactionQueueSize.get()));
550     sb = Strings.appendKeyValue(sb, "flushQueueSize",
551       Integer.valueOf(this.flushQueueSize.get()));
552     // Duplicate from jvmmetrics because metrics are private there so
553     // inaccessible.
554     MemoryUsage memory =
555       ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
556     sb = Strings.appendKeyValue(sb, "usedHeapMB",
557       Long.valueOf(memory.getUsed()/MB));
558     sb = Strings.appendKeyValue(sb, "maxHeapMB",
559       Long.valueOf(memory.getMax()/MB));
560     sb = Strings.appendKeyValue(sb, this.blockCacheSize.getName()+"MB",
561     	StringUtils.limitDecimalTo2((float)this.blockCacheSize.get()/MB));
562     sb = Strings.appendKeyValue(sb, this.blockCacheFree.getName()+"MB",
563     	StringUtils.limitDecimalTo2((float)this.blockCacheFree.get()/MB));
564     sb = Strings.appendKeyValue(sb, this.blockCacheCount.getName(),
565         Long.valueOf(this.blockCacheCount.get()));
566     sb = Strings.appendKeyValue(sb, this.blockCacheHitCount.getName(),
567         Long.valueOf(this.blockCacheHitCount.get()));
568     sb = Strings.appendKeyValue(sb, this.blockCacheMissCount.getName(),
569         Long.valueOf(this.blockCacheMissCount.get()));
570     sb = Strings.appendKeyValue(sb, this.blockCacheEvictedCount.getName(),
571         Long.valueOf(this.blockCacheEvictedCount.get()));
572     sb = Strings.appendKeyValue(sb, this.blockCacheHitRatio.getName(),
573         Long.valueOf(this.blockCacheHitRatio.get())+"%");
574     sb = Strings.appendKeyValue(sb, this.blockCacheHitCachingRatio.getName(),
575         Long.valueOf(this.blockCacheHitCachingRatio.get())+"%");
576     sb = Strings.appendKeyValue(sb, this.hdfsBlocksLocalityIndex.getName(),
577         Long.valueOf(this.hdfsBlocksLocalityIndex.get()));
578     sb = Strings.appendKeyValue(sb, "slowHLogAppendCount",
579         Long.valueOf(this.slowHLogAppendCount.get()));
580     sb = appendHistogram(sb, this.fsReadLatencyHistogram);
581     sb = appendHistogram(sb, this.fsPreadLatencyHistogram);
582     sb = appendHistogram(sb, this.fsWriteLatencyHistogram);
583 
584     return sb.toString();
585   }
586   
587   private StringBuilder appendHistogram(StringBuilder sb, 
588       MetricsHistogram histogram) {
589     sb = Strings.appendKeyValue(sb, 
590         histogram.getName() + "Mean", 
591         StringUtils.limitDecimalTo2(histogram.getMean()));
592     sb = Strings.appendKeyValue(sb, 
593         histogram.getName() + "Count", 
594         StringUtils.limitDecimalTo2(histogram.getCount()));
595     final Snapshot s = histogram.getSnapshot();
596     sb = Strings.appendKeyValue(sb, 
597         histogram.getName() + "Median", 
598         StringUtils.limitDecimalTo2(s.getMedian()));
599     sb = Strings.appendKeyValue(sb, 
600         histogram.getName() + "75th", 
601         StringUtils.limitDecimalTo2(s.get75thPercentile()));
602     sb = Strings.appendKeyValue(sb, 
603         histogram.getName() + "95th", 
604         StringUtils.limitDecimalTo2(s.get95thPercentile()));
605     sb = Strings.appendKeyValue(sb, 
606         histogram.getName() + "99th", 
607         StringUtils.limitDecimalTo2(s.get99thPercentile()));
608     sb = Strings.appendKeyValue(sb, 
609         histogram.getName() + "999th", 
610         StringUtils.limitDecimalTo2(s.get999thPercentile()));
611     return sb;
612   }
613 }