1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.metrics;
21
22 import java.io.IOException;
23 import java.lang.management.ManagementFactory;
24 import java.lang.management.MemoryUsage;
25 import java.util.List;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.io.hfile.HFile;
30 import org.apache.hadoop.hbase.metrics.HBaseInfo;
31 import org.apache.hadoop.hbase.metrics.MetricsRate;
32 import org.apache.hadoop.hbase.metrics.PersistentMetricsTimeVaryingRate;
33 import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
34 import com.yammer.metrics.stats.Snapshot;
35 import org.apache.hadoop.hbase.regionserver.wal.HLog;
36 import org.apache.hadoop.hbase.util.Pair;
37 import org.apache.hadoop.hbase.util.Strings;
38 import org.apache.hadoop.metrics.ContextFactory;
39 import org.apache.hadoop.metrics.MetricsContext;
40 import org.apache.hadoop.metrics.MetricsRecord;
41 import org.apache.hadoop.metrics.MetricsUtil;
42 import org.apache.hadoop.metrics.Updater;
43 import org.apache.hadoop.metrics.jvm.JvmMetrics;
44 import org.apache.hadoop.metrics.util.MetricsIntValue;
45 import org.apache.hadoop.metrics.util.MetricsLongValue;
46 import org.apache.hadoop.metrics.util.MetricsRegistry;
47 import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
48 import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong;
49 import org.apache.hadoop.util.StringUtils;
50
51
52
53
54
55
56
57
58 public class RegionServerMetrics implements Updater {
59 @SuppressWarnings({"FieldCanBeLocal"})
60 private final Log LOG = LogFactory.getLog(this.getClass());
61 private final MetricsRecord metricsRecord;
62 private long lastUpdate = System.currentTimeMillis();
63 private long lastExtUpdate = System.currentTimeMillis();
64 private long extendedPeriod = 0;
65 private static final int MB = 1024*1024;
66 private MetricsRegistry registry = new MetricsRegistry();
67 private final RegionServerStatistics statistics;
68
69 public final MetricsTimeVaryingRate atomicIncrementTime =
70 new MetricsTimeVaryingRate("atomicIncrementTime", registry);
71
72
73
74
75 public final MetricsIntValue regions =
76 new MetricsIntValue("regions", registry);
77
78
79
80
81 public final MetricsLongValue blockCacheSize =
82 new MetricsLongValue("blockCacheSize", registry);
83
84
85
86
87 public final MetricsLongValue blockCacheFree =
88 new MetricsLongValue("blockCacheFree", registry);
89
90
91
92
93 public final MetricsLongValue blockCacheCount =
94 new MetricsLongValue("blockCacheCount", registry);
95
96
97
98
99 public final MetricsLongValue blockCacheHitCount =
100 new MetricsLongValue("blockCacheHitCount", registry);
101
102
103
104
105 public final MetricsLongValue blockCacheMissCount =
106 new MetricsLongValue("blockCacheMissCount", registry);
107
108
109
110
111 public final MetricsLongValue blockCacheEvictedCount =
112 new MetricsLongValue("blockCacheEvictedCount", registry);
113
114
115
116
117 public final MetricsIntValue blockCacheHitRatio =
118 new MetricsIntValue("blockCacheHitRatio", registry);
119
120
121
122
123
124 public final MetricsIntValue blockCacheHitCachingRatio =
125 new MetricsIntValue("blockCacheHitCachingRatio", registry);
126
127
128 public final MetricsIntValue blockCacheHitRatioPastNPeriods = new MetricsIntValue("blockCacheHitRatioPastNPeriods", registry);
129
130
131 public final MetricsIntValue blockCacheHitCachingRatioPastNPeriods = new MetricsIntValue("blockCacheHitCachingRatioPastNPeriods", registry);
132
133
134
135
136 public final MetricsRate requests = new MetricsRate("requests", registry);
137
138
139
140
141 public final MetricsIntValue stores = new MetricsIntValue("stores", registry);
142
143
144
145
146 public final MetricsIntValue storefiles =
147 new MetricsIntValue("storefiles", registry);
148
149
150
151
152 public final MetricsLongValue readRequestsCount =
153 new MetricsLongValue("readRequestsCount", registry);
154
155
156
157
158 public final MetricsLongValue writeRequestsCount =
159 new MetricsLongValue("writeRequestsCount", registry);
160
161
162
163 public final MetricsIntValue storefileIndexSizeMB =
164 new MetricsIntValue("storefileIndexSizeMB", registry);
165
166
167 public final MetricsIntValue rootIndexSizeKB =
168 new MetricsIntValue("rootIndexSizeKB", registry);
169
170
171 public final MetricsIntValue totalStaticIndexSizeKB =
172 new MetricsIntValue("totalStaticIndexSizeKB", registry);
173
174
175 public final MetricsIntValue totalStaticBloomSizeKB =
176 new MetricsIntValue("totalStaticBloomSizeKB", registry);
177
178
179
180
181 public final MetricsIntValue hdfsBlocksLocalityIndex =
182 new MetricsIntValue("hdfsBlocksLocalityIndex", registry);
183
184
185
186
187 public final MetricsIntValue memstoreSizeMB =
188 new MetricsIntValue("memstoreSizeMB", registry);
189
190
191
192
193 public final MetricsLongValue numPutsWithoutWAL =
194 new MetricsLongValue("numPutsWithoutWAL", registry);
195
196
197
198
199 public final MetricsIntValue mbInMemoryWithoutWAL =
200 new MetricsIntValue("mbInMemoryWithoutWAL", registry);
201
202
203
204
205 public final MetricsIntValue compactionQueueSize =
206 new MetricsIntValue("compactionQueueSize", registry);
207
208
209
210
211 public final MetricsIntValue flushQueueSize =
212 new MetricsIntValue("flushQueueSize", registry);
213
214
215
216
217 public final MetricsHistogram fsReadLatencyHistogram =
218 new MetricsHistogram("fsReadLatencyHistogram", registry);
219
220
221
222
223 public final MetricsHistogram fsPreadLatencyHistogram =
224 new MetricsHistogram("fsPreadLatencyHistogram", registry);
225
226
227
228
229 public final MetricsHistogram fsWriteLatencyHistogram =
230 new MetricsHistogram("fsWriteLatencyHistogram", registry);
231
232
233
234
235
236 public final MetricsTimeVaryingRate fsReadLatency =
237 new MetricsTimeVaryingRate("fsReadLatency", registry);
238
239
240
241
242 public final MetricsTimeVaryingRate fsPreadLatency =
243 new MetricsTimeVaryingRate("fsPreadLatency", registry);
244
245
246
247
248 public final MetricsTimeVaryingRate fsWriteLatency =
249 new MetricsTimeVaryingRate("fsWriteLatency", registry);
250
251
252
253
254 public final MetricsTimeVaryingRate fsWriteSize =
255 new MetricsTimeVaryingRate("fsWriteSize", registry);
256
257
258
259
260 public final MetricsTimeVaryingRate fsSyncLatency =
261 new MetricsTimeVaryingRate("fsSyncLatency", registry);
262
263
264
265
266
267 protected final PersistentMetricsTimeVaryingRate compactionTime =
268 new PersistentMetricsTimeVaryingRate("compactionTime", registry);
269
270 protected final PersistentMetricsTimeVaryingRate compactionSize =
271 new PersistentMetricsTimeVaryingRate("compactionSize", registry);
272
273
274
275
276 protected final PersistentMetricsTimeVaryingRate flushTime =
277 new PersistentMetricsTimeVaryingRate("flushTime", registry);
278
279 protected final PersistentMetricsTimeVaryingRate flushSize =
280 new PersistentMetricsTimeVaryingRate("flushSize", registry);
281
282 public final MetricsLongValue slowHLogAppendCount =
283 new MetricsLongValue("slowHLogAppendCount", registry);
284
285 public final MetricsTimeVaryingRate slowHLogAppendTime =
286 new MetricsTimeVaryingRate("slowHLogAppendTime", registry);
287
288 public final MetricsTimeVaryingLong regionSplitSuccessCount =
289 new MetricsTimeVaryingLong("regionSplitSuccessCount", registry);
290
291 public final MetricsTimeVaryingLong regionSplitFailureCount =
292 new MetricsTimeVaryingLong("regionSplitFailureCount", registry);
293
294
295
296
297 public final MetricsLongValue checksumFailuresCount =
298 new MetricsLongValue("checksumFailuresCount", registry);
299
300
301
302
303 public final MetricsHistogram updatesBlockedSeconds = new MetricsHistogram(
304 "updatesBlockedSeconds", registry);
305
306
307
308
309 public final MetricsHistogram updatesBlockedSecondsHighWater = new MetricsHistogram(
310 "updatesBlockedSecondsHighWater",registry);
311
312 public RegionServerMetrics() {
313 MetricsContext context = MetricsUtil.getContext("hbase");
314 metricsRecord = MetricsUtil.createRecord(context, "regionserver");
315 String name = Thread.currentThread().getName();
316 metricsRecord.setTag("RegionServer", name);
317 context.registerUpdater(this);
318
319 JvmMetrics.init("RegionServer", name);
320
321 HBaseInfo.init();
322
323
324 statistics = new RegionServerStatistics(this.registry, name);
325
326
327 try {
328 Object m = ContextFactory.getFactory().getAttribute("hbase.extendedperiod");
329 if (m instanceof String) {
330 this.extendedPeriod = Long.parseLong((String) m)*1000;
331 }
332 } catch (IOException ioe) {
333 LOG.info("Couldn't load ContextFactory for Metrics config info");
334 }
335
336 LOG.info("Initialized");
337 }
338
339 public void shutdown() {
340 if (statistics != null)
341 statistics.shutdown();
342 }
343
344
345
346
347
348
349 public void doUpdates(MetricsContext caller) {
350 synchronized (this) {
351 this.lastUpdate = System.currentTimeMillis();
352
353
354 if (this.extendedPeriod > 0 &&
355 this.lastUpdate - this.lastExtUpdate >= this.extendedPeriod) {
356 this.lastExtUpdate = this.lastUpdate;
357 this.compactionTime.resetMinMaxAvg();
358 this.compactionSize.resetMinMaxAvg();
359 this.flushTime.resetMinMaxAvg();
360 this.flushSize.resetMinMaxAvg();
361 this.resetAllMinMax();
362 }
363
364 this.stores.pushMetric(this.metricsRecord);
365 this.storefiles.pushMetric(this.metricsRecord);
366 this.storefileIndexSizeMB.pushMetric(this.metricsRecord);
367 this.rootIndexSizeKB.pushMetric(this.metricsRecord);
368 this.totalStaticIndexSizeKB.pushMetric(this.metricsRecord);
369 this.totalStaticBloomSizeKB.pushMetric(this.metricsRecord);
370 this.memstoreSizeMB.pushMetric(this.metricsRecord);
371 this.mbInMemoryWithoutWAL.pushMetric(this.metricsRecord);
372 this.numPutsWithoutWAL.pushMetric(this.metricsRecord);
373 this.readRequestsCount.pushMetric(this.metricsRecord);
374 this.writeRequestsCount.pushMetric(this.metricsRecord);
375 this.regions.pushMetric(this.metricsRecord);
376 this.requests.pushMetric(this.metricsRecord);
377 this.compactionQueueSize.pushMetric(this.metricsRecord);
378 this.flushQueueSize.pushMetric(this.metricsRecord);
379 this.blockCacheSize.pushMetric(this.metricsRecord);
380 this.blockCacheFree.pushMetric(this.metricsRecord);
381 this.blockCacheCount.pushMetric(this.metricsRecord);
382 this.blockCacheHitCount.pushMetric(this.metricsRecord);
383 this.blockCacheMissCount.pushMetric(this.metricsRecord);
384 this.blockCacheEvictedCount.pushMetric(this.metricsRecord);
385 this.blockCacheHitRatio.pushMetric(this.metricsRecord);
386 this.blockCacheHitCachingRatio.pushMetric(this.metricsRecord);
387 this.hdfsBlocksLocalityIndex.pushMetric(this.metricsRecord);
388 this.blockCacheHitRatioPastNPeriods.pushMetric(this.metricsRecord);
389 this.blockCacheHitCachingRatioPastNPeriods.pushMetric(this.metricsRecord);
390
391
392
393
394
395
396
397
398
399
400
401 addHLogMetric(HLog.getWriteTime(), this.fsWriteLatency);
402 addHLogMetric(HLog.getWriteSize(), this.fsWriteSize);
403 addHLogMetric(HLog.getSyncTime(), this.fsSyncLatency);
404 addHLogMetric(HLog.getSlowAppendTime(), this.slowHLogAppendTime);
405 this.slowHLogAppendCount.set(HLog.getSlowAppendCount());
406
407 int ops = HFile.getReadOps();
408 if (ops != 0) this.fsReadLatency.inc(ops, HFile.getReadTimeMs());
409
410 ops = HFile.getPreadOps();
411 if (ops != 0) this.fsPreadLatency.inc(ops, HFile.getPreadTimeMs());
412 this.checksumFailuresCount.set(HFile.getChecksumFailuresCount());
413
414
415
416
417
418
419
420
421 for(Long latency : HFile.getReadLatenciesNanos()) {
422 this.fsReadLatencyHistogram.update(latency);
423 }
424 for(Long latency : HFile.getPreadLatenciesNanos()) {
425 this.fsPreadLatencyHistogram.update(latency);
426 }
427 for(Long latency : HFile.getWriteLatenciesNanos()) {
428 this.fsWriteLatencyHistogram.update(latency);
429 }
430
431
432
433 this.fsPreadLatency.pushMetric(this.metricsRecord);
434 this.fsReadLatency.pushMetric(this.metricsRecord);
435 this.fsWriteLatency.pushMetric(this.metricsRecord);
436 this.fsWriteSize.pushMetric(this.metricsRecord);
437
438 this.fsReadLatencyHistogram.pushMetric(this.metricsRecord);
439 this.fsWriteLatencyHistogram.pushMetric(this.metricsRecord);
440 this.fsPreadLatencyHistogram.pushMetric(this.metricsRecord);
441
442 this.fsSyncLatency.pushMetric(this.metricsRecord);
443 this.compactionTime.pushMetric(this.metricsRecord);
444 this.compactionSize.pushMetric(this.metricsRecord);
445 this.flushTime.pushMetric(this.metricsRecord);
446 this.flushSize.pushMetric(this.metricsRecord);
447 this.slowHLogAppendCount.pushMetric(this.metricsRecord);
448 this.regionSplitSuccessCount.pushMetric(this.metricsRecord);
449 this.regionSplitFailureCount.pushMetric(this.metricsRecord);
450 this.checksumFailuresCount.pushMetric(this.metricsRecord);
451 this.updatesBlockedSeconds.pushMetric(this.metricsRecord);
452 this.updatesBlockedSecondsHighWater.pushMetric(this.metricsRecord);
453 }
454 this.metricsRecord.update();
455 }
456
457 private void addHLogMetric(HLog.Metric logMetric,
458 MetricsTimeVaryingRate hadoopMetric) {
459 if (logMetric.count > 0)
460 hadoopMetric.inc(logMetric.min);
461 if (logMetric.count > 1)
462 hadoopMetric.inc(logMetric.max);
463 if (logMetric.count > 2) {
464 int ops = logMetric.count - 2;
465 hadoopMetric.inc(ops, logMetric.total - logMetric.max - logMetric.min);
466 }
467 }
468
469 public void resetAllMinMax() {
470 this.atomicIncrementTime.resetMinMax();
471 this.fsReadLatency.resetMinMax();
472 this.fsWriteLatency.resetMinMax();
473 this.fsWriteSize.resetMinMax();
474 this.fsSyncLatency.resetMinMax();
475 this.slowHLogAppendTime.resetMinMax();
476 }
477
478
479
480
481 public float getRequests() {
482 return this.requests.getPreviousIntervalValue();
483 }
484
485
486
487
488
489 public synchronized void addCompaction(long time, long size) {
490 this.compactionTime.inc(time);
491 this.compactionSize.inc(size);
492 }
493
494
495
496
497 public synchronized void addFlush(final List<Pair<Long,Long>> flushes) {
498 for (Pair<Long,Long> f : flushes) {
499 this.flushTime.inc(f.getFirst());
500 this.flushSize.inc(f.getSecond());
501 }
502 }
503
504
505
506
507 public void incrementRequests(final int inc) {
508 this.requests.inc(inc);
509 }
510
511 public void incrementSplitSuccessCount() {
512 this.regionSplitSuccessCount.inc();
513 }
514
515 public void incrementSplitFailureCount() {
516 this.regionSplitFailureCount.inc();
517 }
518
519 @Override
520 public String toString() {
521 StringBuilder sb = new StringBuilder();
522 sb = Strings.appendKeyValue(sb, "requestsPerSecond", Integer
523 .valueOf((int) this.requests.getPreviousIntervalValue()));
524 sb = Strings.appendKeyValue(sb, "numberOfOnlineRegions",
525 Integer.valueOf(this.regions.get()));
526 sb = Strings.appendKeyValue(sb, "numberOfStores",
527 Integer.valueOf(this.stores.get()));
528 sb = Strings.appendKeyValue(sb, "numberOfStorefiles",
529 Integer.valueOf(this.storefiles.get()));
530 sb = Strings.appendKeyValue(sb, this.storefileIndexSizeMB.getName(),
531 Integer.valueOf(this.storefileIndexSizeMB.get()));
532 sb = Strings.appendKeyValue(sb, "rootIndexSizeKB",
533 Integer.valueOf(this.rootIndexSizeKB.get()));
534 sb = Strings.appendKeyValue(sb, "totalStaticIndexSizeKB",
535 Integer.valueOf(this.totalStaticIndexSizeKB.get()));
536 sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB",
537 Integer.valueOf(this.totalStaticBloomSizeKB.get()));
538 sb = Strings.appendKeyValue(sb, this.memstoreSizeMB.getName(),
539 Integer.valueOf(this.memstoreSizeMB.get()));
540 sb = Strings.appendKeyValue(sb, "mbInMemoryWithoutWAL",
541 Integer.valueOf(this.mbInMemoryWithoutWAL.get()));
542 sb = Strings.appendKeyValue(sb, "numberOfPutsWithoutWAL",
543 Long.valueOf(this.numPutsWithoutWAL.get()));
544 sb = Strings.appendKeyValue(sb, "readRequestsCount",
545 Long.valueOf(this.readRequestsCount.get()));
546 sb = Strings.appendKeyValue(sb, "writeRequestsCount",
547 Long.valueOf(this.writeRequestsCount.get()));
548 sb = Strings.appendKeyValue(sb, "compactionQueueSize",
549 Integer.valueOf(this.compactionQueueSize.get()));
550 sb = Strings.appendKeyValue(sb, "flushQueueSize",
551 Integer.valueOf(this.flushQueueSize.get()));
552
553
554 MemoryUsage memory =
555 ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
556 sb = Strings.appendKeyValue(sb, "usedHeapMB",
557 Long.valueOf(memory.getUsed()/MB));
558 sb = Strings.appendKeyValue(sb, "maxHeapMB",
559 Long.valueOf(memory.getMax()/MB));
560 sb = Strings.appendKeyValue(sb, this.blockCacheSize.getName()+"MB",
561 StringUtils.limitDecimalTo2((float)this.blockCacheSize.get()/MB));
562 sb = Strings.appendKeyValue(sb, this.blockCacheFree.getName()+"MB",
563 StringUtils.limitDecimalTo2((float)this.blockCacheFree.get()/MB));
564 sb = Strings.appendKeyValue(sb, this.blockCacheCount.getName(),
565 Long.valueOf(this.blockCacheCount.get()));
566 sb = Strings.appendKeyValue(sb, this.blockCacheHitCount.getName(),
567 Long.valueOf(this.blockCacheHitCount.get()));
568 sb = Strings.appendKeyValue(sb, this.blockCacheMissCount.getName(),
569 Long.valueOf(this.blockCacheMissCount.get()));
570 sb = Strings.appendKeyValue(sb, this.blockCacheEvictedCount.getName(),
571 Long.valueOf(this.blockCacheEvictedCount.get()));
572 sb = Strings.appendKeyValue(sb, this.blockCacheHitRatio.getName(),
573 Long.valueOf(this.blockCacheHitRatio.get())+"%");
574 sb = Strings.appendKeyValue(sb, this.blockCacheHitCachingRatio.getName(),
575 Long.valueOf(this.blockCacheHitCachingRatio.get())+"%");
576 sb = Strings.appendKeyValue(sb, this.hdfsBlocksLocalityIndex.getName(),
577 Long.valueOf(this.hdfsBlocksLocalityIndex.get()));
578 sb = Strings.appendKeyValue(sb, "slowHLogAppendCount",
579 Long.valueOf(this.slowHLogAppendCount.get()));
580 sb = appendHistogram(sb, this.fsReadLatencyHistogram);
581 sb = appendHistogram(sb, this.fsPreadLatencyHistogram);
582 sb = appendHistogram(sb, this.fsWriteLatencyHistogram);
583
584 return sb.toString();
585 }
586
587 private StringBuilder appendHistogram(StringBuilder sb,
588 MetricsHistogram histogram) {
589 sb = Strings.appendKeyValue(sb,
590 histogram.getName() + "Mean",
591 StringUtils.limitDecimalTo2(histogram.getMean()));
592 sb = Strings.appendKeyValue(sb,
593 histogram.getName() + "Count",
594 StringUtils.limitDecimalTo2(histogram.getCount()));
595 final Snapshot s = histogram.getSnapshot();
596 sb = Strings.appendKeyValue(sb,
597 histogram.getName() + "Median",
598 StringUtils.limitDecimalTo2(s.getMedian()));
599 sb = Strings.appendKeyValue(sb,
600 histogram.getName() + "75th",
601 StringUtils.limitDecimalTo2(s.get75thPercentile()));
602 sb = Strings.appendKeyValue(sb,
603 histogram.getName() + "95th",
604 StringUtils.limitDecimalTo2(s.get95thPercentile()));
605 sb = Strings.appendKeyValue(sb,
606 histogram.getName() + "99th",
607 StringUtils.limitDecimalTo2(s.get99thPercentile()));
608 sb = Strings.appendKeyValue(sb,
609 histogram.getName() + "999th",
610 StringUtils.limitDecimalTo2(s.get999thPercentile()));
611 return sb;
612 }
613 }