1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.metrics;
21
22 import java.io.IOException;
23 import java.lang.management.ManagementFactory;
24 import java.lang.management.MemoryUsage;
25 import java.util.List;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.io.hfile.HFile;
30 import org.apache.hadoop.hbase.metrics.HBaseInfo;
31 import org.apache.hadoop.hbase.metrics.MetricsRate;
32 import org.apache.hadoop.hbase.metrics.PersistentMetricsTimeVaryingRate;
33 import org.apache.hadoop.hbase.metrics.histogram.MetricsHistogram;
34 import com.yammer.metrics.stats.Snapshot;
35 import org.apache.hadoop.hbase.regionserver.wal.HLog;
36 import org.apache.hadoop.hbase.util.Pair;
37 import org.apache.hadoop.hbase.util.Strings;
38 import org.apache.hadoop.metrics.ContextFactory;
39 import org.apache.hadoop.metrics.MetricsContext;
40 import org.apache.hadoop.metrics.MetricsRecord;
41 import org.apache.hadoop.metrics.MetricsUtil;
42 import org.apache.hadoop.metrics.Updater;
43 import org.apache.hadoop.metrics.jvm.JvmMetrics;
44 import org.apache.hadoop.metrics.util.MetricsIntValue;
45 import org.apache.hadoop.metrics.util.MetricsLongValue;
46 import org.apache.hadoop.metrics.util.MetricsRegistry;
47 import org.apache.hadoop.metrics.util.MetricsTimeVaryingRate;
48 import org.apache.hadoop.metrics.util.MetricsTimeVaryingLong;
49 import org.apache.hadoop.util.StringUtils;
50
51
52
53
54
55
56
57
58 public class RegionServerMetrics implements Updater {
59 @SuppressWarnings({"FieldCanBeLocal"})
60 private final Log LOG = LogFactory.getLog(this.getClass());
61 private final MetricsRecord metricsRecord;
62 private long lastUpdate = System.currentTimeMillis();
63 private long lastExtUpdate = System.currentTimeMillis();
64 private long extendedPeriod = 0;
65 private static final int MB = 1024*1024;
66 private MetricsRegistry registry = new MetricsRegistry();
67 private final RegionServerStatistics statistics;
68
69 public final MetricsTimeVaryingRate atomicIncrementTime =
70 new MetricsTimeVaryingRate("atomicIncrementTime", registry);
71
72
73
74
75 public final MetricsIntValue regions =
76 new MetricsIntValue("regions", registry);
77
78
79
80
81 public final MetricsLongValue blockCacheSize =
82 new MetricsLongValue("blockCacheSize", registry);
83
84
85
86
87 public final MetricsLongValue blockCacheFree =
88 new MetricsLongValue("blockCacheFree", registry);
89
90
91
92
93 public final MetricsLongValue blockCacheCount =
94 new MetricsLongValue("blockCacheCount", registry);
95
96
97
98
99 public final MetricsLongValue blockCacheHitCount =
100 new MetricsLongValue("blockCacheHitCount", registry);
101
102
103
104
105 public final MetricsLongValue blockCacheMissCount =
106 new MetricsLongValue("blockCacheMissCount", registry);
107
108
109
110
111 public final MetricsLongValue blockCacheEvictedCount =
112 new MetricsLongValue("blockCacheEvictedCount", registry);
113
114
115
116
117 public final MetricsIntValue blockCacheHitRatio =
118 new MetricsIntValue("blockCacheHitRatio", registry);
119
120
121
122
123
124 public final MetricsIntValue blockCacheHitCachingRatio =
125 new MetricsIntValue("blockCacheHitCachingRatio", registry);
126
127
128 public final MetricsIntValue blockCacheHitRatioPastNPeriods = new MetricsIntValue("blockCacheHitRatioPastNPeriods", registry);
129
130
131 public final MetricsIntValue blockCacheHitCachingRatioPastNPeriods = new MetricsIntValue("blockCacheHitCachingRatioPastNPeriods", registry);
132
133
134
135
136 public final MetricsRate requests = new MetricsRate("requests", registry);
137
138
139
140
141 public final MetricsIntValue stores = new MetricsIntValue("stores", registry);
142
143
144
145
146 public final MetricsIntValue storefiles =
147 new MetricsIntValue("storefiles", registry);
148
149
150
151
152 public final MetricsIntValue hlogFileCount =
153 new MetricsIntValue("hlogFileCount", registry);
154
155
156
157
158 public final MetricsLongValue hlogFileSizeMB =
159 new MetricsLongValue("hlogFileSizeMB", registry);
160
161
162
163
164 public final MetricsLongValue readRequestsCount =
165 new MetricsLongValue("readRequestsCount", registry);
166
167
168
169
170 public final MetricsLongValue writeRequestsCount =
171 new MetricsLongValue("writeRequestsCount", registry);
172
173
174
175 public final MetricsIntValue storefileIndexSizeMB =
176 new MetricsIntValue("storefileIndexSizeMB", registry);
177
178
179 public final MetricsIntValue rootIndexSizeKB =
180 new MetricsIntValue("rootIndexSizeKB", registry);
181
182
183 public final MetricsIntValue totalStaticIndexSizeKB =
184 new MetricsIntValue("totalStaticIndexSizeKB", registry);
185
186
187 public final MetricsIntValue totalStaticBloomSizeKB =
188 new MetricsIntValue("totalStaticBloomSizeKB", registry);
189
190
191
192
193 public final MetricsIntValue hdfsBlocksLocalityIndex =
194 new MetricsIntValue("hdfsBlocksLocalityIndex", registry);
195
196
197
198
199 public final MetricsIntValue memstoreSizeMB =
200 new MetricsIntValue("memstoreSizeMB", registry);
201
202
203
204
205 public final MetricsLongValue numPutsWithoutWAL =
206 new MetricsLongValue("numPutsWithoutWAL", registry);
207
208
209
210
211 public final MetricsIntValue mbInMemoryWithoutWAL =
212 new MetricsIntValue("mbInMemoryWithoutWAL", registry);
213
214
215
216
217 public final MetricsIntValue compactionQueueSize =
218 new MetricsIntValue("compactionQueueSize", registry);
219
220
221
222
223 public final MetricsIntValue flushQueueSize =
224 new MetricsIntValue("flushQueueSize", registry);
225
226
227
228
229 public final MetricsHistogram fsReadLatencyHistogram =
230 new MetricsHistogram("fsReadLatencyHistogram", registry);
231
232
233
234
235 public final MetricsHistogram fsPreadLatencyHistogram =
236 new MetricsHistogram("fsPreadLatencyHistogram", registry);
237
238
239
240
241 public final MetricsHistogram fsWriteLatencyHistogram =
242 new MetricsHistogram("fsWriteLatencyHistogram", registry);
243
244
245
246
247
248 public final MetricsTimeVaryingRate fsReadLatency =
249 new MetricsTimeVaryingRate("fsReadLatency", registry);
250
251
252
253
254 public final MetricsTimeVaryingRate fsPreadLatency =
255 new MetricsTimeVaryingRate("fsPreadLatency", registry);
256
257
258
259
260 public final MetricsTimeVaryingRate fsWriteLatency =
261 new MetricsTimeVaryingRate("fsWriteLatency", registry);
262
263
264
265
266 public final MetricsTimeVaryingRate fsWriteSize =
267 new MetricsTimeVaryingRate("fsWriteSize", registry);
268
269
270
271
272 public final MetricsTimeVaryingRate fsSyncLatency =
273 new MetricsTimeVaryingRate("fsSyncLatency", registry);
274
275
276
277
278
279 protected final PersistentMetricsTimeVaryingRate compactionTime =
280 new PersistentMetricsTimeVaryingRate("compactionTime", registry);
281
282 protected final PersistentMetricsTimeVaryingRate compactionSize =
283 new PersistentMetricsTimeVaryingRate("compactionSize", registry);
284
285
286
287
288 protected final PersistentMetricsTimeVaryingRate flushTime =
289 new PersistentMetricsTimeVaryingRate("flushTime", registry);
290
291 protected final PersistentMetricsTimeVaryingRate flushSize =
292 new PersistentMetricsTimeVaryingRate("flushSize", registry);
293
294 public final MetricsLongValue slowHLogAppendCount =
295 new MetricsLongValue("slowHLogAppendCount", registry);
296
297 public final MetricsTimeVaryingRate slowHLogAppendTime =
298 new MetricsTimeVaryingRate("slowHLogAppendTime", registry);
299
300 public final MetricsTimeVaryingLong regionSplitSuccessCount =
301 new MetricsTimeVaryingLong("regionSplitSuccessCount", registry);
302
303 public final MetricsTimeVaryingLong regionSplitFailureCount =
304 new MetricsTimeVaryingLong("regionSplitFailureCount", registry);
305
306
307
308
309 public final MetricsLongValue checksumFailuresCount =
310 new MetricsLongValue("checksumFailuresCount", registry);
311
312
313
314
315 public final MetricsLongValue updatesBlockedSeconds = new MetricsLongValue(
316 "updatesBlockedSeconds", registry);
317
318
319
320
321 public final MetricsLongValue updatesBlockedSecondsHighWater = new MetricsLongValue(
322 "updatesBlockedSecondsHighWater",registry);
323
324 public RegionServerMetrics() {
325 MetricsContext context = MetricsUtil.getContext("hbase");
326 metricsRecord = MetricsUtil.createRecord(context, "regionserver");
327 String name = Thread.currentThread().getName();
328 metricsRecord.setTag("RegionServer", name);
329 context.registerUpdater(this);
330
331 JvmMetrics.init("RegionServer", name);
332
333 HBaseInfo.init();
334
335
336 statistics = new RegionServerStatistics(this.registry, name);
337
338
339 try {
340 Object m = ContextFactory.getFactory().getAttribute("hbase.extendedperiod");
341 if (m instanceof String) {
342 this.extendedPeriod = Long.parseLong((String) m)*1000;
343 }
344 } catch (IOException ioe) {
345 LOG.info("Couldn't load ContextFactory for Metrics config info");
346 }
347
348 LOG.info("Initialized");
349 }
350
351 public void shutdown() {
352 if (statistics != null)
353 statistics.shutdown();
354 }
355
356
357
358
359
360
361 public void doUpdates(MetricsContext caller) {
362 synchronized (this) {
363 this.lastUpdate = System.currentTimeMillis();
364
365
366 if (this.extendedPeriod > 0 &&
367 this.lastUpdate - this.lastExtUpdate >= this.extendedPeriod) {
368 this.lastExtUpdate = this.lastUpdate;
369 this.compactionTime.resetMinMaxAvg();
370 this.compactionSize.resetMinMaxAvg();
371 this.flushTime.resetMinMaxAvg();
372 this.flushSize.resetMinMaxAvg();
373 this.resetAllMinMax();
374 }
375
376 this.stores.pushMetric(this.metricsRecord);
377 this.storefiles.pushMetric(this.metricsRecord);
378 this.hlogFileCount.pushMetric(this.metricsRecord);
379 this.hlogFileSizeMB.pushMetric(this.metricsRecord);
380 this.storefileIndexSizeMB.pushMetric(this.metricsRecord);
381 this.rootIndexSizeKB.pushMetric(this.metricsRecord);
382 this.totalStaticIndexSizeKB.pushMetric(this.metricsRecord);
383 this.totalStaticBloomSizeKB.pushMetric(this.metricsRecord);
384 this.memstoreSizeMB.pushMetric(this.metricsRecord);
385 this.mbInMemoryWithoutWAL.pushMetric(this.metricsRecord);
386 this.numPutsWithoutWAL.pushMetric(this.metricsRecord);
387 this.readRequestsCount.pushMetric(this.metricsRecord);
388 this.writeRequestsCount.pushMetric(this.metricsRecord);
389 this.regions.pushMetric(this.metricsRecord);
390 this.requests.pushMetric(this.metricsRecord);
391 this.compactionQueueSize.pushMetric(this.metricsRecord);
392 this.flushQueueSize.pushMetric(this.metricsRecord);
393 this.blockCacheSize.pushMetric(this.metricsRecord);
394 this.blockCacheFree.pushMetric(this.metricsRecord);
395 this.blockCacheCount.pushMetric(this.metricsRecord);
396 this.blockCacheHitCount.pushMetric(this.metricsRecord);
397 this.blockCacheMissCount.pushMetric(this.metricsRecord);
398 this.blockCacheEvictedCount.pushMetric(this.metricsRecord);
399 this.blockCacheHitRatio.pushMetric(this.metricsRecord);
400 this.blockCacheHitCachingRatio.pushMetric(this.metricsRecord);
401 this.hdfsBlocksLocalityIndex.pushMetric(this.metricsRecord);
402 this.blockCacheHitRatioPastNPeriods.pushMetric(this.metricsRecord);
403 this.blockCacheHitCachingRatioPastNPeriods.pushMetric(this.metricsRecord);
404
405
406
407
408
409
410
411
412
413
414
415 addHLogMetric(HLog.getWriteTime(), this.fsWriteLatency);
416 addHLogMetric(HLog.getWriteSize(), this.fsWriteSize);
417 addHLogMetric(HLog.getSyncTime(), this.fsSyncLatency);
418 addHLogMetric(HLog.getSlowAppendTime(), this.slowHLogAppendTime);
419 this.slowHLogAppendCount.set(HLog.getSlowAppendCount());
420
421 int ops = HFile.getReadOps();
422 if (ops != 0) this.fsReadLatency.inc(ops, HFile.getReadTimeMs());
423
424 ops = HFile.getPreadOps();
425 if (ops != 0) this.fsPreadLatency.inc(ops, HFile.getPreadTimeMs());
426 this.checksumFailuresCount.set(HFile.getChecksumFailuresCount());
427
428
429
430
431
432
433
434
435 for(Long latency : HFile.getReadLatenciesNanos()) {
436 this.fsReadLatencyHistogram.update(latency);
437 }
438 for(Long latency : HFile.getPreadLatenciesNanos()) {
439 this.fsPreadLatencyHistogram.update(latency);
440 }
441 for(Long latency : HFile.getWriteLatenciesNanos()) {
442 this.fsWriteLatencyHistogram.update(latency);
443 }
444
445
446
447 this.fsPreadLatency.pushMetric(this.metricsRecord);
448 this.fsReadLatency.pushMetric(this.metricsRecord);
449 this.fsWriteLatency.pushMetric(this.metricsRecord);
450 this.fsWriteSize.pushMetric(this.metricsRecord);
451
452 this.fsReadLatencyHistogram.pushMetric(this.metricsRecord);
453 this.fsWriteLatencyHistogram.pushMetric(this.metricsRecord);
454 this.fsPreadLatencyHistogram.pushMetric(this.metricsRecord);
455
456 this.fsSyncLatency.pushMetric(this.metricsRecord);
457 this.compactionTime.pushMetric(this.metricsRecord);
458 this.compactionSize.pushMetric(this.metricsRecord);
459 this.flushTime.pushMetric(this.metricsRecord);
460 this.flushSize.pushMetric(this.metricsRecord);
461 this.slowHLogAppendCount.pushMetric(this.metricsRecord);
462 this.regionSplitSuccessCount.pushMetric(this.metricsRecord);
463 this.regionSplitFailureCount.pushMetric(this.metricsRecord);
464 this.checksumFailuresCount.pushMetric(this.metricsRecord);
465 this.updatesBlockedSeconds.pushMetric(this.metricsRecord);
466 this.updatesBlockedSecondsHighWater.pushMetric(this.metricsRecord);
467 }
468 this.metricsRecord.update();
469 }
470
471 private void addHLogMetric(HLog.Metric logMetric,
472 MetricsTimeVaryingRate hadoopMetric) {
473 if (logMetric.count > 0)
474 hadoopMetric.inc(logMetric.min);
475 if (logMetric.count > 1)
476 hadoopMetric.inc(logMetric.max);
477 if (logMetric.count > 2) {
478 int ops = logMetric.count - 2;
479 hadoopMetric.inc(ops, logMetric.total - logMetric.max - logMetric.min);
480 }
481 }
482
483 public void resetAllMinMax() {
484 this.atomicIncrementTime.resetMinMax();
485 this.fsReadLatency.resetMinMax();
486 this.fsWriteLatency.resetMinMax();
487 this.fsWriteSize.resetMinMax();
488 this.fsSyncLatency.resetMinMax();
489 this.slowHLogAppendTime.resetMinMax();
490 }
491
492
493
494
495 public float getRequests() {
496 return this.requests.getPreviousIntervalValue();
497 }
498
499
500
501
502
503 public synchronized void addCompaction(long time, long size) {
504 this.compactionTime.inc(time);
505 this.compactionSize.inc(size);
506 }
507
508
509
510
511 public synchronized void addFlush(final List<Pair<Long,Long>> flushes) {
512 for (Pair<Long,Long> f : flushes) {
513 this.flushTime.inc(f.getFirst());
514 this.flushSize.inc(f.getSecond());
515 }
516 }
517
518
519
520
521 public void incrementRequests(final int inc) {
522 this.requests.inc(inc);
523 }
524
525 public void incrementSplitSuccessCount() {
526 this.regionSplitSuccessCount.inc();
527 }
528
529 public void incrementSplitFailureCount() {
530 this.regionSplitFailureCount.inc();
531 }
532
533 @Override
534 public String toString() {
535 StringBuilder sb = new StringBuilder();
536 sb = Strings.appendKeyValue(sb, "requestsPerSecond", Integer
537 .valueOf((int) this.requests.getPreviousIntervalValue()));
538 sb = Strings.appendKeyValue(sb, "numberOfOnlineRegions",
539 Integer.valueOf(this.regions.get()));
540 sb = Strings.appendKeyValue(sb, "numberOfStores",
541 Integer.valueOf(this.stores.get()));
542 sb = Strings.appendKeyValue(sb, this.hlogFileCount.getName(),
543 Integer.valueOf(this.hlogFileCount.get()));
544 sb = Strings.appendKeyValue(sb, this.hlogFileSizeMB.getName(),
545 Long.valueOf(this.hlogFileSizeMB.get()));
546 sb = Strings.appendKeyValue(sb, "numberOfStorefiles",
547 Integer.valueOf(this.storefiles.get()));
548 sb = Strings.appendKeyValue(sb, this.storefileIndexSizeMB.getName(),
549 Integer.valueOf(this.storefileIndexSizeMB.get()));
550 sb = Strings.appendKeyValue(sb, "rootIndexSizeKB",
551 Integer.valueOf(this.rootIndexSizeKB.get()));
552 sb = Strings.appendKeyValue(sb, "totalStaticIndexSizeKB",
553 Integer.valueOf(this.totalStaticIndexSizeKB.get()));
554 sb = Strings.appendKeyValue(sb, "totalStaticBloomSizeKB",
555 Integer.valueOf(this.totalStaticBloomSizeKB.get()));
556 sb = Strings.appendKeyValue(sb, this.memstoreSizeMB.getName(),
557 Integer.valueOf(this.memstoreSizeMB.get()));
558 sb = Strings.appendKeyValue(sb, "mbInMemoryWithoutWAL",
559 Integer.valueOf(this.mbInMemoryWithoutWAL.get()));
560 sb = Strings.appendKeyValue(sb, "numberOfPutsWithoutWAL",
561 Long.valueOf(this.numPutsWithoutWAL.get()));
562 sb = Strings.appendKeyValue(sb, "readRequestsCount",
563 Long.valueOf(this.readRequestsCount.get()));
564 sb = Strings.appendKeyValue(sb, "writeRequestsCount",
565 Long.valueOf(this.writeRequestsCount.get()));
566 sb = Strings.appendKeyValue(sb, "compactionQueueSize",
567 Integer.valueOf(this.compactionQueueSize.get()));
568 sb = Strings.appendKeyValue(sb, "flushQueueSize",
569 Integer.valueOf(this.flushQueueSize.get()));
570
571
572 MemoryUsage memory =
573 ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
574 sb = Strings.appendKeyValue(sb, "usedHeapMB",
575 Long.valueOf(memory.getUsed()/MB));
576 sb = Strings.appendKeyValue(sb, "maxHeapMB",
577 Long.valueOf(memory.getMax()/MB));
578 sb = Strings.appendKeyValue(sb, this.blockCacheSize.getName()+"MB",
579 StringUtils.limitDecimalTo2((float)this.blockCacheSize.get()/MB));
580 sb = Strings.appendKeyValue(sb, this.blockCacheFree.getName()+"MB",
581 StringUtils.limitDecimalTo2((float)this.blockCacheFree.get()/MB));
582 sb = Strings.appendKeyValue(sb, this.blockCacheCount.getName(),
583 Long.valueOf(this.blockCacheCount.get()));
584 sb = Strings.appendKeyValue(sb, this.blockCacheHitCount.getName(),
585 Long.valueOf(this.blockCacheHitCount.get()));
586 sb = Strings.appendKeyValue(sb, this.blockCacheMissCount.getName(),
587 Long.valueOf(this.blockCacheMissCount.get()));
588 sb = Strings.appendKeyValue(sb, this.blockCacheEvictedCount.getName(),
589 Long.valueOf(this.blockCacheEvictedCount.get()));
590 sb = Strings.appendKeyValue(sb, this.blockCacheHitRatio.getName(),
591 Long.valueOf(this.blockCacheHitRatio.get())+"%");
592 sb = Strings.appendKeyValue(sb, this.blockCacheHitCachingRatio.getName(),
593 Long.valueOf(this.blockCacheHitCachingRatio.get())+"%");
594 sb = Strings.appendKeyValue(sb, this.hdfsBlocksLocalityIndex.getName(),
595 Long.valueOf(this.hdfsBlocksLocalityIndex.get()));
596 sb = Strings.appendKeyValue(sb, "slowHLogAppendCount",
597 Long.valueOf(this.slowHLogAppendCount.get()));
598 sb = appendHistogram(sb, this.fsReadLatencyHistogram);
599 sb = appendHistogram(sb, this.fsPreadLatencyHistogram);
600 sb = appendHistogram(sb, this.fsWriteLatencyHistogram);
601
602 return sb.toString();
603 }
604
605 private StringBuilder appendHistogram(StringBuilder sb,
606 MetricsHistogram histogram) {
607 sb = Strings.appendKeyValue(sb,
608 histogram.getName() + "Mean",
609 StringUtils.limitDecimalTo2(histogram.getMean()));
610 sb = Strings.appendKeyValue(sb,
611 histogram.getName() + "Count",
612 StringUtils.limitDecimalTo2(histogram.getCount()));
613 final Snapshot s = histogram.getSnapshot();
614 sb = Strings.appendKeyValue(sb,
615 histogram.getName() + "Median",
616 StringUtils.limitDecimalTo2(s.getMedian()));
617 sb = Strings.appendKeyValue(sb,
618 histogram.getName() + "75th",
619 StringUtils.limitDecimalTo2(s.get75thPercentile()));
620 sb = Strings.appendKeyValue(sb,
621 histogram.getName() + "95th",
622 StringUtils.limitDecimalTo2(s.get95thPercentile()));
623 sb = Strings.appendKeyValue(sb,
624 histogram.getName() + "99th",
625 StringUtils.limitDecimalTo2(s.get99thPercentile()));
626 sb = Strings.appendKeyValue(sb,
627 histogram.getName() + "999th",
628 StringUtils.limitDecimalTo2(s.get999thPercentile()));
629 return sb;
630 }
631 }