1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import com.google.common.annotations.VisibleForTesting;
21 import com.google.protobuf.Descriptors.MethodDescriptor;
22 import com.google.protobuf.Message;
23 import com.yammer.metrics.core.Counter;
24 import com.yammer.metrics.core.Histogram;
25 import com.yammer.metrics.core.MetricsRegistry;
26 import com.yammer.metrics.core.Timer;
27 import com.yammer.metrics.reporting.JmxReporter;
28 import com.yammer.metrics.util.RatioGauge;
29 import org.apache.hadoop.hbase.ServerName;
30 import org.apache.hadoop.hbase.classification.InterfaceAudience;
31 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
32 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
33 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
34 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
35 import org.apache.hadoop.hbase.util.Bytes;
36
37 import java.util.concurrent.ConcurrentHashMap;
38 import java.util.concurrent.ConcurrentSkipListMap;
39 import java.util.concurrent.ConcurrentMap;
40 import java.util.concurrent.ThreadPoolExecutor;
41 import java.util.concurrent.TimeUnit;
42
43
44
45
46
47
48
49
50
51
52 @InterfaceAudience.Private
53 public class MetricsConnection implements StatisticTrackable {
54
55
56 public static final String CLIENT_SIDE_METRICS_ENABLED_KEY = "hbase.client.metrics.enable";
57
58 private static final String DRTN_BASE = "rpcCallDurationMs_";
59 private static final String REQ_BASE = "rpcCallRequestSizeBytes_";
60 private static final String RESP_BASE = "rpcCallResponseSizeBytes_";
61 private static final String MEMLOAD_BASE = "memstoreLoad_";
62 private static final String HEAP_BASE = "heapOccupancy_";
63 private static final String CLIENT_SVC = ClientService.getDescriptor().getName();
64
65
66 public static class CallStats {
67 private long requestSizeBytes = 0;
68 private long responseSizeBytes = 0;
69 private long startTime = 0;
70 private long callTimeMs = 0;
71
72 public long getRequestSizeBytes() {
73 return requestSizeBytes;
74 }
75
76 public void setRequestSizeBytes(long requestSizeBytes) {
77 this.requestSizeBytes = requestSizeBytes;
78 }
79
80 public long getResponseSizeBytes() {
81 return responseSizeBytes;
82 }
83
84 public void setResponseSizeBytes(long responseSizeBytes) {
85 this.responseSizeBytes = responseSizeBytes;
86 }
87
88 public long getStartTime() {
89 return startTime;
90 }
91
92 public void setStartTime(long startTime) {
93 this.startTime = startTime;
94 }
95
96 public long getCallTimeMs() {
97 return callTimeMs;
98 }
99
100 public void setCallTimeMs(long callTimeMs) {
101 this.callTimeMs = callTimeMs;
102 }
103 }
104
105 @VisibleForTesting
106 protected final class CallTracker {
107 private final String name;
108 @VisibleForTesting final Timer callTimer;
109 @VisibleForTesting final Histogram reqHist;
110 @VisibleForTesting final Histogram respHist;
111
112 private CallTracker(MetricsRegistry registry, String name, String subName, String scope) {
113 StringBuilder sb = new StringBuilder(CLIENT_SVC).append("_").append(name);
114 if (subName != null) {
115 sb.append("(").append(subName).append(")");
116 }
117 this.name = sb.toString();
118 this.callTimer = registry.newTimer(MetricsConnection.class, DRTN_BASE + this.name, scope);
119 this.reqHist = registry.newHistogram(MetricsConnection.class, REQ_BASE + this.name, scope);
120 this.respHist = registry.newHistogram(MetricsConnection.class, RESP_BASE + this.name, scope);
121 }
122
123 private CallTracker(MetricsRegistry registry, String name, String scope) {
124 this(registry, name, null, scope);
125 }
126
127 public void updateRpc(CallStats stats) {
128 this.callTimer.update(stats.getCallTimeMs(), TimeUnit.MILLISECONDS);
129 this.reqHist.update(stats.getRequestSizeBytes());
130 this.respHist.update(stats.getResponseSizeBytes());
131 }
132
133 @Override
134 public String toString() {
135 return "CallTracker:" + name;
136 }
137 }
138
139 protected static class RegionStats {
140 final String name;
141 final Histogram memstoreLoadHist;
142 final Histogram heapOccupancyHist;
143
144 public RegionStats(MetricsRegistry registry, String name) {
145 this.name = name;
146 this.memstoreLoadHist = registry.newHistogram(MetricsConnection.class,
147 MEMLOAD_BASE + this.name);
148 this.heapOccupancyHist = registry.newHistogram(MetricsConnection.class,
149 HEAP_BASE + this.name);
150 }
151
152 public void update(ClientProtos.RegionLoadStats regionStatistics) {
153 this.memstoreLoadHist.update(regionStatistics.getMemstoreLoad());
154 this.heapOccupancyHist.update(regionStatistics.getHeapOccupancy());
155 }
156 }
157
158 @VisibleForTesting
159 protected static class RunnerStats {
160 final Counter normalRunners;
161 final Counter delayRunners;
162 final Histogram delayIntevalHist;
163
164 public RunnerStats(MetricsRegistry registry) {
165 this.normalRunners = registry.newCounter(MetricsConnection.class, "normalRunnersCount");
166 this.delayRunners = registry.newCounter(MetricsConnection.class, "delayRunnersCount");
167 this.delayIntevalHist = registry.newHistogram(MetricsConnection.class, "delayIntervalHist");
168 }
169
170 public void incrNormalRunners() {
171 this.normalRunners.inc();
172 }
173
174 public void incrDelayRunners() {
175 this.delayRunners.inc();
176 }
177
178 public void updateDelayInterval(long interval) {
179 this.delayIntevalHist.update(interval);
180 }
181 }
182
183 @VisibleForTesting
184 protected ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>> serverStats
185 = new ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>>();
186
187 public void updateServerStats(ServerName serverName, byte[] regionName,
188 Object r) {
189 if (!(r instanceof Result)) {
190 return;
191 }
192 Result result = (Result) r;
193 ClientProtos.RegionLoadStats stats = result.getStats();
194 if (stats == null) {
195 return;
196 }
197 updateRegionStats(serverName, regionName, stats);
198 }
199
200 @Override
201 public void updateRegionStats(ServerName serverName, byte[] regionName,
202 ClientProtos.RegionLoadStats stats) {
203 String name = serverName.getServerName() + "," + Bytes.toStringBinary(regionName);
204 ConcurrentMap<byte[], RegionStats> rsStats = null;
205 if (serverStats.containsKey(serverName)) {
206 rsStats = serverStats.get(serverName);
207 } else {
208 rsStats = serverStats.putIfAbsent(serverName,
209 new ConcurrentSkipListMap<byte[], RegionStats>(Bytes.BYTES_COMPARATOR));
210 if (rsStats == null) {
211 rsStats = serverStats.get(serverName);
212 }
213 }
214 RegionStats regionStats = null;
215 if (rsStats.containsKey(regionName)) {
216 regionStats = rsStats.get(regionName);
217 } else {
218 regionStats = rsStats.putIfAbsent(regionName, new RegionStats(this.registry, name));
219 if (regionStats == null) {
220 regionStats = rsStats.get(regionName);
221 }
222 }
223 regionStats.update(stats);
224 }
225
226
227
228 private static interface NewMetric<T> {
229 T newMetric(Class<?> clazz, String name, String scope);
230 }
231
232
233 private static final int CAPACITY = 50;
234
235 private static final float LOAD_FACTOR = 0.75f;
236
237
238
239
240 private static final int CONCURRENCY_LEVEL = 256;
241
242 private final MetricsRegistry registry;
243 private final JmxReporter reporter;
244 private final String scope;
245
246 private final NewMetric<Timer> timerFactory = new NewMetric<Timer>() {
247 @Override public Timer newMetric(Class<?> clazz, String name, String scope) {
248 return registry.newTimer(clazz, name, scope);
249 }
250 };
251
252 private final NewMetric<Histogram> histogramFactory = new NewMetric<Histogram>() {
253 @Override public Histogram newMetric(Class<?> clazz, String name, String scope) {
254 return registry.newHistogram(clazz, name, scope);
255 }
256 };
257
258
259
260 @VisibleForTesting protected final Counter metaCacheHits;
261 @VisibleForTesting protected final Counter metaCacheMisses;
262 @VisibleForTesting protected final CallTracker getTracker;
263 @VisibleForTesting protected final CallTracker scanTracker;
264 @VisibleForTesting protected final CallTracker appendTracker;
265 @VisibleForTesting protected final CallTracker deleteTracker;
266 @VisibleForTesting protected final CallTracker incrementTracker;
267 @VisibleForTesting protected final CallTracker putTracker;
268 @VisibleForTesting protected final CallTracker multiTracker;
269 @VisibleForTesting protected final RunnerStats runnerStats;
270
271
272
273
274
275
276 @VisibleForTesting protected final ConcurrentMap<String, Timer> rpcTimers =
277 new ConcurrentHashMap<String, Timer>(CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL);
278 @VisibleForTesting protected final ConcurrentMap<String, Histogram> rpcHistograms =
279 new ConcurrentHashMap<String, Histogram>(
280 CAPACITY * 2
281 LOAD_FACTOR, CONCURRENCY_LEVEL);
282
283 public MetricsConnection(final HConnectionManager.HConnectionImplementation conn) {
284 this.scope = conn.toString();
285 this.registry = new MetricsRegistry();
286 final ThreadPoolExecutor batchPool = (ThreadPoolExecutor) conn.getCurrentBatchPool();
287
288 this.registry.newGauge(this.getClass(), "executorPoolActiveThreads", scope,
289 new RatioGauge() {
290 @Override protected double getNumerator() {
291 return batchPool.getActiveCount();
292 }
293 @Override protected double getDenominator() {
294 return batchPool.getMaximumPoolSize();
295 }
296 });
297 this.metaCacheHits = registry.newCounter(this.getClass(), "metaCacheHits", scope);
298 this.metaCacheMisses = registry.newCounter(this.getClass(), "metaCacheMisses", scope);
299 this.getTracker = new CallTracker(this.registry, "Get", scope);
300 this.scanTracker = new CallTracker(this.registry, "Scan", scope);
301 this.appendTracker = new CallTracker(this.registry, "Mutate", "Append", scope);
302 this.deleteTracker = new CallTracker(this.registry, "Mutate", "Delete", scope);
303 this.incrementTracker = new CallTracker(this.registry, "Mutate", "Increment", scope);
304 this.putTracker = new CallTracker(this.registry, "Mutate", "Put", scope);
305 this.multiTracker = new CallTracker(this.registry, "Multi", scope);
306 this.runnerStats = new RunnerStats(this.registry);
307
308 this.reporter = new JmxReporter(this.registry);
309 this.reporter.start();
310 }
311
312 public void shutdown() {
313 this.reporter.shutdown();
314 this.registry.shutdown();
315 }
316
317
318 public static CallStats newCallStats() {
319
320 return new CallStats();
321 }
322
323
324 public void incrMetaCacheHit() {
325 metaCacheHits.inc();
326 }
327
328
329 public void incrMetaCacheMiss() {
330 metaCacheMisses.inc();
331 }
332
333
334 public void incrNormalRunners() {
335 this.runnerStats.incrNormalRunners();
336 }
337
338
339 public void incrDelayRunners() {
340 this.runnerStats.incrDelayRunners();
341 }
342
343
344 public void updateDelayInterval(long interval) {
345 this.runnerStats.updateDelayInterval(interval);
346 }
347
348
349
350
351 private <T> T getMetric(String key, ConcurrentMap<String, T> map, NewMetric<T> factory) {
352 T t = map.get(key);
353 if (t == null) {
354 t = factory.newMetric(this.getClass(), key, scope);
355 map.putIfAbsent(key, t);
356 }
357 return t;
358 }
359
360
361 private void updateRpcGeneric(MethodDescriptor method, CallStats stats) {
362 final String methodName = method.getService().getName() + "_" + method.getName();
363 getMetric(DRTN_BASE + methodName, rpcTimers, timerFactory)
364 .update(stats.getCallTimeMs(), TimeUnit.MILLISECONDS);
365 getMetric(REQ_BASE + methodName, rpcHistograms, histogramFactory)
366 .update(stats.getRequestSizeBytes());
367 getMetric(RESP_BASE + methodName, rpcHistograms, histogramFactory)
368 .update(stats.getResponseSizeBytes());
369 }
370
371
372 public void updateRpc(MethodDescriptor method, Message param, CallStats stats) {
373
374
375 if (method.getService() == ClientService.getDescriptor()) {
376 switch(method.getIndex()) {
377 case 0:
378 assert "Get".equals(method.getName());
379 getTracker.updateRpc(stats);
380 return;
381 case 1:
382 assert "Mutate".equals(method.getName());
383 final MutationType mutationType = ((MutateRequest) param).getMutation().getMutateType();
384 switch(mutationType) {
385 case APPEND:
386 appendTracker.updateRpc(stats);
387 return;
388 case DELETE:
389 deleteTracker.updateRpc(stats);
390 return;
391 case INCREMENT:
392 incrementTracker.updateRpc(stats);
393 return;
394 case PUT:
395 putTracker.updateRpc(stats);
396 return;
397 default:
398 throw new RuntimeException("Unrecognized mutation type " + mutationType);
399 }
400 case 2:
401 assert "Scan".equals(method.getName());
402 scanTracker.updateRpc(stats);
403 return;
404 case 3:
405 assert "BulkLoadHFile".equals(method.getName());
406
407 break;
408 case 4:
409 assert "ExecService".equals(method.getName());
410
411 break;
412 case 5:
413 assert "ExecRegionServerService".equals(method.getName());
414
415 break;
416 case 6:
417 assert "Multi".equals(method.getName());
418 multiTracker.updateRpc(stats);
419 return;
420 default:
421 throw new RuntimeException("Unrecognized ClientService RPC type " + method.getFullName());
422 }
423 }
424
425 updateRpcGeneric(method, stats);
426 }
427 }