1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import com.google.common.annotations.VisibleForTesting;
21 import com.google.protobuf.Descriptors.MethodDescriptor;
22 import com.google.protobuf.Message;
23 import com.yammer.metrics.core.Counter;
24 import com.yammer.metrics.core.Histogram;
25 import com.yammer.metrics.core.MetricsRegistry;
26 import com.yammer.metrics.core.Timer;
27 import com.yammer.metrics.reporting.JmxReporter;
28 import com.yammer.metrics.util.RatioGauge;
29 import org.apache.hadoop.hbase.ServerName;
30 import org.apache.hadoop.hbase.classification.InterfaceAudience;
31 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
32 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
33 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
34 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
35 import org.apache.hadoop.hbase.util.Bytes;
36
37 import java.util.concurrent.ConcurrentHashMap;
38 import java.util.concurrent.ConcurrentSkipListMap;
39 import java.util.concurrent.ConcurrentMap;
40 import java.util.concurrent.ThreadPoolExecutor;
41 import java.util.concurrent.TimeUnit;
42
43
44
45
46
47
48
49
50
51
52 @InterfaceAudience.Private
53 public class MetricsConnection {
54
55
56 public static final String CLIENT_SIDE_METRICS_ENABLED_KEY = "hbase.client.metrics.enable";
57
58 private static final String DRTN_BASE = "rpcCallDurationMs_";
59 private static final String REQ_BASE = "rpcCallRequestSizeBytes_";
60 private static final String RESP_BASE = "rpcCallResponseSizeBytes_";
61 private static final String MEMLOAD_BASE = "memstoreLoad_";
62 private static final String HEAP_BASE = "heapOccupancy_";
63 private static final String CLIENT_SVC = ClientService.getDescriptor().getName();
64
65
66 public static class CallStats {
67 private long requestSizeBytes = 0;
68 private long responseSizeBytes = 0;
69 private long startTime = 0;
70 private long callTimeMs = 0;
71
72 public long getRequestSizeBytes() {
73 return requestSizeBytes;
74 }
75
76 public void setRequestSizeBytes(long requestSizeBytes) {
77 this.requestSizeBytes = requestSizeBytes;
78 }
79
80 public long getResponseSizeBytes() {
81 return responseSizeBytes;
82 }
83
84 public void setResponseSizeBytes(long responseSizeBytes) {
85 this.responseSizeBytes = responseSizeBytes;
86 }
87
88 public long getStartTime() {
89 return startTime;
90 }
91
92 public void setStartTime(long startTime) {
93 this.startTime = startTime;
94 }
95
96 public long getCallTimeMs() {
97 return callTimeMs;
98 }
99
100 public void setCallTimeMs(long callTimeMs) {
101 this.callTimeMs = callTimeMs;
102 }
103 }
104
105 @VisibleForTesting
106 protected final class CallTracker {
107 private final String name;
108 @VisibleForTesting final Timer callTimer;
109 @VisibleForTesting final Histogram reqHist;
110 @VisibleForTesting final Histogram respHist;
111
112 private CallTracker(MetricsRegistry registry, String name, String subName, String scope) {
113 StringBuilder sb = new StringBuilder(CLIENT_SVC).append("_").append(name);
114 if (subName != null) {
115 sb.append("(").append(subName).append(")");
116 }
117 this.name = sb.toString();
118 this.callTimer = registry.newTimer(MetricsConnection.class, DRTN_BASE + this.name, scope);
119 this.reqHist = registry.newHistogram(MetricsConnection.class, REQ_BASE + this.name, scope);
120 this.respHist = registry.newHistogram(MetricsConnection.class, RESP_BASE + this.name, scope);
121 }
122
123 private CallTracker(MetricsRegistry registry, String name, String scope) {
124 this(registry, name, null, scope);
125 }
126
127 public void updateRpc(CallStats stats) {
128 this.callTimer.update(stats.getCallTimeMs(), TimeUnit.MILLISECONDS);
129 this.reqHist.update(stats.getRequestSizeBytes());
130 this.respHist.update(stats.getResponseSizeBytes());
131 }
132
133 @Override
134 public String toString() {
135 return "CallTracker:" + name;
136 }
137 }
138
139 protected static class RegionStats {
140 final String name;
141 final Histogram memstoreLoadHist;
142 final Histogram heapOccupancyHist;
143
144 public RegionStats(MetricsRegistry registry, String name) {
145 this.name = name;
146 this.memstoreLoadHist = registry.newHistogram(MetricsConnection.class,
147 MEMLOAD_BASE + this.name);
148 this.heapOccupancyHist = registry.newHistogram(MetricsConnection.class,
149 HEAP_BASE + this.name);
150 }
151
152 public void update(ClientProtos.RegionLoadStats regionStatistics) {
153 this.memstoreLoadHist.update(regionStatistics.getMemstoreLoad());
154 this.heapOccupancyHist.update(regionStatistics.getHeapOccupancy());
155 }
156 }
157
158 @VisibleForTesting
159 protected static class RunnerStats {
160 final Counter normalRunners;
161 final Counter delayRunners;
162 final Histogram delayIntevalHist;
163
164 public RunnerStats(MetricsRegistry registry) {
165 this.normalRunners = registry.newCounter(MetricsConnection.class, "normalRunnersCount");
166 this.delayRunners = registry.newCounter(MetricsConnection.class, "delayRunnersCount");
167 this.delayIntevalHist = registry.newHistogram(MetricsConnection.class, "delayIntervalHist");
168 }
169
170 public void incrNormalRunners() {
171 this.normalRunners.inc();
172 }
173
174 public void incrDelayRunners() {
175 this.delayRunners.inc();
176 }
177
178 public void updateDelayInterval(long interval) {
179 this.delayIntevalHist.update(interval);
180 }
181 }
182
183 @VisibleForTesting
184 protected ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>> serverStats
185 = new ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>>();
186
187 public void updateServerStats(ServerName serverName, byte[] regionName,
188 Object r) {
189 if (!(r instanceof Result)) {
190 return;
191 }
192 Result result = (Result) r;
193 ClientProtos.RegionLoadStats stats = result.getStats();
194 if(stats == null){
195 return;
196 }
197 String name = serverName.getServerName() + "," + Bytes.toStringBinary(regionName);
198 ConcurrentMap<byte[], RegionStats> rsStats = null;
199 if (serverStats.containsKey(serverName)) {
200 rsStats = serverStats.get(serverName);
201 } else {
202 rsStats = serverStats.putIfAbsent(serverName,
203 new ConcurrentSkipListMap<byte[], RegionStats>(Bytes.BYTES_COMPARATOR));
204 if (rsStats == null) {
205 rsStats = serverStats.get(serverName);
206 }
207 }
208 RegionStats regionStats = null;
209 if (rsStats.containsKey(regionName)) {
210 regionStats = rsStats.get(regionName);
211 } else {
212 regionStats = rsStats.putIfAbsent(regionName, new RegionStats(this.registry, name));
213 if (regionStats == null) {
214 regionStats = rsStats.get(regionName);
215 }
216 }
217 regionStats.update(stats);
218 }
219
220
221
222 private static interface NewMetric<T> {
223 T newMetric(Class<?> clazz, String name, String scope);
224 }
225
226
227 private static final int CAPACITY = 50;
228
229 private static final float LOAD_FACTOR = 0.75f;
230
231
232
233
234 private static final int CONCURRENCY_LEVEL = 256;
235
236 private final MetricsRegistry registry;
237 private final JmxReporter reporter;
238 private final String scope;
239
240 private final NewMetric<Timer> timerFactory = new NewMetric<Timer>() {
241 @Override public Timer newMetric(Class<?> clazz, String name, String scope) {
242 return registry.newTimer(clazz, name, scope);
243 }
244 };
245
246 private final NewMetric<Histogram> histogramFactory = new NewMetric<Histogram>() {
247 @Override public Histogram newMetric(Class<?> clazz, String name, String scope) {
248 return registry.newHistogram(clazz, name, scope);
249 }
250 };
251
252
253
254 @VisibleForTesting protected final Counter metaCacheHits;
255 @VisibleForTesting protected final Counter metaCacheMisses;
256 @VisibleForTesting protected final CallTracker getTracker;
257 @VisibleForTesting protected final CallTracker scanTracker;
258 @VisibleForTesting protected final CallTracker appendTracker;
259 @VisibleForTesting protected final CallTracker deleteTracker;
260 @VisibleForTesting protected final CallTracker incrementTracker;
261 @VisibleForTesting protected final CallTracker putTracker;
262 @VisibleForTesting protected final CallTracker multiTracker;
263 @VisibleForTesting protected final RunnerStats runnerStats;
264
265
266
267
268
269
270 @VisibleForTesting protected final ConcurrentMap<String, Timer> rpcTimers =
271 new ConcurrentHashMap<String, Timer>(CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL);
272 @VisibleForTesting protected final ConcurrentMap<String, Histogram> rpcHistograms =
273 new ConcurrentHashMap<String, Histogram>(
274 CAPACITY * 2
275 LOAD_FACTOR, CONCURRENCY_LEVEL);
276
277 public MetricsConnection(final HConnectionManager.HConnectionImplementation conn) {
278 this.scope = conn.toString();
279 this.registry = new MetricsRegistry();
280 final ThreadPoolExecutor batchPool = (ThreadPoolExecutor) conn.getCurrentBatchPool();
281
282 this.registry.newGauge(this.getClass(), "executorPoolActiveThreads", scope,
283 new RatioGauge() {
284 @Override protected double getNumerator() {
285 return batchPool.getActiveCount();
286 }
287 @Override protected double getDenominator() {
288 return batchPool.getMaximumPoolSize();
289 }
290 });
291 this.metaCacheHits = registry.newCounter(this.getClass(), "metaCacheHits", scope);
292 this.metaCacheMisses = registry.newCounter(this.getClass(), "metaCacheMisses", scope);
293 this.getTracker = new CallTracker(this.registry, "Get", scope);
294 this.scanTracker = new CallTracker(this.registry, "Scan", scope);
295 this.appendTracker = new CallTracker(this.registry, "Mutate", "Append", scope);
296 this.deleteTracker = new CallTracker(this.registry, "Mutate", "Delete", scope);
297 this.incrementTracker = new CallTracker(this.registry, "Mutate", "Increment", scope);
298 this.putTracker = new CallTracker(this.registry, "Mutate", "Put", scope);
299 this.multiTracker = new CallTracker(this.registry, "Multi", scope);
300 this.runnerStats = new RunnerStats(this.registry);
301
302 this.reporter = new JmxReporter(this.registry);
303 this.reporter.start();
304 }
305
306 public void shutdown() {
307 this.reporter.shutdown();
308 this.registry.shutdown();
309 }
310
311
312 public static CallStats newCallStats() {
313
314 return new CallStats();
315 }
316
317
318 public void incrMetaCacheHit() {
319 metaCacheHits.inc();
320 }
321
322
323 public void incrMetaCacheMiss() {
324 metaCacheMisses.inc();
325 }
326
327
328 public void incrNormalRunners() {
329 this.runnerStats.incrNormalRunners();
330 }
331
332
333 public void incrDelayRunners() {
334 this.runnerStats.incrDelayRunners();
335 }
336
337
338 public void updateDelayInterval(long interval) {
339 this.runnerStats.updateDelayInterval(interval);
340 }
341
342
343
344
345 private <T> T getMetric(String key, ConcurrentMap<String, T> map, NewMetric<T> factory) {
346 T t = map.get(key);
347 if (t == null) {
348 t = factory.newMetric(this.getClass(), key, scope);
349 map.putIfAbsent(key, t);
350 }
351 return t;
352 }
353
354
355 private void updateRpcGeneric(MethodDescriptor method, CallStats stats) {
356 final String methodName = method.getService().getName() + "_" + method.getName();
357 getMetric(DRTN_BASE + methodName, rpcTimers, timerFactory)
358 .update(stats.getCallTimeMs(), TimeUnit.MILLISECONDS);
359 getMetric(REQ_BASE + methodName, rpcHistograms, histogramFactory)
360 .update(stats.getRequestSizeBytes());
361 getMetric(RESP_BASE + methodName, rpcHistograms, histogramFactory)
362 .update(stats.getResponseSizeBytes());
363 }
364
365
366 public void updateRpc(MethodDescriptor method, Message param, CallStats stats) {
367
368
369 if (method.getService() == ClientService.getDescriptor()) {
370 switch(method.getIndex()) {
371 case 0:
372 assert "Get".equals(method.getName());
373 getTracker.updateRpc(stats);
374 return;
375 case 1:
376 assert "Mutate".equals(method.getName());
377 final MutationType mutationType = ((MutateRequest) param).getMutation().getMutateType();
378 switch(mutationType) {
379 case APPEND:
380 appendTracker.updateRpc(stats);
381 return;
382 case DELETE:
383 deleteTracker.updateRpc(stats);
384 return;
385 case INCREMENT:
386 incrementTracker.updateRpc(stats);
387 return;
388 case PUT:
389 putTracker.updateRpc(stats);
390 return;
391 default:
392 throw new RuntimeException("Unrecognized mutation type " + mutationType);
393 }
394 case 2:
395 assert "Scan".equals(method.getName());
396 scanTracker.updateRpc(stats);
397 return;
398 case 3:
399 assert "BulkLoadHFile".equals(method.getName());
400
401 break;
402 case 4:
403 assert "ExecService".equals(method.getName());
404
405 break;
406 case 5:
407 assert "ExecRegionServerService".equals(method.getName());
408
409 break;
410 case 6:
411 assert "Multi".equals(method.getName());
412 multiTracker.updateRpc(stats);
413 return;
414 default:
415 throw new RuntimeException("Unrecognized ClientService RPC type " + method.getFullName());
416 }
417 }
418
419 updateRpcGeneric(method, stats);
420 }
421 }