View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import com.google.common.annotations.VisibleForTesting;
21  import com.google.protobuf.Descriptors.MethodDescriptor;
22  import com.google.protobuf.Message;
23  import com.yammer.metrics.core.Counter;
24  import com.yammer.metrics.core.Histogram;
25  import com.yammer.metrics.core.MetricsRegistry;
26  import com.yammer.metrics.core.Timer;
27  import com.yammer.metrics.reporting.JmxReporter;
28  import com.yammer.metrics.util.RatioGauge;
29  import org.apache.hadoop.hbase.ServerName;
30  import org.apache.hadoop.hbase.classification.InterfaceAudience;
31  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
32  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
33  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutateRequest;
34  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
35  import org.apache.hadoop.hbase.util.Bytes;
36  
37  import java.util.concurrent.ConcurrentHashMap;
38  import java.util.concurrent.ConcurrentSkipListMap;
39  import java.util.concurrent.ConcurrentMap;
40  import java.util.concurrent.ThreadPoolExecutor;
41  import java.util.concurrent.TimeUnit;
42  
43  /**
44   * This class is for maintaining the various connection statistics and publishing them through
45   * the metrics interfaces.
46   *
47   * This class manages its own {@link MetricsRegistry} and {@link JmxReporter} so as to not
48   * conflict with other uses of Yammer Metrics within the client application. Instantiating
49   * this class implicitly creates and "starts" instances of these classes; be sure to call
50   * {@link #shutdown()} to terminate the thread pools they allocate.
51   */
52  @InterfaceAudience.Private
53  public class MetricsConnection implements StatisticTrackable {
54  
55    /** Set this key to {@code true} to enable metrics collection of client requests. */
56    public static final String CLIENT_SIDE_METRICS_ENABLED_KEY = "hbase.client.metrics.enable";
57  
58    private static final String DRTN_BASE = "rpcCallDurationMs_";
59    private static final String REQ_BASE = "rpcCallRequestSizeBytes_";
60    private static final String RESP_BASE = "rpcCallResponseSizeBytes_";
61    private static final String MEMLOAD_BASE = "memstoreLoad_";
62    private static final String HEAP_BASE = "heapOccupancy_";
63    private static final String CLIENT_SVC = ClientService.getDescriptor().getName();
64  
65    /** A container class for collecting details about the RPC call as it percolates. */
66    public static class CallStats {
67      private long requestSizeBytes = 0;
68      private long responseSizeBytes = 0;
69      private long startTime = 0;
70      private long callTimeMs = 0;
71  
72      public long getRequestSizeBytes() {
73        return requestSizeBytes;
74      }
75  
76      public void setRequestSizeBytes(long requestSizeBytes) {
77        this.requestSizeBytes = requestSizeBytes;
78      }
79  
80      public long getResponseSizeBytes() {
81        return responseSizeBytes;
82      }
83  
84      public void setResponseSizeBytes(long responseSizeBytes) {
85        this.responseSizeBytes = responseSizeBytes;
86      }
87  
88      public long getStartTime() {
89        return startTime;
90      }
91  
92      public void setStartTime(long startTime) {
93        this.startTime = startTime;
94      }
95  
96      public long getCallTimeMs() {
97        return callTimeMs;
98      }
99  
100     public void setCallTimeMs(long callTimeMs) {
101       this.callTimeMs = callTimeMs;
102     }
103   }
104 
105   @VisibleForTesting
106   protected final class CallTracker {
107     private final String name;
108     @VisibleForTesting final Timer callTimer;
109     @VisibleForTesting final Histogram reqHist;
110     @VisibleForTesting final Histogram respHist;
111 
112     private CallTracker(MetricsRegistry registry, String name, String subName, String scope) {
113       StringBuilder sb = new StringBuilder(CLIENT_SVC).append("_").append(name);
114       if (subName != null) {
115         sb.append("(").append(subName).append(")");
116       }
117       this.name = sb.toString();
118       this.callTimer = registry.newTimer(MetricsConnection.class, DRTN_BASE + this.name, scope);
119       this.reqHist = registry.newHistogram(MetricsConnection.class, REQ_BASE + this.name, scope);
120       this.respHist = registry.newHistogram(MetricsConnection.class, RESP_BASE + this.name, scope);
121     }
122 
123     private CallTracker(MetricsRegistry registry, String name, String scope) {
124       this(registry, name, null, scope);
125     }
126 
127     public void updateRpc(CallStats stats) {
128       this.callTimer.update(stats.getCallTimeMs(), TimeUnit.MILLISECONDS);
129       this.reqHist.update(stats.getRequestSizeBytes());
130       this.respHist.update(stats.getResponseSizeBytes());
131     }
132 
133     @Override
134     public String toString() {
135       return "CallTracker:" + name;
136     }
137   }
138 
139   protected static class RegionStats {
140     final String name;
141     final Histogram memstoreLoadHist;
142     final Histogram heapOccupancyHist;
143 
144     public RegionStats(MetricsRegistry registry, String name) {
145       this.name = name;
146       this.memstoreLoadHist = registry.newHistogram(MetricsConnection.class,
147           MEMLOAD_BASE + this.name);
148       this.heapOccupancyHist = registry.newHistogram(MetricsConnection.class,
149           HEAP_BASE + this.name);
150     }
151 
152     public void update(ClientProtos.RegionLoadStats regionStatistics) {
153       this.memstoreLoadHist.update(regionStatistics.getMemstoreLoad());
154       this.heapOccupancyHist.update(regionStatistics.getHeapOccupancy());
155     }
156   }
157 
158   @VisibleForTesting
159   protected static class RunnerStats {
160     final Counter normalRunners;
161     final Counter delayRunners;
162     final Histogram delayIntevalHist;
163 
164     public RunnerStats(MetricsRegistry registry) {
165       this.normalRunners = registry.newCounter(MetricsConnection.class, "normalRunnersCount");
166       this.delayRunners = registry.newCounter(MetricsConnection.class, "delayRunnersCount");
167       this.delayIntevalHist = registry.newHistogram(MetricsConnection.class, "delayIntervalHist");
168     }
169 
170     public void incrNormalRunners() {
171       this.normalRunners.inc();
172     }
173 
174     public void incrDelayRunners() {
175       this.delayRunners.inc();
176     }
177 
178     public void updateDelayInterval(long interval) {
179       this.delayIntevalHist.update(interval);
180     }
181   }
182 
183   @VisibleForTesting
184   protected ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>> serverStats
185           = new ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>>();
186 
187   public void updateServerStats(ServerName serverName, byte[] regionName,
188                                 Object r) {
189     if (!(r instanceof Result)) {
190       return;
191     }
192     Result result = (Result) r;
193     ClientProtos.RegionLoadStats stats = result.getStats();
194     if (stats == null) {
195       return;
196     }
197     updateRegionStats(serverName, regionName, stats);
198   }
199 
200   @Override
201   public void updateRegionStats(ServerName serverName, byte[] regionName,
202     ClientProtos.RegionLoadStats stats) {
203     String name = serverName.getServerName() + "," + Bytes.toStringBinary(regionName);
204     ConcurrentMap<byte[], RegionStats> rsStats = null;
205     if (serverStats.containsKey(serverName)) {
206       rsStats = serverStats.get(serverName);
207     } else {
208       rsStats = serverStats.putIfAbsent(serverName,
209           new ConcurrentSkipListMap<byte[], RegionStats>(Bytes.BYTES_COMPARATOR));
210       if (rsStats == null) {
211         rsStats = serverStats.get(serverName);
212       }
213     }
214     RegionStats regionStats = null;
215     if (rsStats.containsKey(regionName)) {
216       regionStats = rsStats.get(regionName);
217     } else {
218       regionStats = rsStats.putIfAbsent(regionName, new RegionStats(this.registry, name));
219       if (regionStats == null) {
220         regionStats = rsStats.get(regionName);
221       }
222     }
223     regionStats.update(stats);
224   }
225 
226 
227   /** A lambda for dispatching to the appropriate metric factory method */
228   private static interface NewMetric<T> {
229     T newMetric(Class<?> clazz, String name, String scope);
230   }
231 
232   /** Anticipated number of metric entries */
233   private static final int CAPACITY = 50;
234   /** Default load factor from {@link java.util.HashMap#DEFAULT_LOAD_FACTOR} */
235   private static final float LOAD_FACTOR = 0.75f;
236   /**
237    * Anticipated number of concurrent accessor threads, from
238    * {@link HConnectionManager.HConnectionImplementation#getBatchPool()}
239    */
240   private static final int CONCURRENCY_LEVEL = 256;
241 
242   private final MetricsRegistry registry;
243   private final JmxReporter reporter;
244   private final String scope;
245 
246   private final NewMetric<Timer> timerFactory = new NewMetric<Timer>() {
247     @Override public Timer newMetric(Class<?> clazz, String name, String scope) {
248       return registry.newTimer(clazz, name, scope);
249     }
250   };
251 
252   private final NewMetric<Histogram> histogramFactory = new NewMetric<Histogram>() {
253     @Override public Histogram newMetric(Class<?> clazz, String name, String scope) {
254       return registry.newHistogram(clazz, name, scope);
255     }
256   };
257 
258   // static metrics
259 
260   @VisibleForTesting protected final Counter metaCacheHits;
261   @VisibleForTesting protected final Counter metaCacheMisses;
262   @VisibleForTesting protected final CallTracker getTracker;
263   @VisibleForTesting protected final CallTracker scanTracker;
264   @VisibleForTesting protected final CallTracker appendTracker;
265   @VisibleForTesting protected final CallTracker deleteTracker;
266   @VisibleForTesting protected final CallTracker incrementTracker;
267   @VisibleForTesting protected final CallTracker putTracker;
268   @VisibleForTesting protected final CallTracker multiTracker;
269   @VisibleForTesting protected final RunnerStats runnerStats;
270 
271   // dynamic metrics
272 
273   // These maps are used to cache references to the metric instances that are managed by the
274   // registry. I don't think their use perfectly removes redundant allocations, but it's
275   // a big improvement over calling registry.newMetric each time.
276   @VisibleForTesting protected final ConcurrentMap<String, Timer> rpcTimers =
277       new ConcurrentHashMap<String, Timer>(CAPACITY, LOAD_FACTOR, CONCURRENCY_LEVEL);
278   @VisibleForTesting protected final ConcurrentMap<String, Histogram> rpcHistograms =
279       new ConcurrentHashMap<String, Histogram>(
280           CAPACITY * 2 /* tracking both request and response sizes */,
281           LOAD_FACTOR, CONCURRENCY_LEVEL);
282 
283   public MetricsConnection(final HConnectionManager.HConnectionImplementation conn) {
284     this.scope = conn.toString();
285     this.registry = new MetricsRegistry();
286     final ThreadPoolExecutor batchPool = (ThreadPoolExecutor) conn.getCurrentBatchPool();
287 
288     this.registry.newGauge(this.getClass(), "executorPoolActiveThreads", scope,
289         new RatioGauge() {
290           @Override protected double getNumerator() {
291             return batchPool.getActiveCount();
292           }
293           @Override protected double getDenominator() {
294             return batchPool.getMaximumPoolSize();
295           }
296         });
297     this.metaCacheHits = registry.newCounter(this.getClass(), "metaCacheHits", scope);
298     this.metaCacheMisses = registry.newCounter(this.getClass(), "metaCacheMisses", scope);
299     this.getTracker = new CallTracker(this.registry, "Get", scope);
300     this.scanTracker = new CallTracker(this.registry, "Scan", scope);
301     this.appendTracker = new CallTracker(this.registry, "Mutate", "Append", scope);
302     this.deleteTracker = new CallTracker(this.registry, "Mutate", "Delete", scope);
303     this.incrementTracker = new CallTracker(this.registry, "Mutate", "Increment", scope);
304     this.putTracker = new CallTracker(this.registry, "Mutate", "Put", scope);
305     this.multiTracker = new CallTracker(this.registry, "Multi", scope);
306     this.runnerStats = new RunnerStats(this.registry);
307 
308     this.reporter = new JmxReporter(this.registry);
309     this.reporter.start();
310   }
311 
312   public void shutdown() {
313     this.reporter.shutdown();
314     this.registry.shutdown();
315   }
316 
317   /** Produce an instance of {@link CallStats} for clients to attach to RPCs. */
318   public static CallStats newCallStats() {
319     // TODO: instance pool to reduce GC?
320     return new CallStats();
321   }
322 
323   /** Increment the number of meta cache hits. */
324   public void incrMetaCacheHit() {
325     metaCacheHits.inc();
326   }
327 
328   /** Increment the number of meta cache misses. */
329   public void incrMetaCacheMiss() {
330     metaCacheMisses.inc();
331   }
332 
333   /** Increment the number of normal runner counts. */
334   public void incrNormalRunners() {
335     this.runnerStats.incrNormalRunners();
336   }
337 
338   /** Increment the number of delay runner counts. */
339   public void incrDelayRunners() {
340     this.runnerStats.incrDelayRunners();
341   }
342 
343   /** Update delay interval of delay runner. */
344   public void updateDelayInterval(long interval) {
345     this.runnerStats.updateDelayInterval(interval);
346   }
347 
348   /**
349    * Get a metric for {@code key} from {@code map}, or create it with {@code factory}.
350    */
351   private <T> T getMetric(String key, ConcurrentMap<String, T> map, NewMetric<T> factory) {
352     T t = map.get(key);
353     if (t == null) {
354       t = factory.newMetric(this.getClass(), key, scope);
355       map.putIfAbsent(key, t);
356     }
357     return t;
358   }
359 
360   /** Update call stats for non-critical-path methods */
361   private void updateRpcGeneric(MethodDescriptor method, CallStats stats) {
362     final String methodName = method.getService().getName() + "_" + method.getName();
363     getMetric(DRTN_BASE + methodName, rpcTimers, timerFactory)
364         .update(stats.getCallTimeMs(), TimeUnit.MILLISECONDS);
365     getMetric(REQ_BASE + methodName, rpcHistograms, histogramFactory)
366         .update(stats.getRequestSizeBytes());
367     getMetric(RESP_BASE + methodName, rpcHistograms, histogramFactory)
368         .update(stats.getResponseSizeBytes());
369   }
370 
371   /** Report RPC context to metrics system. */
372   public void updateRpc(MethodDescriptor method, Message param, CallStats stats) {
373     // this implementation is tied directly to protobuf implementation details. would be better
374     // if we could dispatch based on something static, ie, request Message type.
375     if (method.getService() == ClientService.getDescriptor()) {
376       switch(method.getIndex()) {
377       case 0:
378         assert "Get".equals(method.getName());
379         getTracker.updateRpc(stats);
380         return;
381       case 1:
382         assert "Mutate".equals(method.getName());
383         final MutationType mutationType = ((MutateRequest) param).getMutation().getMutateType();
384         switch(mutationType) {
385         case APPEND:
386           appendTracker.updateRpc(stats);
387           return;
388         case DELETE:
389           deleteTracker.updateRpc(stats);
390           return;
391         case INCREMENT:
392           incrementTracker.updateRpc(stats);
393           return;
394         case PUT:
395           putTracker.updateRpc(stats);
396           return;
397         default:
398           throw new RuntimeException("Unrecognized mutation type " + mutationType);
399         }
400       case 2:
401         assert "Scan".equals(method.getName());
402         scanTracker.updateRpc(stats);
403         return;
404       case 3:
405         assert "BulkLoadHFile".equals(method.getName());
406         // use generic implementation
407         break;
408       case 4:
409         assert "ExecService".equals(method.getName());
410         // use generic implementation
411         break;
412       case 5:
413         assert "ExecRegionServerService".equals(method.getName());
414         // use generic implementation
415         break;
416       case 6:
417         assert "Multi".equals(method.getName());
418         multiTracker.updateRpc(stats);
419         return;
420       default:
421         throw new RuntimeException("Unrecognized ClientService RPC type " + method.getFullName());
422       }
423     }
424     // Fallback to dynamic registry lookup for DDL methods.
425     updateRpcGeneric(method, stats);
426   }
427 }