View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import org.apache.commons.logging.Log;
21  import org.apache.commons.logging.LogFactory;
22  import org.apache.hadoop.conf.Configuration;
23  import org.apache.hadoop.hbase.*;
24  import org.apache.hadoop.hbase.client.*;
25  import org.apache.hadoop.hbase.test.MetricsAssertHelper;
26  import org.apache.hadoop.hbase.util.Bytes;
27  import org.apache.hadoop.hbase.util.Threads;
28  import org.apache.log4j.Level;
29  import org.apache.log4j.Logger;
30  import org.junit.AfterClass;
31  import org.junit.BeforeClass;
32  import org.junit.Test;
33  import org.junit.experimental.categories.Category;
34  import static org.junit.Assert.*;
35  
36  import java.io.IOException;
37  
38  
39  @Category(MediumTests.class)
40  public class TestRegionServerMetrics {
41    private static final Log LOG = LogFactory.getLog(TestRegionServerMetrics.class);
42    private static MetricsAssertHelper metricsHelper;
43  
44    static {
45      Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
46    }
47  
48    private static MiniHBaseCluster cluster;
49    private static HRegionServer rs;
50    private static Configuration conf;
51    private static HBaseTestingUtility TEST_UTIL;
52    private static MetricsRegionServer metricsRegionServer;
53    private static MetricsRegionServerSource serverSource;
54  
55    @BeforeClass
56    public static void startCluster() throws Exception {
57      metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
58      TEST_UTIL = new HBaseTestingUtility();
59      conf = TEST_UTIL.getConfiguration();
60      conf.getLong("hbase.splitlog.max.resubmit", 0);
61      // Make the failure test faster
62      conf.setInt("zookeeper.recovery.retry", 0);
63      conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
64  
65      TEST_UTIL.startMiniCluster(1, 1);
66      cluster = TEST_UTIL.getHBaseCluster();
67  
68      cluster.waitForActiveAndReadyMaster();
69  
70      while (cluster.getLiveRegionServerThreads().size() < 1) {
71        Threads.sleep(100);
72      }
73  
74      rs = cluster.getRegionServer(0);
75      metricsRegionServer = rs.getMetrics();
76      serverSource = metricsRegionServer.getMetricsSource();
77    }
78  
79    @AfterClass
80    public static void after() throws Exception {
81      if (TEST_UTIL != null) {
82        TEST_UTIL.shutdownMiniCluster();
83      }
84    }
85  
86    @Test(timeout = 300000)
87    public void testRegionCount() throws Exception {
88      String regionMetricsKey = "regionCount";
89      long regions = metricsHelper.getGaugeLong(regionMetricsKey, serverSource);
90      // Creating a table should add one region
91      TEST_UTIL.createTable(Bytes.toBytes("table"), Bytes.toBytes("cf"));
92      metricsHelper.assertGaugeGt(regionMetricsKey, regions, serverSource);
93    }
94  
95    @Test
96    public void testLocalFiles() throws Exception {
97      metricsHelper.assertGauge("percentFilesLocal", 0, serverSource);
98    }
99  
100   @Test
101   public void testRequestCount() throws Exception {
102     String tableNameString = "testRequestCount";
103     byte[] tName = Bytes.toBytes(tableNameString);
104     byte[] cfName = Bytes.toBytes("d");
105     byte[] row = Bytes.toBytes("rk");
106     byte[] qualifier = Bytes.toBytes("qual");
107     byte[] initValue = Bytes.toBytes("Value");
108     byte[] nextValue = Bytes.toBytes("NEXT VAL");
109 
110 
111     TEST_UTIL.createTable(tName, cfName);
112 
113     new HTable(conf, tName).close(); //wait for the table to come up.
114     metricsRegionServer.getRegionServerWrapper().forceRecompute();
115     long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
116     long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
117     long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
118 
119     HTable table = new HTable(conf, tName);
120 
121     Put p = new Put(row);
122 
123 
124     p.add(cfName, qualifier, initValue);
125 
126     for (int i=0; i< 30; i++) {
127       table.put(p);
128     }
129 
130 
131     table.flushCommits();
132 
133     Get g = new Get(row);
134     for (int i=0; i< 10; i++) {
135       table.get(g);
136     }
137 
138 
139     for ( HRegionInfo i:table.getRegionLocations().keySet()) {
140       MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
141           .getMetrics()
142           .getSource()
143           .getAggregateSource();
144       String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
145           "_table_"+tableNameString +
146           "_region_" + i.getEncodedName()+
147           "_metric";
148       metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
149       metricsHelper.assertCounter(prefix + "_mutateCount", 30, agg);
150     }
151 
152 
153     metricsRegionServer.getRegionServerWrapper().forceRecompute();
154     metricsHelper.assertCounterGt("totalRequestCount", requests + 39, serverSource);
155     metricsHelper.assertCounterGt("readRequestCount", readRequests + 9, serverSource);
156     metricsHelper.assertCounterGt("writeRequestCount", writeRequests + 29, serverSource);
157 
158     table.close();
159   }
160 
161   @Test
162   public void testMutationsWithoutWal() throws Exception {
163     byte[] tableName = Bytes.toBytes("testMutationsWithoutWal");
164     byte[] cf = Bytes.toBytes("d");
165     byte[] row = Bytes.toBytes("rk");
166     byte[] qualifier = Bytes.toBytes("qual");
167     byte[] val = Bytes.toBytes("Value");
168 
169     metricsRegionServer.getRegionServerWrapper().forceRecompute();
170 
171     TEST_UTIL.createTable(tableName, cf);
172 
173     HTable t = new HTable(conf, tableName);
174 
175     Put p = new Put(row);
176     p.add(cf, qualifier, val);
177     p.setDurability(Durability.SKIP_WAL);
178 
179     t.put(p);
180     t.flushCommits();
181 
182     metricsRegionServer.getRegionServerWrapper().forceRecompute();
183     metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
184     long minLength = row.length + cf.length + qualifier.length + val.length;
185     metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
186 
187     t.close();
188   }
189 
190   @Test
191   public void testStoreCount() throws Exception {
192     byte[] tableName = Bytes.toBytes("testStoreCount");
193     byte[] cf = Bytes.toBytes("d");
194     byte[] row = Bytes.toBytes("rk");
195     byte[] qualifier = Bytes.toBytes("qual");
196     byte[] val = Bytes.toBytes("Value");
197 
198     metricsRegionServer.getRegionServerWrapper().forceRecompute();
199     long stores = metricsHelper.getGaugeLong("storeCount", serverSource);
200     long storeFiles = metricsHelper.getGaugeLong("storeFileCount", serverSource);
201 
202     TEST_UTIL.createTable(tableName, cf);
203 
204     //Force a hfile.
205     HTable t = new HTable(conf, tableName);
206     Put p = new Put(row);
207     p.add(cf, qualifier, val);
208     t.put(p);
209     t.flushCommits();
210     TEST_UTIL.getHBaseAdmin().flush(tableName);
211 
212     metricsRegionServer.getRegionServerWrapper().forceRecompute();
213     metricsHelper.assertGauge("storeCount", stores +1, serverSource);
214     metricsHelper.assertGauge("storeFileCount", storeFiles + 1, serverSource);
215 
216     t.close();
217   }
218 
219   @Test
220   public void testCheckAndPutCount() throws Exception {
221     String tableNameString = "testCheckAndPutCount";
222     byte[] tableName = Bytes.toBytes(tableNameString);
223     byte[] cf = Bytes.toBytes("d");
224     byte[] row = Bytes.toBytes("rk");
225     byte[] qualifier = Bytes.toBytes("qual");
226     byte[] valOne = Bytes.toBytes("Value");
227     byte[] valTwo = Bytes.toBytes("ValueTwo");
228     byte[] valThree = Bytes.toBytes("ValueThree");
229 
230     TEST_UTIL.createTable(tableName, cf);
231     HTable t = new HTable(conf, tableName);
232     Put p = new Put(row);
233     p.add(cf, qualifier, valOne);
234     t.put(p);
235     t.flushCommits();
236 
237     Put pTwo = new Put(row);
238     pTwo.add(cf, qualifier, valTwo);
239     t.checkAndPut(row, cf, qualifier, valOne, pTwo);
240     t.flushCommits();
241 
242     Put pThree = new Put(row);
243     pThree.add(cf, qualifier, valThree);
244     t.checkAndPut(row, cf, qualifier, valOne, pThree);
245     t.flushCommits();
246 
247 
248     metricsRegionServer.getRegionServerWrapper().forceRecompute();
249     metricsHelper.assertCounter("checkMutateFailedCount", 1, serverSource);
250     metricsHelper.assertCounter("checkMutatePassedCount", 1, serverSource);
251 
252     t.close();
253   }
254 
255   @Test
256   public void testIncrement() throws Exception {
257     String tableNameString = "testIncrement";
258     byte[] tableName = Bytes.toBytes(tableNameString);
259     byte[] cf = Bytes.toBytes("d");
260     byte[] row = Bytes.toBytes("rk");
261     byte[] qualifier = Bytes.toBytes("qual");
262     byte[] val = Bytes.toBytes(0l);
263 
264 
265     TEST_UTIL.createTable(tableName, cf);
266     HTable t = new HTable(conf, tableName);
267 
268     Put p = new Put(row);
269     p.add(cf, qualifier, val);
270     t.put(p);
271     t.flushCommits();
272 
273     for(int count = 0; count< 13; count++) {
274       Increment inc = new Increment(row);
275       inc.addColumn(cf, qualifier, 100);
276       t.increment(inc);
277     }
278 
279     t.flushCommits();
280 
281     metricsRegionServer.getRegionServerWrapper().forceRecompute();
282     metricsHelper.assertCounter("incrementNumOps", 13, serverSource);
283 
284     t.close();
285   }
286 
287   @Test
288   public void testAppend() throws Exception {
289     String tableNameString = "testAppend";
290     byte[] tableName = Bytes.toBytes(tableNameString);
291     byte[] cf = Bytes.toBytes("d");
292     byte[] row = Bytes.toBytes("rk");
293     byte[] qualifier = Bytes.toBytes("qual");
294     byte[] val = Bytes.toBytes("One");
295 
296 
297     TEST_UTIL.createTable(tableName, cf);
298     HTable t = new HTable(conf, tableName);
299 
300     Put p = new Put(row);
301     p.add(cf, qualifier, val);
302     t.put(p);
303     t.flushCommits();
304 
305     for(int count = 0; count< 73; count++) {
306       Append append = new Append(row);
307       append.add(cf, qualifier, Bytes.toBytes(",Test"));
308       t.append(append);
309     }
310 
311     t.flushCommits();
312 
313     metricsRegionServer.getRegionServerWrapper().forceRecompute();
314     metricsHelper.assertCounter("appendNumOps", 73, serverSource);
315 
316     t.close();
317   }
318 
319   @Test
320   public void testScanNext() throws IOException {
321     String tableNameString = "testScanNext";
322     byte[] tableName = Bytes.toBytes(tableNameString);
323     byte[] cf = Bytes.toBytes("d");
324     byte[] qualifier = Bytes.toBytes("qual");
325     byte[] val = Bytes.toBytes("One");
326 
327 
328     TEST_UTIL.createTable(tableName, cf);
329     HTable t = new HTable(conf, tableName);
330     t.setAutoFlush(false, true);
331     for (int insertCount =0; insertCount < 100; insertCount++) {
332       Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
333       p.add(cf, qualifier, val);
334       t.put(p);
335     }
336     t.flushCommits();
337 
338     Scan s = new Scan();
339     s.setBatch(1);
340     s.setCaching(1);
341     ResultScanner resultScanners = t.getScanner(s);
342 
343     for (int nextCount = 0; nextCount < 30; nextCount++) {
344       Result result = resultScanners.next();
345       assertNotNull(result);
346       assertEquals(1, result.size());
347     }
348     for ( HRegionInfo i:t.getRegionLocations().keySet()) {
349       MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
350           .getMetrics()
351           .getSource()
352           .getAggregateSource();
353       String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
354           "_table_"+tableNameString +
355           "_region_" + i.getEncodedName()+
356           "_metric";
357       metricsHelper.assertCounter(prefix + "_scanNextNumOps", 30, agg);
358     }
359   }
360 }