View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import org.apache.hadoop.conf.Configuration;
21  import org.apache.hadoop.hbase.*;
22  import org.apache.hadoop.hbase.client.*;
23  import org.apache.hadoop.hbase.test.MetricsAssertHelper;
24  import org.apache.hadoop.hbase.testclassification.MediumTests;
25  import org.apache.hadoop.hbase.util.Bytes;
26  import org.apache.hadoop.hbase.util.Threads;
27  import org.apache.log4j.Level;
28  import org.apache.log4j.Logger;
29  import org.junit.AfterClass;
30  import org.junit.BeforeClass;
31  import org.junit.Test;
32  import org.junit.experimental.categories.Category;
33  
34  import static org.junit.Assert.*;
35  
36  import java.io.IOException;
37  import java.util.ArrayList;
38  import java.util.List;
39  
40  
41  @Category(MediumTests.class)
42  public class TestRegionServerMetrics {
43    private static MetricsAssertHelper metricsHelper;
44  
45    static {
46      Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
47    }
48  
49    private static MiniHBaseCluster cluster;
50    private static HRegionServer rs;
51    private static Configuration conf;
52    private static HBaseTestingUtility TEST_UTIL;
53    private static MetricsRegionServer metricsRegionServer;
54    private static MetricsRegionServerSource serverSource;
55  
56    @BeforeClass
57    public static void startCluster() throws Exception {
58      metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
59      TEST_UTIL = new HBaseTestingUtility();
60      conf = TEST_UTIL.getConfiguration();
61      conf.getLong("hbase.splitlog.max.resubmit", 0);
62      // Make the failure test faster
63      conf.setInt("zookeeper.recovery.retry", 0);
64      conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
65  
66      TEST_UTIL.startMiniCluster(1, 1);
67      cluster = TEST_UTIL.getHBaseCluster();
68  
69      cluster.waitForActiveAndReadyMaster();
70  
71      while (cluster.getLiveRegionServerThreads().size() < 1) {
72        Threads.sleep(100);
73      }
74  
75      rs = cluster.getRegionServer(0);
76      metricsRegionServer = rs.getRegionServerMetrics();
77      serverSource = metricsRegionServer.getMetricsSource();
78    }
79  
80    @AfterClass
81    public static void after() throws Exception {
82      if (TEST_UTIL != null) {
83        TEST_UTIL.shutdownMiniCluster();
84      }
85    }
86  
87    @Test(timeout = 300000)
88    public void testRegionCount() throws Exception {
89      String regionMetricsKey = "regionCount";
90      long regions = metricsHelper.getGaugeLong(regionMetricsKey, serverSource);
91      // Creating a table should add one region
92      TEST_UTIL.createTable(TableName.valueOf("table"), Bytes.toBytes("cf"));
93      metricsHelper.assertGaugeGt(regionMetricsKey, regions, serverSource);
94    }
95  
96    @Test
97    public void testLocalFiles() throws Exception {
98      metricsHelper.assertGauge("percentFilesLocal", 0, serverSource);
99    }
100 
101   @Test
102   public void testRequestCount() throws Exception {
103     String tableNameString = "testRequestCount";
104     TableName tName = TableName.valueOf(tableNameString);
105     byte[] cfName = Bytes.toBytes("d");
106     byte[] row = Bytes.toBytes("rk");
107     byte[] qualifier = Bytes.toBytes("qual");
108     byte[] initValue = Bytes.toBytes("Value");
109 
110     TEST_UTIL.createTable(tName, cfName);
111 
112     Connection connection = TEST_UTIL.getConnection();
113     connection.getTable(tName).close(); //wait for the table to come up.
114 
115     // Do a first put to be sure that the connection is established, meta is there and so on.
116     Table table = connection.getTable(tName);
117     Put p = new Put(row);
118     p.add(cfName, qualifier, initValue);
119     table.put(p);
120 
121     metricsRegionServer.getRegionServerWrapper().forceRecompute();
122     long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
123     long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
124     long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
125 
126     for (int i=0; i< 30; i++) {
127       table.put(p);
128     }
129 
130     metricsRegionServer.getRegionServerWrapper().forceRecompute();
131     metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource);
132     metricsHelper.assertCounter("readRequestCount", readRequests, serverSource);
133     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
134 
135     Get g = new Get(row);
136     for (int i=0; i< 10; i++) {
137       table.get(g);
138     }
139 
140     metricsRegionServer.getRegionServerWrapper().forceRecompute();
141     metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource);
142     metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource);
143     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
144 
145     try (RegionLocator locator = connection.getRegionLocator(tName)) {
146       for ( HRegionLocation location: locator.getAllRegionLocations()) {
147         HRegionInfo i = location.getRegionInfo();
148         MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
149             .getMetrics()
150             .getSource()
151             .getAggregateSource();
152         String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
153             "_table_"+tableNameString +
154             "_region_" + i.getEncodedName()+
155             "_metric";
156         metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
157         metricsHelper.assertCounter(prefix + "_mutateCount", 31, agg);
158       }
159     }
160     List<Get> gets = new ArrayList<Get>();
161     for (int i=0; i< 10; i++) {
162       gets.add(new Get(row));
163     }
164     table.get(gets);
165 
166     // By default, master doesn't host meta now.
167     // Adding some meta related requests
168     requests += 3;
169     readRequests ++;
170 
171     metricsRegionServer.getRegionServerWrapper().forceRecompute();
172     metricsHelper.assertCounter("totalRequestCount", requests + 50, serverSource);
173     metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource);
174     metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
175 
176     List<Put> puts = new ArrayList<>();
177     for (int i=0; i< 30; i++) {
178       puts.add(p);
179     }
180     table.put(puts);
181 
182     metricsRegionServer.getRegionServerWrapper().forceRecompute();
183     metricsHelper.assertCounter("totalRequestCount", requests + 80, serverSource);
184     metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource);
185     metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
186 
187     table.close();
188   }
189 
190   @Test
191   public void testMutationsWithoutWal() throws Exception {
192     TableName tableName = TableName.valueOf("testMutationsWithoutWal");
193     byte[] cf = Bytes.toBytes("d");
194     byte[] row = Bytes.toBytes("rk");
195     byte[] qualifier = Bytes.toBytes("qual");
196     byte[] val = Bytes.toBytes("Value");
197 
198     metricsRegionServer.getRegionServerWrapper().forceRecompute();
199 
200     TEST_UTIL.createTable(tableName, cf);
201 
202     Table t = new HTable(conf, tableName);
203 
204     Put p = new Put(row);
205     p.add(cf, qualifier, val);
206     p.setDurability(Durability.SKIP_WAL);
207 
208     t.put(p);
209 
210     metricsRegionServer.getRegionServerWrapper().forceRecompute();
211     metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
212     long minLength = row.length + cf.length + qualifier.length + val.length;
213     metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
214 
215     t.close();
216   }
217 
218   @Test
219   public void testStoreCount() throws Exception {
220     TableName tableName = TableName.valueOf("testStoreCount");
221     byte[] cf = Bytes.toBytes("d");
222     byte[] row = Bytes.toBytes("rk");
223     byte[] qualifier = Bytes.toBytes("qual");
224     byte[] val = Bytes.toBytes("Value");
225 
226     metricsRegionServer.getRegionServerWrapper().forceRecompute();
227     long stores = metricsHelper.getGaugeLong("storeCount", serverSource);
228     long storeFiles = metricsHelper.getGaugeLong("storeFileCount", serverSource);
229 
230     TEST_UTIL.createTable(tableName, cf);
231 
232     //Force a hfile.
233     Table t = new HTable(conf, tableName);
234     Put p = new Put(row);
235     p.add(cf, qualifier, val);
236     t.put(p);
237     TEST_UTIL.getHBaseAdmin().flush(tableName);
238 
239     metricsRegionServer.getRegionServerWrapper().forceRecompute();
240     metricsHelper.assertGauge("storeCount", stores +1, serverSource);
241     metricsHelper.assertGauge("storeFileCount", storeFiles + 1, serverSource);
242 
243     t.close();
244   }
245 
246   @Test
247   public void testCheckAndPutCount() throws Exception {
248     String tableNameString = "testCheckAndPutCount";
249     TableName tableName = TableName.valueOf(tableNameString);
250     byte[] cf = Bytes.toBytes("d");
251     byte[] row = Bytes.toBytes("rk");
252     byte[] qualifier = Bytes.toBytes("qual");
253     byte[] valOne = Bytes.toBytes("Value");
254     byte[] valTwo = Bytes.toBytes("ValueTwo");
255     byte[] valThree = Bytes.toBytes("ValueThree");
256 
257     TEST_UTIL.createTable(tableName, cf);
258     Table t = new HTable(conf, tableName);
259     Put p = new Put(row);
260     p.add(cf, qualifier, valOne);
261     t.put(p);
262 
263     Put pTwo = new Put(row);
264     pTwo.add(cf, qualifier, valTwo);
265     t.checkAndPut(row, cf, qualifier, valOne, pTwo);
266 
267     Put pThree = new Put(row);
268     pThree.add(cf, qualifier, valThree);
269     t.checkAndPut(row, cf, qualifier, valOne, pThree);
270 
271     metricsRegionServer.getRegionServerWrapper().forceRecompute();
272     metricsHelper.assertCounter("checkMutateFailedCount", 1, serverSource);
273     metricsHelper.assertCounter("checkMutatePassedCount", 1, serverSource);
274 
275     t.close();
276   }
277 
278   @Test
279   public void testIncrement() throws Exception {
280     String tableNameString = "testIncrement";
281     TableName tableName = TableName.valueOf(tableNameString);
282     byte[] cf = Bytes.toBytes("d");
283     byte[] row = Bytes.toBytes("rk");
284     byte[] qualifier = Bytes.toBytes("qual");
285     byte[] val = Bytes.toBytes(0l);
286 
287 
288     TEST_UTIL.createTable(tableName, cf);
289     Table t = new HTable(conf, tableName);
290 
291     Put p = new Put(row);
292     p.add(cf, qualifier, val);
293     t.put(p);
294 
295     for(int count = 0; count< 13; count++) {
296       Increment inc = new Increment(row);
297       inc.addColumn(cf, qualifier, 100);
298       t.increment(inc);
299     }
300 
301     metricsRegionServer.getRegionServerWrapper().forceRecompute();
302     metricsHelper.assertCounter("incrementNumOps", 13, serverSource);
303 
304     t.close();
305   }
306 
307   @Test
308   public void testAppend() throws Exception {
309     String tableNameString = "testAppend";
310     TableName tableName = TableName.valueOf(tableNameString);
311     byte[] cf = Bytes.toBytes("d");
312     byte[] row = Bytes.toBytes("rk");
313     byte[] qualifier = Bytes.toBytes("qual");
314     byte[] val = Bytes.toBytes("One");
315 
316 
317     TEST_UTIL.createTable(tableName, cf);
318     Table t = new HTable(conf, tableName);
319 
320     Put p = new Put(row);
321     p.add(cf, qualifier, val);
322     t.put(p);
323 
324     for(int count = 0; count< 73; count++) {
325       Append append = new Append(row);
326       append.add(cf, qualifier, Bytes.toBytes(",Test"));
327       t.append(append);
328     }
329 
330     metricsRegionServer.getRegionServerWrapper().forceRecompute();
331     metricsHelper.assertCounter("appendNumOps", 73, serverSource);
332 
333     t.close();
334   }
335 
336   @Test
337   public void testScanNext() throws IOException {
338     String tableNameString = "testScanNext";
339     TableName tableName = TableName.valueOf(tableNameString);
340     byte[] cf = Bytes.toBytes("d");
341     byte[] qualifier = Bytes.toBytes("qual");
342     byte[] val = Bytes.toBytes("One");
343 
344 
345     List<Put> puts = new ArrayList<>();
346     for (int insertCount =0; insertCount < 100; insertCount++) {
347       Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
348       p.add(cf, qualifier, val);
349       puts.add(p);
350     }
351     try (HTable t = TEST_UTIL.createTable(tableName, cf)) {
352       t.put(puts);
353 
354       Scan s = new Scan();
355       s.setBatch(1);
356       s.setCaching(1);
357       ResultScanner resultScanners = t.getScanner(s);
358 
359       for (int nextCount = 0; nextCount < 30; nextCount++) {
360         Result result = resultScanners.next();
361         assertNotNull(result);
362         assertEquals(1, result.size());
363       }
364     }
365     try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
366       for ( HRegionLocation location: locator.getAllRegionLocations()) {
367         HRegionInfo i = location.getRegionInfo();
368         MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
369             .getMetrics()
370             .getSource()
371             .getAggregateSource();
372         String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
373             "_table_"+tableNameString +
374             "_region_" + i.getEncodedName()+
375             "_metric";
376         metricsHelper.assertCounter(prefix + "_scanNextNumOps", 30, agg);
377       }
378     }
379   }
380 }