1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import org.apache.commons.logging.Log;
21 import org.apache.commons.logging.LogFactory;
22 import org.apache.hadoop.conf.Configuration;
23 import org.apache.hadoop.hbase.*;
24 import org.apache.hadoop.hbase.client.*;
25 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
26 import org.apache.hadoop.hbase.testclassification.MediumTests;
27 import org.apache.hadoop.hbase.util.Bytes;
28 import org.apache.hadoop.hbase.util.Threads;
29 import org.apache.log4j.Level;
30 import org.apache.log4j.Logger;
31 import org.junit.AfterClass;
32 import org.junit.BeforeClass;
33 import org.junit.Test;
34 import org.junit.experimental.categories.Category;
35 import static org.junit.Assert.*;
36
37 import java.io.IOException;
38 import java.util.ArrayList;
39 import java.util.List;
40
41
42 @Category(MediumTests.class)
43 public class TestRegionServerMetrics {
44 private static final Log LOG = LogFactory.getLog(TestRegionServerMetrics.class);
45 private static MetricsAssertHelper metricsHelper;
46
47 static {
48 Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
49 }
50
51 private static MiniHBaseCluster cluster;
52 private static HRegionServer rs;
53 private static Configuration conf;
54 private static HBaseTestingUtility TEST_UTIL;
55 private static MetricsRegionServer metricsRegionServer;
56 private static MetricsRegionServerSource serverSource;
57 private static final int NUM_SCAN_NEXT = 30;
58
59 @BeforeClass
60 public static void startCluster() throws Exception {
61 metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
62 TEST_UTIL = new HBaseTestingUtility();
63 conf = TEST_UTIL.getConfiguration();
64 conf.getLong("hbase.splitlog.max.resubmit", 0);
65
66 conf.setInt("zookeeper.recovery.retry", 0);
67 conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
68
69 TEST_UTIL.startMiniCluster(1, 1);
70 cluster = TEST_UTIL.getHBaseCluster();
71
72 cluster.waitForActiveAndReadyMaster();
73
74 while (cluster.getLiveRegionServerThreads().size() < 1) {
75 Threads.sleep(100);
76 }
77
78 rs = cluster.getRegionServer(0);
79 metricsRegionServer = rs.getMetrics();
80 serverSource = metricsRegionServer.getMetricsSource();
81 }
82
83 @AfterClass
84 public static void after() throws Exception {
85 if (TEST_UTIL != null) {
86 TEST_UTIL.shutdownMiniCluster();
87 }
88 }
89
90 @Test(timeout = 300000)
91 public void testRegionCount() throws Exception {
92 String regionMetricsKey = "regionCount";
93 long regions = metricsHelper.getGaugeLong(regionMetricsKey, serverSource);
94
95 TEST_UTIL.createTable(Bytes.toBytes("table"), Bytes.toBytes("cf"));
96 metricsHelper.assertGaugeGt(regionMetricsKey, regions, serverSource);
97 }
98
99 @Test
100 public void testLocalFiles() throws Exception {
101 metricsHelper.assertGauge("percentFilesLocal", 0, serverSource);
102 }
103
104 @Test
105 public void testRequestCount() throws Exception {
106 String tableNameString = "testRequestCount";
107 byte[] tName = Bytes.toBytes(tableNameString);
108 byte[] cfName = Bytes.toBytes("d");
109 byte[] row = Bytes.toBytes("rk");
110 byte[] qualifier = Bytes.toBytes("qual");
111 byte[] initValue = Bytes.toBytes("Value");
112 byte[] nextValue = Bytes.toBytes("NEXT VAL");
113
114
115 TEST_UTIL.createTable(tName, cfName);
116
117 new HTable(conf, tName).close();
118
119
120 HTable table = new HTable(conf, tName);
121 Put p = new Put(row);
122 p.add(cfName, qualifier, initValue);
123 table.put(p);
124
125 metricsRegionServer.getRegionServerWrapper().forceRecompute();
126 long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
127 long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
128 long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
129
130 for (int i=0; i< 30; i++) {
131 table.put(p);
132 }
133
134 metricsRegionServer.getRegionServerWrapper().forceRecompute();
135 metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource);
136 metricsHelper.assertCounter("readRequestCount", readRequests, serverSource);
137 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
138
139 Get g = new Get(row);
140 for (int i=0; i< 10; i++) {
141 table.get(g);
142 }
143
144 metricsRegionServer.getRegionServerWrapper().forceRecompute();
145 metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource);
146 metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource);
147 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
148
149 for ( HRegionInfo i:table.getRegionLocations().keySet()) {
150 MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
151 .getMetrics()
152 .getSource()
153 .getAggregateSource();
154 String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
155 "_table_"+tableNameString +
156 "_region_" + i.getEncodedName()+
157 "_metric";
158 metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
159 metricsHelper.assertCounter(prefix + "_mutateCount", 31, agg);
160 }
161
162
163 metricsRegionServer.getRegionServerWrapper().forceRecompute();
164 metricsHelper.assertCounter("totalRequestCount", requests + 40 + 3, serverSource);
165 metricsHelper.assertCounter("readRequestCount", readRequests + 10 + 1, serverSource);
166
167
168 List<Get> gets = new ArrayList<Get>();
169 for (int i=0; i< 10; i++) {
170 gets.add(new Get(row));
171 }
172 table.get(gets);
173
174 metricsRegionServer.getRegionServerWrapper().forceRecompute();
175 metricsHelper.assertCounter("totalRequestCount", requests + 50 + 3, serverSource);
176 metricsHelper.assertCounter("readRequestCount", readRequests + 20 + 1, serverSource);
177 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
178
179 table.setAutoFlushTo(false);
180 for (int i=0; i< 30; i++) {
181 table.put(p);
182 }
183 table.flushCommits();
184
185 metricsRegionServer.getRegionServerWrapper().forceRecompute();
186 metricsHelper.assertCounter("totalRequestCount", requests + 80 + 3, serverSource);
187 metricsHelper.assertCounter("readRequestCount", readRequests + 20 + 1, serverSource);
188 metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
189
190 table.close();
191 }
192
193 @Test
194 public void testMutationsWithoutWal() throws Exception {
195 byte[] tableName = Bytes.toBytes("testMutationsWithoutWal");
196 byte[] cf = Bytes.toBytes("d");
197 byte[] row = Bytes.toBytes("rk");
198 byte[] qualifier = Bytes.toBytes("qual");
199 byte[] val = Bytes.toBytes("Value");
200
201 metricsRegionServer.getRegionServerWrapper().forceRecompute();
202
203 TEST_UTIL.createTable(tableName, cf);
204
205 HTable t = new HTable(conf, tableName);
206
207 Put p = new Put(row);
208 p.add(cf, qualifier, val);
209 p.setDurability(Durability.SKIP_WAL);
210
211 t.put(p);
212 t.flushCommits();
213
214 metricsRegionServer.getRegionServerWrapper().forceRecompute();
215 metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
216 long minLength = row.length + cf.length + qualifier.length + val.length;
217 metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
218
219 t.close();
220 }
221
222 @Test
223 public void testStoreCount() throws Exception {
224 byte[] tableName = Bytes.toBytes("testStoreCount");
225 byte[] cf = Bytes.toBytes("d");
226 byte[] row = Bytes.toBytes("rk");
227 byte[] qualifier = Bytes.toBytes("qual");
228 byte[] val = Bytes.toBytes("Value");
229
230 metricsRegionServer.getRegionServerWrapper().forceRecompute();
231 long stores = metricsHelper.getGaugeLong("storeCount", serverSource);
232 long storeFiles = metricsHelper.getGaugeLong("storeFileCount", serverSource);
233
234 TEST_UTIL.createTable(tableName, cf);
235
236
237 HTable t = new HTable(conf, tableName);
238 Put p = new Put(row);
239 p.add(cf, qualifier, val);
240 t.put(p);
241 t.flushCommits();
242 TEST_UTIL.getHBaseAdmin().flush(tableName);
243
244 metricsRegionServer.getRegionServerWrapper().forceRecompute();
245 metricsHelper.assertGauge("storeCount", stores +1, serverSource);
246 metricsHelper.assertGauge("storeFileCount", storeFiles + 1, serverSource);
247
248 t.close();
249 }
250
251 @Test
252 public void testCheckAndPutCount() throws Exception {
253 String tableNameString = "testCheckAndPutCount";
254 byte[] tableName = Bytes.toBytes(tableNameString);
255 byte[] cf = Bytes.toBytes("d");
256 byte[] row = Bytes.toBytes("rk");
257 byte[] qualifier = Bytes.toBytes("qual");
258 byte[] valOne = Bytes.toBytes("Value");
259 byte[] valTwo = Bytes.toBytes("ValueTwo");
260 byte[] valThree = Bytes.toBytes("ValueThree");
261
262 TEST_UTIL.createTable(tableName, cf);
263 HTable t = new HTable(conf, tableName);
264 Put p = new Put(row);
265 p.add(cf, qualifier, valOne);
266 t.put(p);
267 t.flushCommits();
268
269 Put pTwo = new Put(row);
270 pTwo.add(cf, qualifier, valTwo);
271 t.checkAndPut(row, cf, qualifier, valOne, pTwo);
272 t.flushCommits();
273
274 Put pThree = new Put(row);
275 pThree.add(cf, qualifier, valThree);
276 t.checkAndPut(row, cf, qualifier, valOne, pThree);
277 t.flushCommits();
278
279
280 metricsRegionServer.getRegionServerWrapper().forceRecompute();
281 metricsHelper.assertCounter("checkMutateFailedCount", 1, serverSource);
282 metricsHelper.assertCounter("checkMutatePassedCount", 1, serverSource);
283
284 t.close();
285 }
286
287 @Test
288 public void testIncrement() throws Exception {
289 String tableNameString = "testIncrement";
290 byte[] tableName = Bytes.toBytes(tableNameString);
291 byte[] cf = Bytes.toBytes("d");
292 byte[] row = Bytes.toBytes("rk");
293 byte[] qualifier = Bytes.toBytes("qual");
294 byte[] val = Bytes.toBytes(0l);
295
296
297 TEST_UTIL.createTable(tableName, cf);
298 HTable t = new HTable(conf, tableName);
299
300 Put p = new Put(row);
301 p.add(cf, qualifier, val);
302 t.put(p);
303 t.flushCommits();
304
305 for(int count = 0; count< 13; count++) {
306 Increment inc = new Increment(row);
307 inc.addColumn(cf, qualifier, 100);
308 t.increment(inc);
309 }
310
311 t.flushCommits();
312
313 metricsRegionServer.getRegionServerWrapper().forceRecompute();
314 metricsHelper.assertCounter("incrementNumOps", 13, serverSource);
315
316 t.close();
317 }
318
319 @Test
320 public void testAppend() throws Exception {
321 String tableNameString = "testAppend";
322 byte[] tableName = Bytes.toBytes(tableNameString);
323 byte[] cf = Bytes.toBytes("d");
324 byte[] row = Bytes.toBytes("rk");
325 byte[] qualifier = Bytes.toBytes("qual");
326 byte[] val = Bytes.toBytes("One");
327
328
329 TEST_UTIL.createTable(tableName, cf);
330 HTable t = new HTable(conf, tableName);
331
332 Put p = new Put(row);
333 p.add(cf, qualifier, val);
334 t.put(p);
335 t.flushCommits();
336
337 for(int count = 0; count< 73; count++) {
338 Append append = new Append(row);
339 append.add(cf, qualifier, Bytes.toBytes(",Test"));
340 t.append(append);
341 }
342
343 t.flushCommits();
344
345 metricsRegionServer.getRegionServerWrapper().forceRecompute();
346 metricsHelper.assertCounter("appendNumOps", 73, serverSource);
347
348 t.close();
349 }
350
351 @Test
352 public void testScanNext() throws IOException {
353 String tableNameString = "testScanNext";
354 byte[] tableName = Bytes.toBytes(tableNameString);
355 byte[] cf = Bytes.toBytes("d");
356 byte[] qualifier = Bytes.toBytes("qual");
357 byte[] val = Bytes.toBytes("One");
358
359 TEST_UTIL.createTable(tableName, cf);
360 HTable t = new HTable(conf, tableName);
361 t.setAutoFlush(false, true);
362 for (int insertCount =0; insertCount < 100; insertCount++) {
363 Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
364 p.add(cf, qualifier, val);
365 t.put(p);
366 }
367 t.flushCommits();
368
369 Scan s = new Scan();
370 s.setBatch(1);
371 s.setCaching(1);
372 ResultScanner resultScanners = t.getScanner(s);
373
374 long numScanNext = metricsHelper.getCounter("ScanNext_num_ops", serverSource);
375 for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
376 Result result = resultScanners.next();
377 assertNotNull(result);
378 assertEquals(1, result.size());
379 }
380 numScanNext += NUM_SCAN_NEXT;
381 metricsHelper.assertCounter("ScanNext_num_ops", numScanNext, serverSource);
382 for ( HRegionInfo i:t.getRegionLocations().keySet()) {
383 MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
384 .getMetrics()
385 .getSource()
386 .getAggregateSource();
387 String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
388 "_table_"+tableNameString +
389 "_region_" + i.getEncodedName()+
390 "_metric";
391 metricsHelper.assertCounter(prefix + "_scanNextNumOps", NUM_SCAN_NEXT, agg);
392 }
393
394 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
395 admin.disableTable(tableName);
396 admin.deleteTable(tableName);
397 }
398
399 @Test
400 public void testScanNextForSmallScan() throws IOException {
401 String tableNameString = "testScanNextSmall";
402 TableName tableName = TableName.valueOf(tableNameString);
403 byte[] cf = Bytes.toBytes("d");
404 byte[] qualifier = Bytes.toBytes("qual");
405 byte[] val = Bytes.toBytes("One");
406
407 TEST_UTIL.createTable(tableName, cf);
408 HTable t = new HTable(conf, tableName);
409 t.setAutoFlush(false, true);
410 for (int insertCount =0; insertCount < 100; insertCount++) {
411 Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
412 p.add(cf, qualifier, val);
413 t.put(p);
414 }
415 t.flushCommits();
416
417 Scan s = new Scan();
418 s.setSmall(true);
419 s.setCaching(1);
420 ResultScanner resultScanners = t.getScanner(s);
421
422 long numScanNext = metricsHelper.getCounter("ScanNext_num_ops", serverSource);
423 for (int nextCount = 0; nextCount < NUM_SCAN_NEXT; nextCount++) {
424 Result result = resultScanners.next();
425 assertNotNull(result);
426 assertEquals(1, result.size());
427 }
428 numScanNext += NUM_SCAN_NEXT;
429 metricsHelper.assertCounter("ScanNext_num_ops", numScanNext, serverSource);
430 for ( HRegionInfo i:t.getRegionLocations().keySet()) {
431 MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
432 .getMetrics()
433 .getSource()
434 .getAggregateSource();
435 String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
436 "_table_"+tableNameString +
437 "_region_" + i.getEncodedName()+
438 "_metric";
439 metricsHelper.assertCounter(prefix + "_scanNextNumOps", NUM_SCAN_NEXT, agg);
440 }
441
442 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
443 admin.disableTable(tableName);
444 admin.deleteTable(tableName);
445 }
446 }