1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20 import org.apache.hadoop.conf.Configuration;
21 import org.apache.hadoop.hbase.*;
22 import org.apache.hadoop.hbase.client.*;
23 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
24 import org.apache.hadoop.hbase.testclassification.MediumTests;
25 import org.apache.hadoop.hbase.util.Bytes;
26 import org.apache.hadoop.hbase.util.Threads;
27 import org.apache.log4j.Level;
28 import org.apache.log4j.Logger;
29 import org.junit.AfterClass;
30 import org.junit.BeforeClass;
31 import org.junit.Test;
32 import org.junit.experimental.categories.Category;
33
34 import static org.junit.Assert.*;
35
36 import java.io.IOException;
37 import java.util.ArrayList;
38 import java.util.List;
39
40
41 @Category(MediumTests.class)
42 public class TestRegionServerMetrics {
43 private static MetricsAssertHelper metricsHelper;
44
45 static {
46 Logger.getLogger("org.apache.hadoop.hbase").setLevel(Level.DEBUG);
47 }
48
49 private static MiniHBaseCluster cluster;
50 private static HRegionServer rs;
51 private static Configuration conf;
52 private static HBaseTestingUtility TEST_UTIL;
53 private static MetricsRegionServer metricsRegionServer;
54 private static MetricsRegionServerSource serverSource;
55
56 @BeforeClass
57 public static void startCluster() throws Exception {
58 metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
59 TEST_UTIL = new HBaseTestingUtility();
60 conf = TEST_UTIL.getConfiguration();
61 conf.getLong("hbase.splitlog.max.resubmit", 0);
62
63 conf.setInt("zookeeper.recovery.retry", 0);
64 conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
65
66 TEST_UTIL.startMiniCluster(1, 1);
67 cluster = TEST_UTIL.getHBaseCluster();
68
69 cluster.waitForActiveAndReadyMaster();
70
71 while (cluster.getLiveRegionServerThreads().size() < 1) {
72 Threads.sleep(100);
73 }
74
75 rs = cluster.getRegionServer(0);
76 metricsRegionServer = rs.getRegionServerMetrics();
77 serverSource = metricsRegionServer.getMetricsSource();
78 }
79
80 @AfterClass
81 public static void after() throws Exception {
82 if (TEST_UTIL != null) {
83 TEST_UTIL.shutdownMiniCluster();
84 }
85 }
86
87 @Test(timeout = 300000)
88 public void testRegionCount() throws Exception {
89 String regionMetricsKey = "regionCount";
90 long regions = metricsHelper.getGaugeLong(regionMetricsKey, serverSource);
91
92 TEST_UTIL.createTable(TableName.valueOf("table"), Bytes.toBytes("cf"));
93 metricsHelper.assertGaugeGt(regionMetricsKey, regions, serverSource);
94 }
95
96 @Test
97 public void testLocalFiles() throws Exception {
98 metricsHelper.assertGauge("percentFilesLocal", 0, serverSource);
99 }
100
101 @Test
102 public void testRequestCount() throws Exception {
103 String tableNameString = "testRequestCount";
104 TableName tName = TableName.valueOf(tableNameString);
105 byte[] cfName = Bytes.toBytes("d");
106 byte[] row = Bytes.toBytes("rk");
107 byte[] qualifier = Bytes.toBytes("qual");
108 byte[] initValue = Bytes.toBytes("Value");
109
110 TEST_UTIL.createTable(tName, cfName);
111
112 Connection connection = TEST_UTIL.getConnection();
113 connection.getTable(tName).close();
114
115
116 Table table = connection.getTable(tName);
117 Put p = new Put(row);
118 p.add(cfName, qualifier, initValue);
119 table.put(p);
120
121 metricsRegionServer.getRegionServerWrapper().forceRecompute();
122 long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
123 long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
124 long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
125
126 for (int i=0; i< 30; i++) {
127 table.put(p);
128 }
129
130 metricsRegionServer.getRegionServerWrapper().forceRecompute();
131 metricsHelper.assertCounter("totalRequestCount", requests + 30, serverSource);
132 metricsHelper.assertCounter("readRequestCount", readRequests, serverSource);
133 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
134
135 Get g = new Get(row);
136 for (int i=0; i< 10; i++) {
137 table.get(g);
138 }
139
140 metricsRegionServer.getRegionServerWrapper().forceRecompute();
141 metricsHelper.assertCounter("totalRequestCount", requests + 40, serverSource);
142 metricsHelper.assertCounter("readRequestCount", readRequests + 10, serverSource);
143 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
144
145 try (RegionLocator locator = connection.getRegionLocator(tName)) {
146 for ( HRegionLocation location: locator.getAllRegionLocations()) {
147 HRegionInfo i = location.getRegionInfo();
148 MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
149 .getMetrics()
150 .getSource()
151 .getAggregateSource();
152 String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
153 "_table_"+tableNameString +
154 "_region_" + i.getEncodedName()+
155 "_metric";
156 metricsHelper.assertCounter(prefix + "_getNumOps", 10, agg);
157 metricsHelper.assertCounter(prefix + "_mutateCount", 31, agg);
158 }
159 }
160 List<Get> gets = new ArrayList<Get>();
161 for (int i=0; i< 10; i++) {
162 gets.add(new Get(row));
163 }
164 table.get(gets);
165
166
167
168 requests += 3;
169 readRequests ++;
170
171 metricsRegionServer.getRegionServerWrapper().forceRecompute();
172 metricsHelper.assertCounter("totalRequestCount", requests + 50, serverSource);
173 metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource);
174 metricsHelper.assertCounter("writeRequestCount", writeRequests + 30, serverSource);
175
176 List<Put> puts = new ArrayList<>();
177 for (int i=0; i< 30; i++) {
178 puts.add(p);
179 }
180 table.put(puts);
181
182 metricsRegionServer.getRegionServerWrapper().forceRecompute();
183 metricsHelper.assertCounter("totalRequestCount", requests + 80, serverSource);
184 metricsHelper.assertCounter("readRequestCount", readRequests + 20, serverSource);
185 metricsHelper.assertCounter("writeRequestCount", writeRequests + 60, serverSource);
186
187 table.close();
188 }
189
190 @Test
191 public void testMutationsWithoutWal() throws Exception {
192 TableName tableName = TableName.valueOf("testMutationsWithoutWal");
193 byte[] cf = Bytes.toBytes("d");
194 byte[] row = Bytes.toBytes("rk");
195 byte[] qualifier = Bytes.toBytes("qual");
196 byte[] val = Bytes.toBytes("Value");
197
198 metricsRegionServer.getRegionServerWrapper().forceRecompute();
199
200 TEST_UTIL.createTable(tableName, cf);
201
202 Table t = new HTable(conf, tableName);
203
204 Put p = new Put(row);
205 p.add(cf, qualifier, val);
206 p.setDurability(Durability.SKIP_WAL);
207
208 t.put(p);
209
210 metricsRegionServer.getRegionServerWrapper().forceRecompute();
211 metricsHelper.assertGauge("mutationsWithoutWALCount", 1, serverSource);
212 long minLength = row.length + cf.length + qualifier.length + val.length;
213 metricsHelper.assertGaugeGt("mutationsWithoutWALSize", minLength, serverSource);
214
215 t.close();
216 }
217
218 @Test
219 public void testStoreCount() throws Exception {
220 TableName tableName = TableName.valueOf("testStoreCount");
221 byte[] cf = Bytes.toBytes("d");
222 byte[] row = Bytes.toBytes("rk");
223 byte[] qualifier = Bytes.toBytes("qual");
224 byte[] val = Bytes.toBytes("Value");
225
226 metricsRegionServer.getRegionServerWrapper().forceRecompute();
227 long stores = metricsHelper.getGaugeLong("storeCount", serverSource);
228 long storeFiles = metricsHelper.getGaugeLong("storeFileCount", serverSource);
229
230 TEST_UTIL.createTable(tableName, cf);
231
232
233 Table t = new HTable(conf, tableName);
234 Put p = new Put(row);
235 p.add(cf, qualifier, val);
236 t.put(p);
237 TEST_UTIL.getHBaseAdmin().flush(tableName);
238
239 metricsRegionServer.getRegionServerWrapper().forceRecompute();
240 metricsHelper.assertGauge("storeCount", stores +1, serverSource);
241 metricsHelper.assertGauge("storeFileCount", storeFiles + 1, serverSource);
242
243 t.close();
244 }
245
246 @Test
247 public void testCheckAndPutCount() throws Exception {
248 String tableNameString = "testCheckAndPutCount";
249 TableName tableName = TableName.valueOf(tableNameString);
250 byte[] cf = Bytes.toBytes("d");
251 byte[] row = Bytes.toBytes("rk");
252 byte[] qualifier = Bytes.toBytes("qual");
253 byte[] valOne = Bytes.toBytes("Value");
254 byte[] valTwo = Bytes.toBytes("ValueTwo");
255 byte[] valThree = Bytes.toBytes("ValueThree");
256
257 TEST_UTIL.createTable(tableName, cf);
258 Table t = new HTable(conf, tableName);
259 Put p = new Put(row);
260 p.add(cf, qualifier, valOne);
261 t.put(p);
262
263 Put pTwo = new Put(row);
264 pTwo.add(cf, qualifier, valTwo);
265 t.checkAndPut(row, cf, qualifier, valOne, pTwo);
266
267 Put pThree = new Put(row);
268 pThree.add(cf, qualifier, valThree);
269 t.checkAndPut(row, cf, qualifier, valOne, pThree);
270
271 metricsRegionServer.getRegionServerWrapper().forceRecompute();
272 metricsHelper.assertCounter("checkMutateFailedCount", 1, serverSource);
273 metricsHelper.assertCounter("checkMutatePassedCount", 1, serverSource);
274
275 t.close();
276 }
277
278 @Test
279 public void testIncrement() throws Exception {
280 String tableNameString = "testIncrement";
281 TableName tableName = TableName.valueOf(tableNameString);
282 byte[] cf = Bytes.toBytes("d");
283 byte[] row = Bytes.toBytes("rk");
284 byte[] qualifier = Bytes.toBytes("qual");
285 byte[] val = Bytes.toBytes(0l);
286
287
288 TEST_UTIL.createTable(tableName, cf);
289 Table t = new HTable(conf, tableName);
290
291 Put p = new Put(row);
292 p.add(cf, qualifier, val);
293 t.put(p);
294
295 for(int count = 0; count< 13; count++) {
296 Increment inc = new Increment(row);
297 inc.addColumn(cf, qualifier, 100);
298 t.increment(inc);
299 }
300
301 metricsRegionServer.getRegionServerWrapper().forceRecompute();
302 metricsHelper.assertCounter("incrementNumOps", 13, serverSource);
303
304 t.close();
305 }
306
307 @Test
308 public void testAppend() throws Exception {
309 String tableNameString = "testAppend";
310 TableName tableName = TableName.valueOf(tableNameString);
311 byte[] cf = Bytes.toBytes("d");
312 byte[] row = Bytes.toBytes("rk");
313 byte[] qualifier = Bytes.toBytes("qual");
314 byte[] val = Bytes.toBytes("One");
315
316
317 TEST_UTIL.createTable(tableName, cf);
318 Table t = new HTable(conf, tableName);
319
320 Put p = new Put(row);
321 p.add(cf, qualifier, val);
322 t.put(p);
323
324 for(int count = 0; count< 73; count++) {
325 Append append = new Append(row);
326 append.add(cf, qualifier, Bytes.toBytes(",Test"));
327 t.append(append);
328 }
329
330 metricsRegionServer.getRegionServerWrapper().forceRecompute();
331 metricsHelper.assertCounter("appendNumOps", 73, serverSource);
332
333 t.close();
334 }
335
336 @Test
337 public void testScanNext() throws IOException {
338 String tableNameString = "testScanNext";
339 TableName tableName = TableName.valueOf(tableNameString);
340 byte[] cf = Bytes.toBytes("d");
341 byte[] qualifier = Bytes.toBytes("qual");
342 byte[] val = Bytes.toBytes("One");
343
344
345 List<Put> puts = new ArrayList<>();
346 for (int insertCount =0; insertCount < 100; insertCount++) {
347 Put p = new Put(Bytes.toBytes("" + insertCount + "row"));
348 p.add(cf, qualifier, val);
349 puts.add(p);
350 }
351 try (HTable t = TEST_UTIL.createTable(tableName, cf)) {
352 t.put(puts);
353
354 Scan s = new Scan();
355 s.setBatch(1);
356 s.setCaching(1);
357 ResultScanner resultScanners = t.getScanner(s);
358
359 for (int nextCount = 0; nextCount < 30; nextCount++) {
360 Result result = resultScanners.next();
361 assertNotNull(result);
362 assertEquals(1, result.size());
363 }
364 }
365 try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
366 for ( HRegionLocation location: locator.getAllRegionLocations()) {
367 HRegionInfo i = location.getRegionInfo();
368 MetricsRegionAggregateSource agg = rs.getRegion(i.getRegionName())
369 .getMetrics()
370 .getSource()
371 .getAggregateSource();
372 String prefix = "namespace_"+NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR+
373 "_table_"+tableNameString +
374 "_region_" + i.getEncodedName()+
375 "_metric";
376 metricsHelper.assertCounter(prefix + "_scanNextNumOps", 30, agg);
377 }
378 }
379 }
380 }