1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.coprocessor;
22
23 import static org.junit.Assert.assertArrayEquals;
24 import static org.junit.Assert.assertFalse;
25 import static org.junit.Assert.assertNotNull;
26 import static org.junit.Assert.assertTrue;
27
28 import java.io.IOException;
29 import java.lang.reflect.Method;
30 import java.util.ArrayList;
31 import java.util.Arrays;
32 import java.util.List;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hbase.*;
40 import org.apache.hadoop.hbase.client.*;
41 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
42 import org.apache.hadoop.hbase.io.hfile.HFile;
43 import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
44 import org.apache.hadoop.hbase.regionserver.HRegion;
45 import org.apache.hadoop.hbase.regionserver.InternalScanner;
46 import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
47 import org.apache.hadoop.hbase.regionserver.Store;
48 import org.apache.hadoop.hbase.regionserver.StoreFile;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
51 import org.apache.hadoop.hbase.util.JVMClusterUtil;
52 import org.junit.AfterClass;
53 import org.junit.BeforeClass;
54 import org.junit.Test;
55 import org.junit.experimental.categories.Category;
56
57 @Category(MediumTests.class)
58 public class TestRegionObserverInterface {
59 static final Log LOG = LogFactory.getLog(TestRegionObserverInterface.class);
60
61 public static final byte[] TEST_TABLE = Bytes.toBytes("TestTable");
62 public final static byte[] A = Bytes.toBytes("a");
63 public final static byte[] B = Bytes.toBytes("b");
64 public final static byte[] C = Bytes.toBytes("c");
65 public final static byte[] ROW = Bytes.toBytes("testrow");
66
67 private static HBaseTestingUtility util = new HBaseTestingUtility();
68 private static MiniHBaseCluster cluster = null;
69
70 @BeforeClass
71 public static void setupBeforeClass() throws Exception {
72
73 Configuration conf = util.getConfiguration();
74 conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
75 "org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver");
76
77 util.startMiniCluster();
78 cluster = util.getMiniHBaseCluster();
79 }
80
81 @AfterClass
82 public static void tearDownAfterClass() throws Exception {
83 util.shutdownMiniCluster();
84 }
85
86 @Test
87 public void testRegionObserver() throws IOException {
88 byte[] tableName = TEST_TABLE;
89
90
91 HTable table = util.createTable(tableName, new byte[][] {A, B, C});
92 verifyMethodResult(SimpleRegionObserver.class,
93 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
94 "hadDelete"},
95 TEST_TABLE,
96 new Boolean[] {false, false, false, false, false});
97
98 Put put = new Put(ROW);
99 put.add(A, A, A);
100 put.add(B, B, B);
101 put.add(C, C, C);
102 table.put(put);
103
104 verifyMethodResult(SimpleRegionObserver.class,
105 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
106 "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"},
107 TEST_TABLE,
108 new Boolean[] {false, false, true, true, true, true, false}
109 );
110
111 Get get = new Get(ROW);
112 get.addColumn(A, A);
113 get.addColumn(B, B);
114 get.addColumn(C, C);
115 table.get(get);
116
117 verifyMethodResult(SimpleRegionObserver.class,
118 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
119 "hadDelete"},
120 TEST_TABLE,
121 new Boolean[] {true, true, true, true, false}
122 );
123
124 Delete delete = new Delete(ROW);
125 delete.deleteColumn(A, A);
126 delete.deleteColumn(B, B);
127 delete.deleteColumn(C, C);
128 table.delete(delete);
129
130 verifyMethodResult(SimpleRegionObserver.class,
131 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
132 "hadPreBatchMutate", "hadPostBatchMutate", "hadDelete"},
133 TEST_TABLE,
134 new Boolean[] {true, true, true, true, true, true, true}
135 );
136 util.deleteTable(tableName);
137 table.close();
138 }
139
140 @Test
141 public void testRowMutation() throws IOException {
142 byte[] tableName = TEST_TABLE;
143 HTable table = util.createTable(tableName, new byte[][] {A, B, C});
144 verifyMethodResult(SimpleRegionObserver.class,
145 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
146 "hadDeleted"},
147 TEST_TABLE,
148 new Boolean[] {false, false, false, false, false});
149
150 Put put = new Put(ROW);
151 put.add(A, A, A);
152 put.add(B, B, B);
153 put.add(C, C, C);
154
155 Delete delete = new Delete(ROW);
156 delete.deleteColumn(A, A);
157 delete.deleteColumn(B, B);
158 delete.deleteColumn(C, C);
159
160 RowMutations arm = new RowMutations(ROW);
161 arm.add(put);
162 arm.add(delete);
163 table.mutateRow(arm);
164
165 verifyMethodResult(SimpleRegionObserver.class,
166 new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut",
167 "hadDeleted"},
168 TEST_TABLE,
169 new Boolean[] {false, false, true, true, true}
170 );
171 util.deleteTable(tableName);
172 table.close();
173 }
174
175 @Test
176 public void testIncrementHook() throws IOException {
177 byte[] tableName = TEST_TABLE;
178
179 HTable table = util.createTable(tableName, new byte[][] {A, B, C});
180 Increment inc = new Increment(Bytes.toBytes(0));
181 inc.addColumn(A, A, 1);
182
183 verifyMethodResult(SimpleRegionObserver.class,
184 new String[] {"hadPreIncrement", "hadPostIncrement"},
185 tableName,
186 new Boolean[] {false, false}
187 );
188
189 table.increment(inc);
190
191 verifyMethodResult(SimpleRegionObserver.class,
192 new String[] {"hadPreIncrement", "hadPostIncrement"},
193 tableName,
194 new Boolean[] {true, true}
195 );
196 util.deleteTable(tableName);
197 table.close();
198 }
199
200 @Test
201
202 public void testHBase3583() throws IOException {
203 byte[] tableName = Bytes.toBytes("testHBase3583");
204 util.createTable(tableName, new byte[][] {A, B, C});
205
206 verifyMethodResult(SimpleRegionObserver.class,
207 new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled",
208 "wasScannerCloseCalled"},
209 tableName,
210 new Boolean[] {false, false, false, false}
211 );
212
213 HTable table = new HTable(util.getConfiguration(), tableName);
214 Put put = new Put(ROW);
215 put.add(A, A, A);
216 table.put(put);
217
218 Get get = new Get(ROW);
219 get.addColumn(A, A);
220 table.get(get);
221
222
223
224 verifyMethodResult(SimpleRegionObserver.class,
225 new String[] {"hadPreGet", "hadPostGet", "wasScannerNextCalled",
226 "wasScannerCloseCalled"},
227 tableName,
228 new Boolean[] {true, true, false, false}
229 );
230
231 Scan s = new Scan();
232 ResultScanner scanner = table.getScanner(s);
233 try {
234 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
235 }
236 } finally {
237 scanner.close();
238 }
239
240
241 verifyMethodResult(SimpleRegionObserver.class,
242 new String[] {"wasScannerNextCalled", "wasScannerCloseCalled"},
243 tableName,
244 new Boolean[] {true, true}
245 );
246 util.deleteTable(tableName);
247 table.close();
248 }
249
250 @Test
251
252 public void testHBase3758() throws IOException {
253 byte[] tableName = Bytes.toBytes("testHBase3758");
254 util.createTable(tableName, new byte[][] {A, B, C});
255
256 verifyMethodResult(SimpleRegionObserver.class,
257 new String[] {"hadDeleted", "wasScannerOpenCalled"},
258 tableName,
259 new Boolean[] {false, false}
260 );
261
262 HTable table = new HTable(util.getConfiguration(), tableName);
263 Put put = new Put(ROW);
264 put.add(A, A, A);
265 table.put(put);
266
267 Delete delete = new Delete(ROW);
268 table.delete(delete);
269
270 verifyMethodResult(SimpleRegionObserver.class,
271 new String[] {"hadDeleted", "wasScannerOpenCalled"},
272 tableName,
273 new Boolean[] {true, false}
274 );
275
276 Scan s = new Scan();
277 ResultScanner scanner = table.getScanner(s);
278 try {
279 for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
280 }
281 } finally {
282 scanner.close();
283 }
284
285
286 verifyMethodResult(SimpleRegionObserver.class,
287 new String[] {"wasScannerOpenCalled"},
288 tableName,
289 new Boolean[] {true}
290 );
291 util.deleteTable(tableName);
292 table.close();
293 }
294
295
296 public static class EvenOnlyCompactor extends BaseRegionObserver {
297 long lastCompaction;
298 long lastFlush;
299
300 @Override
301 public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
302 Store store, final InternalScanner scanner) {
303 return new InternalScanner() {
304 @Override
305 public boolean next(List<KeyValue> results) throws IOException {
306 return next(results, -1);
307 }
308
309 @Override
310 public boolean next(List<KeyValue> results, String metric)
311 throws IOException {
312 return next(results, -1, metric);
313 }
314
315 @Override
316 public boolean next(List<KeyValue> results, int limit)
317 throws IOException{
318 return next(results, limit, null);
319 }
320
321 @Override
322 public boolean next(List<KeyValue> results, int limit, String metric)
323 throws IOException {
324 List<KeyValue> internalResults = new ArrayList<KeyValue>();
325 boolean hasMore;
326 do {
327 hasMore = scanner.next(internalResults, limit, metric);
328 if (!internalResults.isEmpty()) {
329 long row = Bytes.toLong(internalResults.get(0).getRow());
330 if (row % 2 == 0) {
331
332 break;
333 }
334
335 internalResults.clear();
336 }
337 } while (hasMore);
338
339 if (!internalResults.isEmpty()) {
340 results.addAll(internalResults);
341 }
342 return hasMore;
343 }
344
345 @Override
346 public void close() throws IOException {
347 scanner.close();
348 }
349 };
350 }
351
352 @Override
353 public void postCompact(ObserverContext<RegionCoprocessorEnvironment> e,
354 Store store, StoreFile resultFile) {
355 lastCompaction = EnvironmentEdgeManager.currentTimeMillis();
356 }
357
358 @Override
359 public void postFlush(ObserverContext<RegionCoprocessorEnvironment> e) {
360 lastFlush = EnvironmentEdgeManager.currentTimeMillis();
361 }
362 }
363
364
365
366
367 @Test
368 public void testCompactionOverride() throws Exception {
369 byte[] compactTable = Bytes.toBytes("TestCompactionOverride");
370 HBaseAdmin admin = util.getHBaseAdmin();
371 if (admin.tableExists(compactTable)) {
372 admin.disableTable(compactTable);
373 admin.deleteTable(compactTable);
374 }
375
376 HTableDescriptor htd = new HTableDescriptor(compactTable);
377 htd.addFamily(new HColumnDescriptor(A));
378 htd.addCoprocessor(EvenOnlyCompactor.class.getName());
379 admin.createTable(htd);
380
381 HTable table = new HTable(util.getConfiguration(), compactTable);
382 for (long i=1; i<=10; i++) {
383 byte[] iBytes = Bytes.toBytes(i);
384 Put put = new Put(iBytes);
385 put.setWriteToWAL(false);
386 put.add(A, A, iBytes);
387 table.put(put);
388 }
389
390 HRegion firstRegion = cluster.getRegions(compactTable).get(0);
391 Coprocessor cp = firstRegion.getCoprocessorHost().findCoprocessor(
392 EvenOnlyCompactor.class.getName());
393 assertNotNull("EvenOnlyCompactor coprocessor should be loaded", cp);
394 EvenOnlyCompactor compactor = (EvenOnlyCompactor)cp;
395
396
397 long ts = System.currentTimeMillis();
398 admin.flush(compactTable);
399
400 for (int i=0; i<10; i++) {
401 if (compactor.lastFlush >= ts) {
402 break;
403 }
404 Thread.sleep(1000);
405 }
406 assertTrue("Flush didn't complete", compactor.lastFlush >= ts);
407 LOG.debug("Flush complete");
408
409 ts = compactor.lastFlush;
410 admin.majorCompact(compactTable);
411
412 for (int i=0; i<30; i++) {
413 if (compactor.lastCompaction >= ts) {
414 break;
415 }
416 Thread.sleep(1000);
417 }
418 LOG.debug("Last compaction was at "+compactor.lastCompaction);
419 assertTrue("Compaction didn't complete", compactor.lastCompaction >= ts);
420
421
422 ResultScanner scanner = table.getScanner(new Scan());
423 try {
424 for (long i=2; i<=10; i+=2) {
425 Result r = scanner.next();
426 assertNotNull(r);
427 assertFalse(r.isEmpty());
428 byte[] iBytes = Bytes.toBytes(i);
429 assertArrayEquals("Row should be "+i, r.getRow(), iBytes);
430 assertArrayEquals("Value should be "+i, r.getValue(A, A), iBytes);
431 }
432 } finally {
433 scanner.close();
434 }
435 table.close();
436 }
437
438 @Test
439 public void bulkLoadHFileTest() throws Exception {
440 String testName = TestRegionObserverInterface.class.getName()+".bulkLoadHFileTest";
441 byte[] tableName = TEST_TABLE;
442 Configuration conf = util.getConfiguration();
443 HTable table = util.createTable(tableName, new byte[][] {A, B, C});
444
445 verifyMethodResult(SimpleRegionObserver.class,
446 new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"},
447 tableName,
448 new Boolean[] {false, false}
449 );
450
451 FileSystem fs = util.getTestFileSystem();
452 final Path dir = util.getDataTestDir(testName).makeQualified(fs);
453 Path familyDir = new Path(dir, Bytes.toString(A));
454
455 createHFile(util.getConfiguration(), fs, new Path(familyDir,Bytes.toString(A)), A, A);
456
457
458 new LoadIncrementalHFiles(conf).doBulkLoad(dir, new HTable(conf, tableName));
459
460 verifyMethodResult(SimpleRegionObserver.class,
461 new String[] {"hadPreBulkLoadHFile", "hadPostBulkLoadHFile"},
462 tableName,
463 new Boolean[] {true, true}
464 );
465 util.deleteTable(tableName);
466 table.close();
467 }
468
469
470 private void verifyMethodResult(Class c, String methodName[], byte[] tableName,
471 Object value[]) throws IOException {
472 try {
473 for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) {
474 for (HRegionInfo r : t.getRegionServer().getOnlineRegions()) {
475 if (!Arrays.equals(r.getTableName(), tableName)) {
476 continue;
477 }
478 RegionCoprocessorHost cph = t.getRegionServer().getOnlineRegion(r.getRegionName()).
479 getCoprocessorHost();
480
481 Coprocessor cp = cph.findCoprocessor(c.getName());
482 assertNotNull(cp);
483 for (int i = 0; i < methodName.length; ++i) {
484 Method m = c.getMethod(methodName[i]);
485 Object o = m.invoke(cp);
486 assertTrue("Result of " + c.getName() + "." + methodName[i]
487 + " is expected to be " + value[i].toString()
488 + ", while we get " + o.toString(), o.equals(value[i]));
489 }
490 }
491 }
492 } catch (Exception e) {
493 throw new IOException(e.toString());
494 }
495 }
496
497 private static void createHFile(
498 Configuration conf,
499 FileSystem fs, Path path,
500 byte[] family, byte[] qualifier) throws IOException {
501 HFile.Writer writer = HFile.getWriterFactory(conf, new CacheConfig(conf))
502 .withPath(fs, path)
503 .withComparator(KeyValue.KEY_COMPARATOR)
504 .create();
505 long now = System.currentTimeMillis();
506 try {
507 for (int i =1;i<=9;i++) {
508 KeyValue kv = new KeyValue(Bytes.toBytes(i+""), family, qualifier, now, Bytes.toBytes(i+""));
509 writer.append(kv);
510 }
511 } finally {
512 writer.close();
513 }
514 }
515
516 private static byte [][] makeN(byte [] base, int n) {
517 byte [][] ret = new byte[n][];
518 for(int i=0;i<n;i++) {
519 ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%02d", i)));
520 }
521 return ret;
522 }
523
524 @org.junit.Rule
525 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
526 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
527 }
528
529