1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.coprocessor;
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.hbase.*;
26 import org.apache.hadoop.hbase.client.HBaseAdmin;
27 import org.apache.hadoop.hbase.regionserver.HRegion;
28 import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
29 import org.apache.hadoop.hbase.testclassification.MediumTests;
30 import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
31 import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
32 import org.apache.hadoop.hdfs.MiniDFSCluster;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.ServerLoad;
36 import org.apache.hadoop.hbase.RegionLoad;
37
38 import java.io.*;
39 import java.util.*;
40
41 import org.junit.*;
42 import org.junit.experimental.categories.Category;
43
44 import static org.junit.Assert.assertEquals;
45 import static org.junit.Assert.assertNotNull;
46 import static org.junit.Assert.assertTrue;
47 import static org.junit.Assert.assertFalse;
48
49
50
51
52 @Category(MediumTests.class)
53 public class TestClassLoading {
54 private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
55 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56
57 private static MiniDFSCluster cluster;
58
59 static final String tableName = "TestClassLoading";
60 static final String cpName1 = "TestCP1";
61 static final String cpName2 = "TestCP2";
62 static final String cpName3 = "TestCP3";
63 static final String cpName4 = "TestCP4";
64 static final String cpName5 = "TestCP5";
65 static final String cpName6 = "TestCP6";
66
67 private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
68
69 private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
70 private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
71 private static Class<?> masterCoprocessor = BaseMasterObserver.class;
72
73 private static final String[] regionServerSystemCoprocessors =
74 new String[]{
75 regionServerCoprocessor.getSimpleName()
76 };
77
78 private static final String[] masterRegionServerSystemCoprocessors = new String[] {
79 regionCoprocessor1.getSimpleName(), MultiRowMutationEndpoint.class.getSimpleName(),
80 regionServerCoprocessor.getSimpleName() };
81
82 @BeforeClass
83 public static void setUpBeforeClass() throws Exception {
84 Configuration conf = TEST_UTIL.getConfiguration();
85
86
87
88 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
89 regionCoprocessor1.getName());
90
91
92
93
94 conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
95 regionCoprocessor2.getName());
96
97 conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
98 regionServerCoprocessor.getName());
99 conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
100 masterCoprocessor.getName());
101 TEST_UTIL.startMiniCluster(1);
102 cluster = TEST_UTIL.getDFSCluster();
103 }
104
105 @AfterClass
106 public static void tearDownAfterClass() throws Exception {
107 TEST_UTIL.shutdownMiniCluster();
108 }
109
110 static File buildCoprocessorJar(String className) throws Exception {
111 String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
112 "public class " + className + " extends BaseRegionObserver {}";
113 return ClassLoaderTestHelper.buildJar(
114 TEST_UTIL.getDataTestDir().toString(), className, code);
115 }
116
117 @Test
118
119 public void testClassLoadingFromHDFS() throws Exception {
120 FileSystem fs = cluster.getFileSystem();
121
122 File jarFile1 = buildCoprocessorJar(cpName1);
123 File jarFile2 = buildCoprocessorJar(cpName2);
124
125
126 fs.copyFromLocalFile(new Path(jarFile1.getPath()),
127 new Path(fs.getUri().toString() + Path.SEPARATOR));
128 String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
129 jarFile1.getName();
130 Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
131 assertTrue("Copy jar file to HDFS failed.",
132 fs.exists(pathOnHDFS1));
133 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
134
135 fs.copyFromLocalFile(new Path(jarFile2.getPath()),
136 new Path(fs.getUri().toString() + Path.SEPARATOR));
137 String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
138 jarFile2.getName();
139 Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
140 assertTrue("Copy jar file to HDFS failed.",
141 fs.exists(pathOnHDFS2));
142 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
143
144
145 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
146 htd.addFamily(new HColumnDescriptor("test"));
147
148 htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
149 "|" + Coprocessor.PRIORITY_USER);
150
151 htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
152 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
153 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
154 if (admin.tableExists(tableName)) {
155 if (admin.isTableEnabled(tableName)) {
156 admin.disableTable(tableName);
157 }
158 admin.deleteTable(tableName);
159 }
160 CoprocessorClassLoader.clearCache();
161 byte[] startKey = {10, 63};
162 byte[] endKey = {12, 43};
163 admin.createTable(htd, startKey, endKey, 4);
164 waitForTable(htd.getTableName());
165
166
167 boolean foundTableRegion=false;
168 boolean found1 = true, found2 = true, found2_k1 = true, found2_k2 = true, found2_k3 = true;
169 Map<HRegion, Set<ClassLoader>> regionsActiveClassLoaders =
170 new HashMap<HRegion, Set<ClassLoader>>();
171 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
172 for (HRegion region:
173 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
174 if (region.getRegionNameAsString().startsWith(tableName)) {
175 foundTableRegion = true;
176 CoprocessorEnvironment env;
177 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
178 found1 = found1 && (env != null);
179 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
180 found2 = found2 && (env != null);
181 if (env != null) {
182 Configuration conf = env.getConfiguration();
183 found2_k1 = found2_k1 && (conf.get("k1") != null);
184 found2_k2 = found2_k2 && (conf.get("k2") != null);
185 found2_k3 = found2_k3 && (conf.get("k3") != null);
186 } else {
187 found2_k1 = found2_k2 = found2_k3 = false;
188 }
189 regionsActiveClassLoaders
190 .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
191 }
192 }
193
194 assertTrue("No region was found for table " + tableName, foundTableRegion);
195 assertTrue("Class " + cpName1 + " was missing on a region", found1);
196 assertTrue("Class " + cpName2 + " was missing on a region", found2);
197 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
198 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
199 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
200
201 assertNotNull(jarFileOnHDFS1 + " was not cached",
202 CoprocessorClassLoader.getIfCached(pathOnHDFS1));
203 assertNotNull(jarFileOnHDFS2 + " was not cached",
204 CoprocessorClassLoader.getIfCached(pathOnHDFS2));
205
206 assertEquals("The number of cached classloaders should be equal to the number" +
207 " of external jar files",
208 2, CoprocessorClassLoader.getAllCached().size());
209
210 Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
211 CoprocessorClassLoader.getAllCached());
212 for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
213 assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
214 + " ClassLoader Cache:" + externalClassLoaders
215 + " Region ClassLoaders:" + regionCP.getValue(),
216 externalClassLoaders.containsAll(regionCP.getValue()));
217 }
218 }
219
220 private String getLocalPath(File file) {
221 return new Path(file.toURI()).toString();
222 }
223
224 @Test
225
226 public void testClassLoadingFromLocalFS() throws Exception {
227 File jarFile = buildCoprocessorJar(cpName3);
228
229
230 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
231 htd.addFamily(new HColumnDescriptor("test"));
232 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
233 Coprocessor.PRIORITY_USER);
234 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
235 admin.createTable(htd);
236 waitForTable(htd.getTableName());
237
238
239 boolean found = false;
240 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
241 for (HRegion region:
242 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
243 if (region.getRegionNameAsString().startsWith(cpName3)) {
244 found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
245 }
246 }
247 assertTrue("Class " + cpName3 + " was missing on a region", found);
248 }
249
250 @Test
251
252 public void testPrivateClassLoader() throws Exception {
253 File jarFile = buildCoprocessorJar(cpName4);
254
255
256 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
257 htd.addFamily(new HColumnDescriptor("test"));
258 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
259 Coprocessor.PRIORITY_USER);
260 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
261 admin.createTable(htd);
262 waitForTable(htd.getTableName());
263
264
265 boolean found = false;
266 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
267 for (HRegion region:
268 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
269 if (region.getRegionNameAsString().startsWith(cpName4)) {
270 Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
271 if (cp != null) {
272 found = true;
273 assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
274 cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
275 }
276 }
277 }
278 assertTrue("Class " + cpName4 + " was missing on a region", found);
279 }
280
281 @Test
282
283
284 public void testHBase3810() throws Exception {
285
286
287 File jarFile1 = buildCoprocessorJar(cpName1);
288 File jarFile2 = buildCoprocessorJar(cpName2);
289 File jarFile5 = buildCoprocessorJar(cpName5);
290 File jarFile6 = buildCoprocessorJar(cpName6);
291
292 String cpKey1 = "COPROCESSOR$1";
293 String cpKey2 = " Coprocessor$2 ";
294 String cpKey3 = " coprocessor$03 ";
295
296 String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
297 Coprocessor.PRIORITY_USER;
298 String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
299
300 String cpValue3 =
301 " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
302
303
304 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
305 htd.addFamily(new HColumnDescriptor("test"));
306
307
308 htd.setValue(cpKey1, cpValue1);
309 htd.setValue(cpKey2, cpValue2);
310 htd.setValue(cpKey3, cpValue3);
311
312
313 htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
314 Coprocessor.PRIORITY_USER, null);
315 Map<String, String> kvs = new HashMap<String, String>();
316 kvs.put("k1", "v1");
317 kvs.put("k2", "v2");
318 kvs.put("k3", "v3");
319 htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
320 Coprocessor.PRIORITY_USER, kvs);
321
322 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
323 if (admin.tableExists(tableName)) {
324 if (admin.isTableEnabled(tableName)) {
325 admin.disableTable(tableName);
326 }
327 admin.deleteTable(tableName);
328 }
329 admin.createTable(htd);
330 waitForTable(htd.getTableName());
331
332
333 boolean found_2 = false, found_1 = false, found_3 = false,
334 found_5 = false, found_6 = false;
335 boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
336 found6_k4 = false;
337
338 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
339 for (HRegion region:
340 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
341 if (region.getRegionNameAsString().startsWith(tableName)) {
342 found_1 = found_1 ||
343 (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
344 found_2 = found_2 ||
345 (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
346 found_3 = found_3 ||
347 (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
348 != null);
349 found_5 = found_5 ||
350 (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
351
352 CoprocessorEnvironment env =
353 region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
354 if (env != null) {
355 found_6 = true;
356 Configuration conf = env.getConfiguration();
357 found6_k1 = conf.get("k1") != null;
358 found6_k2 = conf.get("k2") != null;
359 found6_k3 = conf.get("k3") != null;
360 }
361 }
362 }
363
364 assertTrue("Class " + cpName1 + " was missing on a region", found_1);
365 assertTrue("Class " + cpName2 + " was missing on a region", found_2);
366 assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
367 assertTrue("Class " + cpName5 + " was missing on a region", found_5);
368 assertTrue("Class " + cpName6 + " was missing on a region", found_6);
369
370 assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
371 assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
372 assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
373 assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
374 }
375
376 @Test
377 public void testClassLoadingFromLibDirInJar() throws Exception {
378 loadingClassFromLibDirInJar("/lib/");
379 }
380
381 @Test
382 public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
383 loadingClassFromLibDirInJar("lib/");
384 }
385
386 void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
387 FileSystem fs = cluster.getFileSystem();
388
389 File innerJarFile1 = buildCoprocessorJar(cpName1);
390 File innerJarFile2 = buildCoprocessorJar(cpName2);
391 File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
392
393 ClassLoaderTestHelper.addJarFilesToJar(
394 outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
395
396
397 fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
398 new Path(fs.getUri().toString() + Path.SEPARATOR));
399 String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
400 outerJarFile.getName();
401 assertTrue("Copy jar file to HDFS failed.",
402 fs.exists(new Path(jarFileOnHDFS)));
403 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
404
405
406 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
407 htd.addFamily(new HColumnDescriptor("test"));
408
409 htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
410 "|" + Coprocessor.PRIORITY_USER);
411
412 htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
413 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
414 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
415 if (admin.tableExists(tableName)) {
416 if (admin.isTableEnabled(tableName)) {
417 admin.disableTable(tableName);
418 }
419 admin.deleteTable(tableName);
420 }
421 admin.createTable(htd);
422 waitForTable(htd.getTableName());
423
424
425 boolean found1 = false, found2 = false, found2_k1 = false,
426 found2_k2 = false, found2_k3 = false;
427 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
428 for (HRegion region:
429 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
430 if (region.getRegionNameAsString().startsWith(tableName)) {
431 CoprocessorEnvironment env;
432 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
433 if (env != null) {
434 found1 = true;
435 }
436 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
437 if (env != null) {
438 found2 = true;
439 Configuration conf = env.getConfiguration();
440 found2_k1 = conf.get("k1") != null;
441 found2_k2 = conf.get("k2") != null;
442 found2_k3 = conf.get("k3") != null;
443 }
444 }
445 }
446 assertTrue("Class " + cpName1 + " was missing on a region", found1);
447 assertTrue("Class " + cpName2 + " was missing on a region", found2);
448 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
449 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
450 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
451 }
452
453 @Test
454 public void testRegionServerCoprocessorsReported() throws Exception {
455
456
457
458 assertAllRegionServers(null);
459 }
460
461
462
463
464
465
466
467
468
469
470 Map<ServerName, ServerLoad> serversForTable(String tableName) {
471 Map<ServerName, ServerLoad> serverLoadHashMap =
472 new HashMap<ServerName, ServerLoad>();
473 for(Map.Entry<ServerName,ServerLoad> server:
474 TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
475 getOnlineServers().entrySet()) {
476 for( Map.Entry<byte[], RegionLoad> region:
477 server.getValue().getRegionsLoad().entrySet()) {
478 if (region.getValue().getNameAsString().equals(tableName)) {
479
480 serverLoadHashMap.put(server.getKey(),server.getValue());
481
482 break;
483 }
484 }
485 }
486 return serverLoadHashMap;
487 }
488
489 void assertAllRegionServers(String tableName) throws InterruptedException {
490 Map<ServerName, ServerLoad> servers;
491 String[] actualCoprocessors = null;
492 boolean success = false;
493 String[] expectedCoprocessors = regionServerSystemCoprocessors;
494 if (tableName == null) {
495
496 servers = TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().getOnlineServers();
497 } else {
498 servers = serversForTable(tableName);
499 }
500 for (int i = 0; i < 5; i++) {
501 boolean any_failed = false;
502 for(Map.Entry<ServerName,ServerLoad> server: servers.entrySet()) {
503 actualCoprocessors = server.getValue().getRsCoprocessors();
504 if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
505 LOG.debug("failed comparison: actual: " +
506 Arrays.toString(actualCoprocessors) +
507 " ; expected: " + Arrays.toString(expectedCoprocessors));
508 any_failed = true;
509 expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
510 break;
511 }
512 expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
513 }
514 if (any_failed == false) {
515 success = true;
516 break;
517 }
518 LOG.debug("retrying after failed comparison: " + i);
519 Thread.sleep(1000);
520 }
521 assertTrue(success);
522 }
523
524 private String[] switchExpectedCoprocessors(String[] expectedCoprocessors) {
525 if (Arrays.equals(regionServerSystemCoprocessors, expectedCoprocessors)) {
526 expectedCoprocessors = masterRegionServerSystemCoprocessors;
527 } else {
528 expectedCoprocessors = regionServerSystemCoprocessors;
529 }
530 return expectedCoprocessors;
531 }
532
533 @Test
534 public void testMasterCoprocessorsReported() {
535
536
537
538 final String loadedMasterCoprocessorsVerify =
539 "[" + masterCoprocessor.getSimpleName() + "]";
540 String loadedMasterCoprocessors =
541 java.util.Arrays.toString(
542 TEST_UTIL.getHBaseCluster().getMaster().getCoprocessors());
543 assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
544 }
545
546 @Test
547 public void testFindCoprocessors() {
548
549 CoprocessorHost masterCpHost = TEST_UTIL.getHBaseCluster().getMaster().getCoprocessorHost();
550
551 List<MasterObserver> masterObservers = masterCpHost.findCoprocessors(MasterObserver.class);
552
553 assertTrue(masterObservers != null && masterObservers.size() > 0);
554 assertEquals(masterCoprocessor.getSimpleName(),
555 masterObservers.get(0).getClass().getSimpleName());
556 }
557
558 private void waitForTable(TableName name) throws InterruptedException, IOException {
559
560 TEST_UTIL.waitTableEnabled(name.getName());
561
562 Thread.sleep(1000);
563 }
564
565 }
566