1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.coprocessor;
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.hbase.*;
26 import org.apache.hadoop.hbase.client.HBaseAdmin;
27 import org.apache.hadoop.hbase.regionserver.HRegion;
28 import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
29 import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
30 import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
31 import org.apache.hadoop.hdfs.MiniDFSCluster;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.ServerLoad;
35 import org.apache.hadoop.hbase.RegionLoad;
36
37 import java.io.*;
38 import java.util.*;
39
40 import org.junit.*;
41 import org.junit.experimental.categories.Category;
42
43 import static org.junit.Assert.assertEquals;
44 import static org.junit.Assert.assertNotNull;
45 import static org.junit.Assert.assertTrue;
46 import static org.junit.Assert.assertFalse;
47
48
49
50
51 @Category(MediumTests.class)
52 public class TestClassLoading {
53 private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
54 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55
56 private static MiniDFSCluster cluster;
57
58 static final String tableName = "TestClassLoading";
59 static final String cpName1 = "TestCP1";
60 static final String cpName2 = "TestCP2";
61 static final String cpName3 = "TestCP3";
62 static final String cpName4 = "TestCP4";
63 static final String cpName5 = "TestCP5";
64 static final String cpName6 = "TestCP6";
65 static final String cpNameInvalid = "TestCPInvalid";
66
67 private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
68
69 private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
70 private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
71 private static Class<?> masterCoprocessor = BaseMasterObserver.class;
72
73 private static final String[] regionServerSystemCoprocessors =
74 new String[]{
75 regionServerCoprocessor.getSimpleName()
76 };
77
78 @BeforeClass
79 public static void setUpBeforeClass() throws Exception {
80 Configuration conf = TEST_UTIL.getConfiguration();
81
82
83
84 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
85 regionCoprocessor1.getName());
86
87
88
89
90 conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
91 regionCoprocessor2.getName());
92
93 conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
94 regionServerCoprocessor.getName());
95 conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
96 masterCoprocessor.getName());
97 TEST_UTIL.startMiniCluster(1);
98 cluster = TEST_UTIL.getDFSCluster();
99 }
100
101 @AfterClass
102 public static void tearDownAfterClass() throws Exception {
103 TEST_UTIL.shutdownMiniCluster();
104 }
105
106 static File buildCoprocessorJar(String className) throws Exception {
107 String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
108 "public class " + className + " extends BaseRegionObserver {}";
109 return ClassLoaderTestHelper.buildJar(
110 TEST_UTIL.getDataTestDir().toString(), className, code);
111 }
112
113 @Test
114
115 public void testClassLoadingFromHDFS() throws Exception {
116 FileSystem fs = cluster.getFileSystem();
117
118 File jarFile1 = buildCoprocessorJar(cpName1);
119 File jarFile2 = buildCoprocessorJar(cpName2);
120
121
122 fs.copyFromLocalFile(new Path(jarFile1.getPath()),
123 new Path(fs.getUri().toString() + Path.SEPARATOR));
124 String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
125 jarFile1.getName();
126 Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
127 assertTrue("Copy jar file to HDFS failed.",
128 fs.exists(pathOnHDFS1));
129 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
130
131 fs.copyFromLocalFile(new Path(jarFile2.getPath()),
132 new Path(fs.getUri().toString() + Path.SEPARATOR));
133 String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
134 jarFile2.getName();
135 Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
136 assertTrue("Copy jar file to HDFS failed.",
137 fs.exists(pathOnHDFS2));
138 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
139
140
141 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
142 htd.addFamily(new HColumnDescriptor("test"));
143
144 htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
145 "|" + Coprocessor.PRIORITY_USER);
146
147 htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
148 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
149
150 htd.setValue("COPROCESSOR$3", jarFileOnHDFS2.toString() + "|" + cpNameInvalid +
151 "|" + Coprocessor.PRIORITY_USER);
152 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
153 if (admin.tableExists(tableName)) {
154 if (admin.isTableEnabled(tableName)) {
155 admin.disableTable(tableName);
156 }
157 admin.deleteTable(tableName);
158 }
159 CoprocessorClassLoader.clearCache();
160 byte[] startKey = {10, 63};
161 byte[] endKey = {12, 43};
162 admin.createTable(htd, startKey, endKey, 4);
163 waitForTable(htd.getTableName());
164
165
166 boolean foundTableRegion=false;
167 boolean found_invalid = true, found1 = true, found2 = true, found2_k1 = true,
168 found2_k2 = true, found2_k3 = true;
169 Map<HRegion, Set<ClassLoader>> regionsActiveClassLoaders =
170 new HashMap<HRegion, Set<ClassLoader>>();
171 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
172 for (HRegion region:
173 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
174 if (region.getRegionNameAsString().startsWith(tableName)) {
175 foundTableRegion = true;
176 CoprocessorEnvironment env;
177 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
178 found1 = found1 && (env != null);
179 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
180 found2 = found2 && (env != null);
181 if (env != null) {
182 Configuration conf = env.getConfiguration();
183 found2_k1 = found2_k1 && (conf.get("k1") != null);
184 found2_k2 = found2_k2 && (conf.get("k2") != null);
185 found2_k3 = found2_k3 && (conf.get("k3") != null);
186 } else {
187 found2_k1 = found2_k2 = found2_k3 = false;
188 }
189 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpNameInvalid);
190 found_invalid = found_invalid && (env != null);
191
192 regionsActiveClassLoaders
193 .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
194 }
195 }
196
197 assertTrue("No region was found for table " + tableName, foundTableRegion);
198 assertTrue("Class " + cpName1 + " was missing on a region", found1);
199 assertTrue("Class " + cpName2 + " was missing on a region", found2);
200
201 assertFalse("Class " + cpNameInvalid + " was found on a region", found_invalid);
202 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
203 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
204 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
205
206 assertNotNull(jarFileOnHDFS1 + " was not cached",
207 CoprocessorClassLoader.getIfCached(pathOnHDFS1));
208 assertNotNull(jarFileOnHDFS2 + " was not cached",
209 CoprocessorClassLoader.getIfCached(pathOnHDFS2));
210
211 assertEquals("The number of cached classloaders should be equal to the number" +
212 " of external jar files",
213 2, CoprocessorClassLoader.getAllCached().size());
214
215 Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
216 CoprocessorClassLoader.getAllCached());
217 for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
218 assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
219 + " ClassLoader Cache:" + externalClassLoaders
220 + " Region ClassLoaders:" + regionCP.getValue(),
221 externalClassLoaders.containsAll(regionCP.getValue()));
222 }
223 }
224
225 private String getLocalPath(File file) {
226 return new Path(file.toURI()).toString();
227 }
228
229 @Test
230
231 public void testClassLoadingFromLocalFS() throws Exception {
232 File jarFile = buildCoprocessorJar(cpName3);
233
234
235 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
236 htd.addFamily(new HColumnDescriptor("test"));
237 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
238 Coprocessor.PRIORITY_USER);
239 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
240 admin.createTable(htd);
241 waitForTable(htd.getTableName());
242
243
244 boolean found = false;
245 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
246 for (HRegion region:
247 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
248 if (region.getRegionNameAsString().startsWith(cpName3)) {
249 found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
250 }
251 }
252 assertTrue("Class " + cpName3 + " was missing on a region", found);
253 }
254
255 @Test
256
257 public void testPrivateClassLoader() throws Exception {
258 File jarFile = buildCoprocessorJar(cpName4);
259
260
261 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
262 htd.addFamily(new HColumnDescriptor("test"));
263 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
264 Coprocessor.PRIORITY_USER);
265 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
266 admin.createTable(htd);
267 waitForTable(htd.getTableName());
268
269
270 boolean found = false;
271 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
272 for (HRegion region:
273 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
274 if (region.getRegionNameAsString().startsWith(cpName4)) {
275 Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
276 if (cp != null) {
277 found = true;
278 assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
279 cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
280 }
281 }
282 }
283 assertTrue("Class " + cpName4 + " was missing on a region", found);
284 }
285
286 @Test
287
288
289 public void testHBase3810() throws Exception {
290
291
292 File jarFile1 = buildCoprocessorJar(cpName1);
293 File jarFile2 = buildCoprocessorJar(cpName2);
294 File jarFile5 = buildCoprocessorJar(cpName5);
295 File jarFile6 = buildCoprocessorJar(cpName6);
296
297 String cpKey1 = "COPROCESSOR$1";
298 String cpKey2 = " Coprocessor$2 ";
299 String cpKey3 = " coprocessor$03 ";
300
301 String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
302 Coprocessor.PRIORITY_USER;
303 String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
304
305 String cpValue3 =
306 " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
307
308
309 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
310 htd.addFamily(new HColumnDescriptor("test"));
311
312
313 htd.setValue(cpKey1, cpValue1);
314 htd.setValue(cpKey2, cpValue2);
315 htd.setValue(cpKey3, cpValue3);
316
317
318 htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
319 Coprocessor.PRIORITY_USER, null);
320 Map<String, String> kvs = new HashMap<String, String>();
321 kvs.put("k1", "v1");
322 kvs.put("k2", "v2");
323 kvs.put("k3", "v3");
324 htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
325 Coprocessor.PRIORITY_USER, kvs);
326
327 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
328 if (admin.tableExists(tableName)) {
329 if (admin.isTableEnabled(tableName)) {
330 admin.disableTable(tableName);
331 }
332 admin.deleteTable(tableName);
333 }
334 admin.createTable(htd);
335 waitForTable(htd.getTableName());
336
337
338 boolean found_2 = false, found_1 = false, found_3 = false,
339 found_5 = false, found_6 = false;
340 boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
341 found6_k4 = false;
342
343 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
344 for (HRegion region:
345 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
346 if (region.getRegionNameAsString().startsWith(tableName)) {
347 found_1 = found_1 ||
348 (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
349 found_2 = found_2 ||
350 (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
351 found_3 = found_3 ||
352 (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
353 != null);
354 found_5 = found_5 ||
355 (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
356
357 CoprocessorEnvironment env =
358 region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
359 if (env != null) {
360 found_6 = true;
361 Configuration conf = env.getConfiguration();
362 found6_k1 = conf.get("k1") != null;
363 found6_k2 = conf.get("k2") != null;
364 found6_k3 = conf.get("k3") != null;
365 }
366 }
367 }
368
369 assertTrue("Class " + cpName1 + " was missing on a region", found_1);
370 assertTrue("Class " + cpName2 + " was missing on a region", found_2);
371 assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
372 assertTrue("Class " + cpName5 + " was missing on a region", found_5);
373 assertTrue("Class " + cpName6 + " was missing on a region", found_6);
374
375 assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
376 assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
377 assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
378 assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
379 }
380
381 @Test
382 public void testClassLoadingFromLibDirInJar() throws Exception {
383 loadingClassFromLibDirInJar("/lib/");
384 }
385
386 @Test
387 public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
388 loadingClassFromLibDirInJar("lib/");
389 }
390
391 void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
392 FileSystem fs = cluster.getFileSystem();
393
394 File innerJarFile1 = buildCoprocessorJar(cpName1);
395 File innerJarFile2 = buildCoprocessorJar(cpName2);
396 File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
397
398 ClassLoaderTestHelper.addJarFilesToJar(
399 outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
400
401
402 fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
403 new Path(fs.getUri().toString() + Path.SEPARATOR));
404 String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
405 outerJarFile.getName();
406 assertTrue("Copy jar file to HDFS failed.",
407 fs.exists(new Path(jarFileOnHDFS)));
408 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
409
410
411 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
412 htd.addFamily(new HColumnDescriptor("test"));
413
414 htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
415 "|" + Coprocessor.PRIORITY_USER);
416
417 htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
418 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
419 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
420 if (admin.tableExists(tableName)) {
421 if (admin.isTableEnabled(tableName)) {
422 admin.disableTable(tableName);
423 }
424 admin.deleteTable(tableName);
425 }
426 admin.createTable(htd);
427 waitForTable(htd.getTableName());
428
429
430 boolean found1 = false, found2 = false, found2_k1 = false,
431 found2_k2 = false, found2_k3 = false;
432 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
433 for (HRegion region:
434 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
435 if (region.getRegionNameAsString().startsWith(tableName)) {
436 CoprocessorEnvironment env;
437 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
438 if (env != null) {
439 found1 = true;
440 }
441 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
442 if (env != null) {
443 found2 = true;
444 Configuration conf = env.getConfiguration();
445 found2_k1 = conf.get("k1") != null;
446 found2_k2 = conf.get("k2") != null;
447 found2_k3 = conf.get("k3") != null;
448 }
449 }
450 }
451 assertTrue("Class " + cpName1 + " was missing on a region", found1);
452 assertTrue("Class " + cpName2 + " was missing on a region", found2);
453 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
454 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
455 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
456 }
457
458 @Test
459 public void testRegionServerCoprocessorsReported() throws Exception {
460
461
462
463
464 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
465 assertAllRegionServers(regionServerSystemCoprocessors,null);
466 }
467
468
469
470
471
472
473
474
475
476
477 Map<ServerName, ServerLoad> serversForTable(String tableName) {
478 Map<ServerName, ServerLoad> serverLoadHashMap =
479 new HashMap<ServerName, ServerLoad>();
480 for(Map.Entry<ServerName,ServerLoad> server:
481 TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
482 getOnlineServers().entrySet()) {
483 for( Map.Entry<byte[], RegionLoad> region:
484 server.getValue().getRegionsLoad().entrySet()) {
485 if (region.getValue().getNameAsString().equals(tableName)) {
486
487 serverLoadHashMap.put(server.getKey(),server.getValue());
488
489 break;
490 }
491 }
492 }
493 return serverLoadHashMap;
494 }
495
496 void assertAllRegionServers(String[] expectedCoprocessors, String tableName)
497 throws InterruptedException {
498 Map<ServerName, ServerLoad> servers;
499 String[] actualCoprocessors = null;
500 boolean success = false;
501 for(int i = 0; i < 5; i++) {
502 if (tableName == null) {
503
504 servers =
505 TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
506 getOnlineServers();
507 } else {
508 servers = serversForTable(tableName);
509 }
510 boolean any_failed = false;
511 for(Map.Entry<ServerName,ServerLoad> server: servers.entrySet()) {
512 actualCoprocessors = server.getValue().getRsCoprocessors();
513 if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
514 LOG.debug("failed comparison: actual: " +
515 Arrays.toString(actualCoprocessors) +
516 " ; expected: " + Arrays.toString(expectedCoprocessors));
517 any_failed = true;
518 break;
519 }
520 }
521 if (any_failed == false) {
522 success = true;
523 break;
524 }
525 LOG.debug("retrying after failed comparison: " + i);
526 Thread.sleep(1000);
527 }
528 assertTrue(success);
529 }
530
531 @Test
532 public void testMasterCoprocessorsReported() {
533
534
535
536 final String loadedMasterCoprocessorsVerify =
537 "[" + masterCoprocessor.getSimpleName() + "]";
538 String loadedMasterCoprocessors =
539 java.util.Arrays.toString(
540 TEST_UTIL.getHBaseCluster().getMaster().getCoprocessors());
541 assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
542 }
543
544 private void waitForTable(TableName name) throws InterruptedException, IOException {
545
546 TEST_UTIL.waitTableEnabled(name.getName());
547
548 Thread.sleep(1000);
549 }
550
551 }
552