1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.coprocessor;
21
22 import org.apache.commons.logging.Log;
23 import org.apache.commons.logging.LogFactory;
24
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.hbase.*;
27 import org.apache.hadoop.hbase.client.HBaseAdmin;
28 import org.apache.hadoop.hbase.regionserver.HRegion;
29
30 import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
31 import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
32 import org.apache.hadoop.hdfs.MiniDFSCluster;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35
36 import java.io.*;
37 import java.util.*;
38 import java.util.jar.*;
39
40 import org.junit.*;
41 import org.junit.experimental.categories.Category;
42
43 import static org.junit.Assert.assertEquals;
44 import static org.junit.Assert.assertNotNull;
45 import static org.junit.Assert.assertTrue;
46 import static org.junit.Assert.assertFalse;
47
48
49
50
51 @Category(MediumTests.class)
52 public class TestClassLoading {
53 private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
54 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55
56 private static MiniDFSCluster cluster;
57
58 static final int BUFFER_SIZE = 4096;
59 static final String tableName = "TestClassLoading";
60 static final String cpName1 = "TestCP1";
61 static final String cpName2 = "TestCP2";
62 static final String cpName3 = "TestCP3";
63 static final String cpName4 = "TestCP4";
64 static final String cpName5 = "TestCP5";
65 static final String cpName6 = "TestCP6";
66 static final String cpNameInvalid = "TestCPInvalid";
67
68 private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
69 private static Class<?> regionCoprocessor2 = GenericEndpoint.class;
70 private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
71 private static Class<?> masterCoprocessor = BaseMasterObserver.class;
72
73 private static final String[] regionServerSystemCoprocessors =
74 new String[]{
75 regionServerCoprocessor.getSimpleName()
76 };
77
78 @BeforeClass
79 public static void setUpBeforeClass() throws Exception {
80 Configuration conf = TEST_UTIL.getConfiguration();
81
82
83
84 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
85 regionCoprocessor1.getName());
86
87
88
89
90 conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
91 regionCoprocessor2.getName());
92
93 conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
94 regionServerCoprocessor.getName());
95 conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
96 masterCoprocessor.getName());
97 TEST_UTIL.startMiniCluster(1);
98 cluster = TEST_UTIL.getDFSCluster();
99 }
100
101 @AfterClass
102 public static void tearDownAfterClass() throws Exception {
103 TEST_UTIL.shutdownMiniCluster();
104 }
105
106 static File buildCoprocessorJar(String className) throws Exception {
107 String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
108 "public class " + className + " extends BaseRegionObserver {}";
109 return ClassLoaderTestHelper.buildJar(
110 TEST_UTIL.getDataTestDir().toString(), className, code);
111 }
112
113 @Test
114
115 public void testClassLoadingFromHDFS() throws Exception {
116 FileSystem fs = cluster.getFileSystem();
117
118 File jarFile1 = buildCoprocessorJar(cpName1);
119 File jarFile2 = buildCoprocessorJar(cpName2);
120
121
122 fs.copyFromLocalFile(new Path(jarFile1.getPath()),
123 new Path(fs.getUri().toString() + Path.SEPARATOR));
124 String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
125 jarFile1.getName();
126 Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
127 assertTrue("Copy jar file to HDFS failed.",
128 fs.exists(pathOnHDFS1));
129 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
130
131 fs.copyFromLocalFile(new Path(jarFile2.getPath()),
132 new Path(fs.getUri().toString() + Path.SEPARATOR));
133 String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
134 jarFile2.getName();
135 Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
136 assertTrue("Copy jar file to HDFS failed.",
137 fs.exists(pathOnHDFS2));
138 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
139
140
141 HTableDescriptor htd = new HTableDescriptor(tableName);
142 htd.addFamily(new HColumnDescriptor("test"));
143
144 htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
145 "|" + Coprocessor.PRIORITY_USER);
146
147 htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
148 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
149
150 htd.setValue("COPROCESSOR$3", jarFileOnHDFS2.toString() + "|" + cpNameInvalid +
151 "|" + Coprocessor.PRIORITY_USER);
152 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
153 if (admin.tableExists(tableName)) {
154 admin.disableTable(tableName);
155 admin.deleteTable(tableName);
156 }
157 CoprocessorClassLoader.clearCache();
158 byte[] startKey = {10, 63};
159 byte[] endKey = {12, 43};
160 admin.createTable(htd, startKey, endKey, 4);
161 waitForTable(htd.getName());
162
163
164 boolean foundTableRegion=false;
165 boolean found_invalid = true, found1 = true, found2 = true, found2_k1 = true,
166 found2_k2 = true, found2_k3 = true;
167 Map<HRegion, Set<ClassLoader>> regionsActiveClassLoaders =
168 new HashMap<HRegion, Set<ClassLoader>>();
169 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
170 for (HRegion region:
171 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
172 if (region.getRegionNameAsString().startsWith(tableName)) {
173 foundTableRegion = true;
174 CoprocessorEnvironment env;
175 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
176 found1 = found1 && (env != null);
177 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
178 found2 = found2 && (env != null);
179 if (env != null) {
180 Configuration conf = env.getConfiguration();
181 found2_k1 = found2_k1 && (conf.get("k1") != null);
182 found2_k2 = found2_k2 && (conf.get("k2") != null);
183 found2_k3 = found2_k3 && (conf.get("k3") != null);
184 } else {
185 found2_k1 = found2_k2 = found2_k3 = false;
186 }
187 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpNameInvalid);
188 found_invalid = found_invalid && (env != null);
189
190 regionsActiveClassLoaders
191 .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
192 }
193 }
194
195 assertTrue("No region was found for table " + tableName, foundTableRegion);
196 assertTrue("Class " + cpName1 + " was missing on a region", found1);
197 assertTrue("Class " + cpName2 + " was missing on a region", found2);
198
199 assertFalse("Class " + cpNameInvalid + " was found on a region", found_invalid);
200 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
201 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
202 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
203
204 assertNotNull(jarFileOnHDFS1 + " was not cached",
205 CoprocessorClassLoader.getIfCached(pathOnHDFS1));
206 assertNotNull(jarFileOnHDFS2 + " was not cached",
207 CoprocessorClassLoader.getIfCached(pathOnHDFS2));
208
209 assertEquals("The number of cached classloaders should be equal to the number" +
210 " of external jar files",
211 2, CoprocessorClassLoader.getAllCached().size());
212
213 Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
214 CoprocessorClassLoader.getAllCached());
215 for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
216 assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
217 + " ClassLoader Cache:" + externalClassLoaders
218 + " Region ClassLoaders:" + regionCP.getValue(),
219 externalClassLoaders.containsAll(regionCP.getValue()));
220 }
221 }
222
223 private String getLocalPath(File file) {
224 return new Path(file.toURI()).toString();
225 }
226
227 @Test
228
229 public void testClassLoadingFromLocalFS() throws Exception {
230 File jarFile = buildCoprocessorJar(cpName3);
231
232
233 HTableDescriptor htd = new HTableDescriptor(cpName3);
234 htd.addFamily(new HColumnDescriptor("test"));
235 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
236 Coprocessor.PRIORITY_USER);
237 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
238 admin.createTable(htd);
239 waitForTable(htd.getName());
240
241
242 boolean found = false;
243 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
244 for (HRegion region:
245 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
246 if (region.getRegionNameAsString().startsWith(cpName3)) {
247 found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
248 }
249 }
250 assertTrue("Class " + cpName3 + " was missing on a region", found);
251 }
252
253 @Test
254
255 public void testPrivateClassLoader() throws Exception {
256 File jarFile = buildCoprocessorJar(cpName4);
257
258
259 HTableDescriptor htd = new HTableDescriptor(cpName4);
260 htd.addFamily(new HColumnDescriptor("test"));
261 htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
262 Coprocessor.PRIORITY_USER);
263 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
264 admin.createTable(htd);
265 waitForTable(htd.getName());
266
267
268 boolean found = false;
269 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
270 for (HRegion region:
271 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
272 if (region.getRegionNameAsString().startsWith(cpName4)) {
273 Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
274 if (cp != null) {
275 found = true;
276 assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
277 cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
278 }
279 }
280 }
281 assertTrue("Class " + cpName4 + " was missing on a region", found);
282 }
283
284 @Test
285
286
287 public void testHBase3810() throws Exception {
288
289
290 File jarFile1 = buildCoprocessorJar(cpName1);
291 File jarFile2 = buildCoprocessorJar(cpName2);
292 File jarFile5 = buildCoprocessorJar(cpName5);
293 File jarFile6 = buildCoprocessorJar(cpName6);
294
295 String cpKey1 = "COPROCESSOR$1";
296 String cpKey2 = " Coprocessor$2 ";
297 String cpKey3 = " coprocessor$03 ";
298
299 String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
300 Coprocessor.PRIORITY_USER;
301 String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
302
303 String cpValue3 =
304 " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
305
306
307 HTableDescriptor htd = new HTableDescriptor(tableName);
308 htd.addFamily(new HColumnDescriptor("test"));
309
310
311 htd.setValue(cpKey1, cpValue1);
312 htd.setValue(cpKey2, cpValue2);
313 htd.setValue(cpKey3, cpValue3);
314
315
316 htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
317 Coprocessor.PRIORITY_USER, null);
318 Map<String, String> kvs = new HashMap<String, String>();
319 kvs.put("k1", "v1");
320 kvs.put("k2", "v2");
321 kvs.put("k3", "v3");
322 htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
323 Coprocessor.PRIORITY_USER, kvs);
324
325 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
326 if (admin.tableExists(tableName)) {
327 admin.disableTable(tableName);
328 admin.deleteTable(tableName);
329 }
330 admin.createTable(htd);
331 waitForTable(htd.getName());
332
333
334 boolean found_2 = false, found_1 = false, found_3 = false,
335 found_5 = false, found_6 = false;
336 boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
337 found6_k4 = false;
338
339 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
340 for (HRegion region:
341 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
342 if (region.getRegionNameAsString().startsWith(tableName)) {
343 found_1 = found_1 ||
344 (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
345 found_2 = found_2 ||
346 (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
347 found_3 = found_3 ||
348 (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
349 != null);
350 found_5 = found_5 ||
351 (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
352
353 CoprocessorEnvironment env =
354 region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
355 if (env != null) {
356 found_6 = true;
357 Configuration conf = env.getConfiguration();
358 found6_k1 = conf.get("k1") != null;
359 found6_k2 = conf.get("k2") != null;
360 found6_k3 = conf.get("k3") != null;
361 }
362 }
363 }
364
365 assertTrue("Class " + cpName1 + " was missing on a region", found_1);
366 assertTrue("Class " + cpName2 + " was missing on a region", found_2);
367 assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
368 assertTrue("Class " + cpName5 + " was missing on a region", found_5);
369 assertTrue("Class " + cpName6 + " was missing on a region", found_6);
370
371 assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
372 assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
373 assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
374 assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
375 }
376
377 @Test
378 public void testClassLoadingFromLibDirInJar() throws Exception {
379 loadingClassFromLibDirInJar("/lib/");
380 }
381
382 @Test
383 public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
384 loadingClassFromLibDirInJar("lib/");
385 }
386
387 void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
388 FileSystem fs = cluster.getFileSystem();
389
390 File innerJarFile1 = buildCoprocessorJar(cpName1);
391 File innerJarFile2 = buildCoprocessorJar(cpName2);
392 File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
393
394 byte buffer[] = new byte[BUFFER_SIZE];
395
396
397
398 FileOutputStream stream = new FileOutputStream(outerJarFile);
399 JarOutputStream out = new JarOutputStream(stream, new Manifest());
400
401 for (File jarFile: new File[] { innerJarFile1, innerJarFile2 }) {
402
403 JarEntry jarAdd = new JarEntry(libPrefix + jarFile.getName());
404 jarAdd.setTime(jarFile.lastModified());
405 out.putNextEntry(jarAdd);
406
407
408 FileInputStream in = new FileInputStream(jarFile);
409 while (true) {
410 int nRead = in.read(buffer, 0, buffer.length);
411 if (nRead <= 0)
412 break;
413 out.write(buffer, 0, nRead);
414 }
415 in.close();
416 }
417 out.close();
418 stream.close();
419 LOG.info("Adding jar file to outer jar file completed");
420
421
422 fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
423 new Path(fs.getUri().toString() + Path.SEPARATOR));
424 String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
425 outerJarFile.getName();
426 assertTrue("Copy jar file to HDFS failed.",
427 fs.exists(new Path(jarFileOnHDFS)));
428 LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
429
430
431 HTableDescriptor htd = new HTableDescriptor(tableName);
432 htd.addFamily(new HColumnDescriptor("test"));
433
434 htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
435 "|" + Coprocessor.PRIORITY_USER);
436
437 htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
438 "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
439 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
440 if (admin.tableExists(tableName)) {
441 admin.disableTable(tableName);
442 admin.deleteTable(tableName);
443 }
444 admin.createTable(htd);
445 waitForTable(htd.getName());
446
447
448 boolean found1 = false, found2 = false, found2_k1 = false,
449 found2_k2 = false, found2_k3 = false;
450 MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
451 for (HRegion region:
452 hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
453 if (region.getRegionNameAsString().startsWith(tableName)) {
454 CoprocessorEnvironment env;
455 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
456 if (env != null) {
457 found1 = true;
458 }
459 env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
460 if (env != null) {
461 found2 = true;
462 Configuration conf = env.getConfiguration();
463 found2_k1 = conf.get("k1") != null;
464 found2_k2 = conf.get("k2") != null;
465 found2_k3 = conf.get("k3") != null;
466 }
467 }
468 }
469 assertTrue("Class " + cpName1 + " was missing on a region", found1);
470 assertTrue("Class " + cpName2 + " was missing on a region", found2);
471 assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
472 assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
473 assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
474 }
475
476 @Test
477 public void testRegionServerCoprocessorsReported() throws Exception {
478
479
480
481
482 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
483 assertAllRegionServers(regionServerSystemCoprocessors,null);
484 }
485
486
487
488
489
490
491
492
493
494
495 Map<ServerName, HServerLoad> serversForTable(String tableName) {
496 Map<ServerName, HServerLoad> serverLoadHashMap =
497 new HashMap<ServerName, HServerLoad>();
498 for(Map.Entry<ServerName,HServerLoad> server:
499 TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
500 getOnlineServers().entrySet()) {
501 for(Map.Entry<byte[], HServerLoad.RegionLoad> region:
502 server.getValue().getRegionsLoad().entrySet()) {
503 if (region.getValue().getNameAsString().equals(tableName)) {
504
505 serverLoadHashMap.put(server.getKey(),server.getValue());
506
507 break;
508 }
509 }
510 }
511 return serverLoadHashMap;
512 }
513
514 void assertAllRegionServers(String[] expectedCoprocessors, String tableName)
515 throws InterruptedException {
516 Map<ServerName, HServerLoad> servers;
517 String[] actualCoprocessors = null;
518 boolean success = false;
519 for(int i = 0; i < 5; i++) {
520 if (tableName == null) {
521
522 servers =
523 TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
524 getOnlineServers();
525 } else {
526 servers = serversForTable(tableName);
527 }
528 boolean any_failed = false;
529 for(Map.Entry<ServerName,HServerLoad> server: servers.entrySet()) {
530 actualCoprocessors = server.getValue().getRsCoprocessors();
531 if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
532 LOG.debug("failed comparison: actual: " +
533 Arrays.toString(actualCoprocessors) +
534 " ; expected: " + Arrays.toString(expectedCoprocessors));
535 any_failed = true;
536 break;
537 }
538 }
539 if (any_failed == false) {
540 success = true;
541 break;
542 }
543 LOG.debug("retrying after failed comparison: " + i);
544 Thread.sleep(1000);
545 }
546 assertTrue(success);
547 }
548
549 @Test
550 public void testMasterCoprocessorsReported() {
551
552
553
554 final String loadedMasterCoprocessorsVerify =
555 "[" + masterCoprocessor.getSimpleName() + "]";
556 String loadedMasterCoprocessors =
557 java.util.Arrays.toString(
558 TEST_UTIL.getHBaseCluster().getMaster().getCoprocessors());
559 assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
560 }
561
562 private void waitForTable(byte[] name) throws InterruptedException, IOException {
563
564 TEST_UTIL.waitTableEnabled(name, 5000);
565
566 Thread.sleep(1000);
567 }
568
569 @org.junit.Rule
570 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
571 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
572 }
573