View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.coprocessor;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.*;
26  import org.apache.hadoop.hbase.client.HBaseAdmin;
27  import org.apache.hadoop.hbase.regionserver.HRegion;
28  import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
29  import org.apache.hadoop.hbase.testclassification.MediumTests;
30  import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
31  import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
32  import org.apache.hadoop.hdfs.MiniDFSCluster;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.ServerLoad;
36  import org.apache.hadoop.hbase.RegionLoad;
37  
38  import java.io.*;
39  import java.util.*;
40  
41  import org.junit.*;
42  import org.junit.experimental.categories.Category;
43  
44  import static org.junit.Assert.assertEquals;
45  import static org.junit.Assert.assertNotNull;
46  import static org.junit.Assert.assertTrue;
47  import static org.junit.Assert.assertFalse;
48  
49  /**
50   * Test coprocessors class loading.
51   */
52  @Category(MediumTests.class)
53  public class TestClassLoading {
54    private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
55    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56  
57    private static MiniDFSCluster cluster;
58  
59    static final String tableName = "TestClassLoading";
60    static final String cpName1 = "TestCP1";
61    static final String cpName2 = "TestCP2";
62    static final String cpName3 = "TestCP3";
63    static final String cpName4 = "TestCP4";
64    static final String cpName5 = "TestCP5";
65    static final String cpName6 = "TestCP6";
66  
67    private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
68    // TOOD: Fix the import of this handler.  It is coming in from a package that is far away.
69    private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
70    private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
71    private static Class<?> masterCoprocessor = BaseMasterObserver.class;
72  
73    private static final String[] regionServerSystemCoprocessors =
74        new String[]{
75        regionServerCoprocessor.getSimpleName()
76    };
77  
78    private static final String[] masterRegionServerSystemCoprocessors = new String[] {
79        regionCoprocessor1.getSimpleName(), MultiRowMutationEndpoint.class.getSimpleName(),
80        regionServerCoprocessor.getSimpleName() };
81  
82    @BeforeClass
83    public static void setUpBeforeClass() throws Exception {
84      Configuration conf = TEST_UTIL.getConfiguration();
85  
86      // regionCoprocessor1 will be loaded on all regionservers, since it is
87      // loaded for any tables (user or meta).
88      conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
89          regionCoprocessor1.getName());
90  
91      // regionCoprocessor2 will be loaded only on regionservers that serve a
92      // user table region. Therefore, if there are no user tables loaded,
93      // this coprocessor will not be loaded on any regionserver.
94      conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
95          regionCoprocessor2.getName());
96  
97      conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
98          regionServerCoprocessor.getName());
99      conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
100         masterCoprocessor.getName());
101     TEST_UTIL.startMiniCluster(1);
102     cluster = TEST_UTIL.getDFSCluster();
103   }
104 
105   @AfterClass
106   public static void tearDownAfterClass() throws Exception {
107     TEST_UTIL.shutdownMiniCluster();
108   }
109 
110   static File buildCoprocessorJar(String className) throws Exception {
111     String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
112       "public class " + className + " extends BaseRegionObserver {}";
113     return ClassLoaderTestHelper.buildJar(
114       TEST_UTIL.getDataTestDir().toString(), className, code);
115   }
116 
117   @Test
118   // HBASE-3516: Test CP Class loading from HDFS
119   public void testClassLoadingFromHDFS() throws Exception {
120     FileSystem fs = cluster.getFileSystem();
121 
122     File jarFile1 = buildCoprocessorJar(cpName1);
123     File jarFile2 = buildCoprocessorJar(cpName2);
124 
125     // copy the jars into dfs
126     fs.copyFromLocalFile(new Path(jarFile1.getPath()),
127       new Path(fs.getUri().toString() + Path.SEPARATOR));
128     String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
129       jarFile1.getName();
130     Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
131     assertTrue("Copy jar file to HDFS failed.",
132       fs.exists(pathOnHDFS1));
133     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
134 
135     fs.copyFromLocalFile(new Path(jarFile2.getPath()),
136         new Path(fs.getUri().toString() + Path.SEPARATOR));
137     String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
138       jarFile2.getName();
139     Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
140     assertTrue("Copy jar file to HDFS failed.",
141       fs.exists(pathOnHDFS2));
142     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
143 
144     // create a table that references the coprocessors
145     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
146     htd.addFamily(new HColumnDescriptor("test"));
147       // without configuration values
148     htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
149       "|" + Coprocessor.PRIORITY_USER);
150       // with configuration values
151     htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
152       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
153     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
154     if (admin.tableExists(tableName)) {
155       if (admin.isTableEnabled(tableName)) {
156         admin.disableTable(tableName);
157       }
158       admin.deleteTable(tableName);
159     }
160     CoprocessorClassLoader.clearCache();
161     byte[] startKey = {10, 63};
162     byte[] endKey = {12, 43};
163     admin.createTable(htd, startKey, endKey, 4);
164     waitForTable(htd.getTableName());
165 
166     // verify that the coprocessors were loaded
167     boolean foundTableRegion=false;
168     boolean found1 = true, found2 = true, found2_k1 = true, found2_k2 = true, found2_k3 = true;
169     Map<HRegion, Set<ClassLoader>> regionsActiveClassLoaders =
170         new HashMap<HRegion, Set<ClassLoader>>();
171     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
172     for (HRegion region:
173         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
174       if (region.getRegionNameAsString().startsWith(tableName)) {
175         foundTableRegion = true;
176         CoprocessorEnvironment env;
177         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
178         found1 = found1 && (env != null);
179         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
180         found2 = found2 && (env != null);
181         if (env != null) {
182           Configuration conf = env.getConfiguration();
183           found2_k1 = found2_k1 && (conf.get("k1") != null);
184           found2_k2 = found2_k2 && (conf.get("k2") != null);
185           found2_k3 = found2_k3 && (conf.get("k3") != null);
186         } else {
187           found2_k1 = found2_k2 = found2_k3 = false;
188         }
189         regionsActiveClassLoaders
190             .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
191       }
192     }
193 
194     assertTrue("No region was found for table " + tableName, foundTableRegion);
195     assertTrue("Class " + cpName1 + " was missing on a region", found1);
196     assertTrue("Class " + cpName2 + " was missing on a region", found2);
197     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
198     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
199     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
200     // check if CP classloaders are cached
201     assertNotNull(jarFileOnHDFS1 + " was not cached",
202       CoprocessorClassLoader.getIfCached(pathOnHDFS1));
203     assertNotNull(jarFileOnHDFS2 + " was not cached",
204       CoprocessorClassLoader.getIfCached(pathOnHDFS2));
205     //two external jar used, should be one classloader per jar
206     assertEquals("The number of cached classloaders should be equal to the number" +
207       " of external jar files",
208       2, CoprocessorClassLoader.getAllCached().size());
209     //check if region active classloaders are shared across all RS regions
210     Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
211       CoprocessorClassLoader.getAllCached());
212     for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
213       assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
214         + " ClassLoader Cache:" + externalClassLoaders
215         + " Region ClassLoaders:" + regionCP.getValue(),
216         externalClassLoaders.containsAll(regionCP.getValue()));
217     }
218   }
219 
220   private String getLocalPath(File file) {
221     return new Path(file.toURI()).toString();
222   }
223 
224   @Test
225   // HBASE-3516: Test CP Class loading from local file system
226   public void testClassLoadingFromLocalFS() throws Exception {
227     File jarFile = buildCoprocessorJar(cpName3);
228 
229     // create a table that references the jar
230     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
231     htd.addFamily(new HColumnDescriptor("test"));
232     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
233       Coprocessor.PRIORITY_USER);
234     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
235     admin.createTable(htd);
236     waitForTable(htd.getTableName());
237 
238     // verify that the coprocessor was loaded
239     boolean found = false;
240     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
241     for (HRegion region:
242         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
243       if (region.getRegionNameAsString().startsWith(cpName3)) {
244         found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
245       }
246     }
247     assertTrue("Class " + cpName3 + " was missing on a region", found);
248   }
249 
250   @Test
251   // HBASE-6308: Test CP classloader is the CoprocessorClassLoader
252   public void testPrivateClassLoader() throws Exception {
253     File jarFile = buildCoprocessorJar(cpName4);
254 
255     // create a table that references the jar
256     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
257     htd.addFamily(new HColumnDescriptor("test"));
258     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
259       Coprocessor.PRIORITY_USER);
260     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
261     admin.createTable(htd);
262     waitForTable(htd.getTableName());
263 
264     // verify that the coprocessor was loaded correctly
265     boolean found = false;
266     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
267     for (HRegion region:
268         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
269       if (region.getRegionNameAsString().startsWith(cpName4)) {
270         Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
271         if (cp != null) {
272           found = true;
273           assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
274             cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
275         }
276       }
277     }
278     assertTrue("Class " + cpName4 + " was missing on a region", found);
279   }
280 
281   @Test
282   // HBase-3810: Registering a Coprocessor at HTableDescriptor should be
283   // less strict
284   public void testHBase3810() throws Exception {
285     // allowed value pattern: [path] | class name | [priority] | [key values]
286 
287     File jarFile1 = buildCoprocessorJar(cpName1);
288     File jarFile2 = buildCoprocessorJar(cpName2);
289     File jarFile5 = buildCoprocessorJar(cpName5);
290     File jarFile6 = buildCoprocessorJar(cpName6);
291 
292     String cpKey1 = "COPROCESSOR$1";
293     String cpKey2 = " Coprocessor$2 ";
294     String cpKey3 = " coprocessor$03 ";
295 
296     String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
297         Coprocessor.PRIORITY_USER;
298     String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
299     // load from default class loader
300     String cpValue3 =
301         " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
302 
303     // create a table that references the jar
304     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
305     htd.addFamily(new HColumnDescriptor("test"));
306 
307     // add 3 coprocessors by setting htd attributes directly.
308     htd.setValue(cpKey1, cpValue1);
309     htd.setValue(cpKey2, cpValue2);
310     htd.setValue(cpKey3, cpValue3);
311 
312     // add 2 coprocessor by using new htd.addCoprocessor() api
313     htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
314         Coprocessor.PRIORITY_USER, null);
315     Map<String, String> kvs = new HashMap<String, String>();
316     kvs.put("k1", "v1");
317     kvs.put("k2", "v2");
318     kvs.put("k3", "v3");
319     htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
320         Coprocessor.PRIORITY_USER, kvs);
321 
322     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
323     if (admin.tableExists(tableName)) {
324       if (admin.isTableEnabled(tableName)) {
325         admin.disableTable(tableName);
326       }
327       admin.deleteTable(tableName);
328     }
329     admin.createTable(htd);
330     waitForTable(htd.getTableName());
331 
332     // verify that the coprocessor was loaded
333     boolean found_2 = false, found_1 = false, found_3 = false,
334         found_5 = false, found_6 = false;
335     boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
336         found6_k4 = false;
337 
338     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
339     for (HRegion region:
340         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
341       if (region.getRegionNameAsString().startsWith(tableName)) {
342         found_1 = found_1 ||
343             (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
344         found_2 = found_2 ||
345             (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
346         found_3 = found_3 ||
347             (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
348                 != null);
349         found_5 = found_5 ||
350             (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
351 
352         CoprocessorEnvironment env =
353             region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
354         if (env != null) {
355           found_6 = true;
356           Configuration conf = env.getConfiguration();
357           found6_k1 = conf.get("k1") != null;
358           found6_k2 = conf.get("k2") != null;
359           found6_k3 = conf.get("k3") != null;
360         }
361       }
362     }
363 
364     assertTrue("Class " + cpName1 + " was missing on a region", found_1);
365     assertTrue("Class " + cpName2 + " was missing on a region", found_2);
366     assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
367     assertTrue("Class " + cpName5 + " was missing on a region", found_5);
368     assertTrue("Class " + cpName6 + " was missing on a region", found_6);
369 
370     assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
371     assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
372     assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
373     assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
374   }
375 
376   @Test
377   public void testClassLoadingFromLibDirInJar() throws Exception {
378     loadingClassFromLibDirInJar("/lib/");
379   }
380 
381   @Test
382   public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
383     loadingClassFromLibDirInJar("lib/");
384   }
385 
386   void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
387     FileSystem fs = cluster.getFileSystem();
388 
389     File innerJarFile1 = buildCoprocessorJar(cpName1);
390     File innerJarFile2 = buildCoprocessorJar(cpName2);
391     File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
392 
393     ClassLoaderTestHelper.addJarFilesToJar(
394       outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
395 
396     // copy the jars into dfs
397     fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
398       new Path(fs.getUri().toString() + Path.SEPARATOR));
399     String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
400       outerJarFile.getName();
401     assertTrue("Copy jar file to HDFS failed.",
402       fs.exists(new Path(jarFileOnHDFS)));
403     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
404 
405     // create a table that references the coprocessors
406     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
407     htd.addFamily(new HColumnDescriptor("test"));
408       // without configuration values
409     htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
410       "|" + Coprocessor.PRIORITY_USER);
411       // with configuration values
412     htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
413       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
414     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
415     if (admin.tableExists(tableName)) {
416       if (admin.isTableEnabled(tableName)) {
417         admin.disableTable(tableName);
418       }
419       admin.deleteTable(tableName);
420     }
421     admin.createTable(htd);
422     waitForTable(htd.getTableName());
423 
424     // verify that the coprocessors were loaded
425     boolean found1 = false, found2 = false, found2_k1 = false,
426         found2_k2 = false, found2_k3 = false;
427     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
428     for (HRegion region:
429         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
430       if (region.getRegionNameAsString().startsWith(tableName)) {
431         CoprocessorEnvironment env;
432         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
433         if (env != null) {
434           found1 = true;
435         }
436         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
437         if (env != null) {
438           found2 = true;
439           Configuration conf = env.getConfiguration();
440           found2_k1 = conf.get("k1") != null;
441           found2_k2 = conf.get("k2") != null;
442           found2_k3 = conf.get("k3") != null;
443         }
444       }
445     }
446     assertTrue("Class " + cpName1 + " was missing on a region", found1);
447     assertTrue("Class " + cpName2 + " was missing on a region", found2);
448     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
449     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
450     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
451   }
452 
453   @Test
454   public void testRegionServerCoprocessorsReported() throws Exception {
455     // This was a test for HBASE-4070.
456     // We are removing coprocessors from region load in HBASE-5258.
457     // Therefore, this test now only checks system coprocessors.
458     assertAllRegionServers(null);
459   }
460 
461   /**
462    * return the subset of all regionservers
463    * (actually returns set of ServerLoads)
464    * which host some region in a given table.
465    * used by assertAllRegionServers() below to
466    * test reporting of loaded coprocessors.
467    * @param tableName : given table.
468    * @return subset of all servers.
469    */
470   Map<ServerName, ServerLoad> serversForTable(String tableName) {
471     Map<ServerName, ServerLoad> serverLoadHashMap =
472         new HashMap<ServerName, ServerLoad>();
473     for(Map.Entry<ServerName,ServerLoad> server:
474         TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
475             getOnlineServers().entrySet()) {
476       for( Map.Entry<byte[], RegionLoad> region:
477           server.getValue().getRegionsLoad().entrySet()) {
478         if (region.getValue().getNameAsString().equals(tableName)) {
479           // this server hosts a region of tableName: add this server..
480           serverLoadHashMap.put(server.getKey(),server.getValue());
481           // .. and skip the rest of the regions that it hosts.
482           break;
483         }
484       }
485     }
486     return serverLoadHashMap;
487   }
488 
489   void assertAllRegionServers(String tableName) throws InterruptedException {
490     Map<ServerName, ServerLoad> servers;
491     String[] actualCoprocessors = null;
492     boolean success = false;
493     String[] expectedCoprocessors = regionServerSystemCoprocessors;
494     if (tableName == null) {
495       // if no tableName specified, use all servers.
496       servers = TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().getOnlineServers();
497     } else {
498       servers = serversForTable(tableName);
499     }
500     for (int i = 0; i < 5; i++) {
501       boolean any_failed = false;
502       for(Map.Entry<ServerName,ServerLoad> server: servers.entrySet()) {
503         actualCoprocessors = server.getValue().getRsCoprocessors();
504         if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
505           LOG.debug("failed comparison: actual: " +
506               Arrays.toString(actualCoprocessors) +
507               " ; expected: " + Arrays.toString(expectedCoprocessors));
508           any_failed = true;
509           expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
510           break;
511         }
512         expectedCoprocessors = switchExpectedCoprocessors(expectedCoprocessors);
513       }
514       if (any_failed == false) {
515         success = true;
516         break;
517       }
518       LOG.debug("retrying after failed comparison: " + i);
519       Thread.sleep(1000);
520     }
521     assertTrue(success);
522   }
523 
524   private String[] switchExpectedCoprocessors(String[] expectedCoprocessors) {
525     if (Arrays.equals(regionServerSystemCoprocessors, expectedCoprocessors)) {
526       expectedCoprocessors = masterRegionServerSystemCoprocessors;
527     } else {
528       expectedCoprocessors = regionServerSystemCoprocessors;
529     }
530     return expectedCoprocessors;
531   }
532 
533   @Test
534   public void testMasterCoprocessorsReported() {
535     // HBASE 4070: Improve region server metrics to report loaded coprocessors
536     // to master: verify that the master is reporting the correct set of
537     // loaded coprocessors.
538     final String loadedMasterCoprocessorsVerify =
539         "[" + masterCoprocessor.getSimpleName() + "]";
540     String loadedMasterCoprocessors =
541         java.util.Arrays.toString(
542             TEST_UTIL.getHBaseCluster().getMaster().getCoprocessors());
543     assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
544   }
545 
546   @Test
547   public void testFindCoprocessors() {
548     // HBASE 12277: 
549     CoprocessorHost masterCpHost = TEST_UTIL.getHBaseCluster().getMaster().getCoprocessorHost();
550 
551     List<MasterObserver> masterObservers = masterCpHost.findCoprocessors(MasterObserver.class);
552 
553     assertTrue(masterObservers != null && masterObservers.size() > 0);
554     assertEquals(masterCoprocessor.getSimpleName(),
555                  masterObservers.get(0).getClass().getSimpleName());
556   }
557 
558   private void waitForTable(TableName name) throws InterruptedException, IOException {
559     // First wait until all regions are online
560     TEST_UTIL.waitTableEnabled(name.getName());
561     // Now wait a bit longer for the coprocessor hosts to load the CPs
562     Thread.sleep(1000);
563   }
564 
565 }
566