1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.coprocessor;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.hbase.*;
27  import org.apache.hadoop.hbase.client.HBaseAdmin;
28  import org.apache.hadoop.hbase.regionserver.HRegion;
29  
30  import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
31  import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
32  import org.apache.hadoop.hdfs.MiniDFSCluster;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  
36  import java.io.*;
37  import java.util.*;
38  import java.util.jar.*;
39  
40  import org.junit.*;
41  import org.junit.experimental.categories.Category;
42  
43  import static org.junit.Assert.assertEquals;
44  import static org.junit.Assert.assertNotNull;
45  import static org.junit.Assert.assertTrue;
46  import static org.junit.Assert.assertFalse;
47  
48  /**
49   * Test coprocessors class loading.
50   */
51  @Category(MediumTests.class)
52  public class TestClassLoading {
53    private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
54    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55  
56    private static MiniDFSCluster cluster;
57  
58    static final int BUFFER_SIZE = 4096;
59    static final String tableName = "TestClassLoading";
60    static final String cpName1 = "TestCP1";
61    static final String cpName2 = "TestCP2";
62    static final String cpName3 = "TestCP3";
63    static final String cpName4 = "TestCP4";
64    static final String cpName5 = "TestCP5";
65    static final String cpName6 = "TestCP6";
66    static final String cpNameInvalid = "TestCPInvalid";
67  
68    private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
69    private static Class<?> regionCoprocessor2 = GenericEndpoint.class;
70    private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
71    private static Class<?> masterCoprocessor = BaseMasterObserver.class;
72  
73    private static final String[] regionServerSystemCoprocessors =
74        new String[]{
75        regionServerCoprocessor.getSimpleName()
76    };
77  
78    @BeforeClass
79    public static void setUpBeforeClass() throws Exception {
80      Configuration conf = TEST_UTIL.getConfiguration();
81  
82      // regionCoprocessor1 will be loaded on all regionservers, since it is
83      // loaded for any tables (user or meta).
84      conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
85          regionCoprocessor1.getName());
86  
87      // regionCoprocessor2 will be loaded only on regionservers that serve a
88      // user table region. Therefore, if there are no user tables loaded,
89      // this coprocessor will not be loaded on any regionserver.
90      conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
91          regionCoprocessor2.getName());
92  
93      conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
94          regionServerCoprocessor.getName());
95      conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
96          masterCoprocessor.getName());
97      TEST_UTIL.startMiniCluster(1);
98      cluster = TEST_UTIL.getDFSCluster();
99    }
100 
101   @AfterClass
102   public static void tearDownAfterClass() throws Exception {
103     TEST_UTIL.shutdownMiniCluster();
104   }
105 
106   static File buildCoprocessorJar(String className) throws Exception {
107     String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
108       "public class " + className + " extends BaseRegionObserver {}";
109     return ClassLoaderTestHelper.buildJar(
110       TEST_UTIL.getDataTestDir().toString(), className, code);
111   }
112 
113   @Test
114   // HBASE-3516: Test CP Class loading from HDFS
115   public void testClassLoadingFromHDFS() throws Exception {
116     FileSystem fs = cluster.getFileSystem();
117 
118     File jarFile1 = buildCoprocessorJar(cpName1);
119     File jarFile2 = buildCoprocessorJar(cpName2);
120 
121     // copy the jars into dfs
122     fs.copyFromLocalFile(new Path(jarFile1.getPath()),
123       new Path(fs.getUri().toString() + Path.SEPARATOR));
124     String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
125       jarFile1.getName();
126     Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
127     assertTrue("Copy jar file to HDFS failed.",
128       fs.exists(pathOnHDFS1));
129     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
130 
131     fs.copyFromLocalFile(new Path(jarFile2.getPath()),
132         new Path(fs.getUri().toString() + Path.SEPARATOR));
133     String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
134       jarFile2.getName();
135     Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
136     assertTrue("Copy jar file to HDFS failed.",
137       fs.exists(pathOnHDFS2));
138     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
139 
140     // create a table that references the coprocessors
141     HTableDescriptor htd = new HTableDescriptor(tableName);
142     htd.addFamily(new HColumnDescriptor("test"));
143       // without configuration values
144     htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
145       "|" + Coprocessor.PRIORITY_USER);
146       // with configuration values
147     htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
148       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
149     // same jar but invalid class name (should fail to load this class)
150     htd.setValue("COPROCESSOR$3", jarFileOnHDFS2.toString() + "|" + cpNameInvalid +
151       "|" + Coprocessor.PRIORITY_USER);
152     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
153     if (admin.tableExists(tableName)) {
154       admin.disableTable(tableName);
155       admin.deleteTable(tableName);
156     }
157     CoprocessorClassLoader.clearCache();
158     byte[] startKey = {10, 63};
159     byte[] endKey = {12, 43};
160     admin.createTable(htd, startKey, endKey, 4);
161     waitForTable(htd.getName());
162 
163     // verify that the coprocessors were loaded
164     boolean foundTableRegion=false;
165     boolean found_invalid = true, found1 = true, found2 = true, found2_k1 = true,
166         found2_k2 = true, found2_k3 = true;
167     Map<HRegion, Set<ClassLoader>> regionsActiveClassLoaders =
168         new HashMap<HRegion, Set<ClassLoader>>();
169     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
170     for (HRegion region:
171         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
172       if (region.getRegionNameAsString().startsWith(tableName)) {
173         foundTableRegion = true;
174         CoprocessorEnvironment env;
175         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
176         found1 = found1 && (env != null);
177         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
178         found2 = found2 && (env != null);
179         if (env != null) {
180           Configuration conf = env.getConfiguration();
181           found2_k1 = found2_k1 && (conf.get("k1") != null);
182           found2_k2 = found2_k2 && (conf.get("k2") != null);
183           found2_k3 = found2_k3 && (conf.get("k3") != null);
184         } else {
185           found2_k1 = found2_k2 = found2_k3 = false;
186         }
187         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpNameInvalid);
188         found_invalid = found_invalid && (env != null);
189 
190         regionsActiveClassLoaders
191             .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
192       }
193     }
194 
195     assertTrue("No region was found for table " + tableName, foundTableRegion);
196     assertTrue("Class " + cpName1 + " was missing on a region", found1);
197     assertTrue("Class " + cpName2 + " was missing on a region", found2);
198     //an invalid CP class name is defined for this table, validate that it is not loaded
199     assertFalse("Class " + cpNameInvalid + " was found on a region", found_invalid);
200     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
201     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
202     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
203     // check if CP classloaders are cached
204     assertNotNull(jarFileOnHDFS1 + " was not cached",
205       CoprocessorClassLoader.getIfCached(pathOnHDFS1));
206     assertNotNull(jarFileOnHDFS2 + " was not cached",
207       CoprocessorClassLoader.getIfCached(pathOnHDFS2));
208     //two external jar used, should be one classloader per jar
209     assertEquals("The number of cached classloaders should be equal to the number" +
210       " of external jar files",
211       2, CoprocessorClassLoader.getAllCached().size());
212     //check if region active classloaders are shared across all RS regions
213     Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
214       CoprocessorClassLoader.getAllCached());
215     for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
216       assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
217         + " ClassLoader Cache:" + externalClassLoaders
218         + " Region ClassLoaders:" + regionCP.getValue(),
219         externalClassLoaders.containsAll(regionCP.getValue()));
220     }
221   }
222 
223   private String getLocalPath(File file) {
224     return new Path(file.toURI()).toString();
225   }
226 
227   @Test
228   // HBASE-3516: Test CP Class loading from local file system
229   public void testClassLoadingFromLocalFS() throws Exception {
230     File jarFile = buildCoprocessorJar(cpName3);
231 
232     // create a table that references the jar
233     HTableDescriptor htd = new HTableDescriptor(cpName3);
234     htd.addFamily(new HColumnDescriptor("test"));
235     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
236       Coprocessor.PRIORITY_USER);
237     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
238     admin.createTable(htd);
239     waitForTable(htd.getName());
240 
241     // verify that the coprocessor was loaded
242     boolean found = false;
243     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
244     for (HRegion region:
245         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
246       if (region.getRegionNameAsString().startsWith(cpName3)) {
247         found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
248       }
249     }
250     assertTrue("Class " + cpName3 + " was missing on a region", found);
251   }
252 
253   @Test
254   // HBASE-6308: Test CP classloader is the CoprocessorClassLoader
255   public void testPrivateClassLoader() throws Exception {
256     File jarFile = buildCoprocessorJar(cpName4);
257 
258     // create a table that references the jar
259     HTableDescriptor htd = new HTableDescriptor(cpName4);
260     htd.addFamily(new HColumnDescriptor("test"));
261     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
262       Coprocessor.PRIORITY_USER);
263     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
264     admin.createTable(htd);
265     waitForTable(htd.getName());
266 
267     // verify that the coprocessor was loaded correctly
268     boolean found = false;
269     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
270     for (HRegion region:
271         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
272       if (region.getRegionNameAsString().startsWith(cpName4)) {
273         Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
274         if (cp != null) {
275           found = true;
276           assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
277             cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
278         }
279       }
280     }
281     assertTrue("Class " + cpName4 + " was missing on a region", found);
282   }
283 
284   @Test
285   // HBase-3810: Registering a Coprocessor at HTableDescriptor should be
286   // less strict
287   public void testHBase3810() throws Exception {
288     // allowed value pattern: [path] | class name | [priority] | [key values]
289 
290     File jarFile1 = buildCoprocessorJar(cpName1);
291     File jarFile2 = buildCoprocessorJar(cpName2);
292     File jarFile5 = buildCoprocessorJar(cpName5);
293     File jarFile6 = buildCoprocessorJar(cpName6);
294 
295     String cpKey1 = "COPROCESSOR$1";
296     String cpKey2 = " Coprocessor$2 ";
297     String cpKey3 = " coprocessor$03 ";
298 
299     String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
300         Coprocessor.PRIORITY_USER;
301     String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
302     // load from default class loader
303     String cpValue3 =
304         " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
305 
306     // create a table that references the jar
307     HTableDescriptor htd = new HTableDescriptor(tableName);
308     htd.addFamily(new HColumnDescriptor("test"));
309 
310     // add 3 coprocessors by setting htd attributes directly.
311     htd.setValue(cpKey1, cpValue1);
312     htd.setValue(cpKey2, cpValue2);
313     htd.setValue(cpKey3, cpValue3);
314 
315     // add 2 coprocessor by using new htd.addCoprocessor() api
316     htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
317         Coprocessor.PRIORITY_USER, null);
318     Map<String, String> kvs = new HashMap<String, String>();
319     kvs.put("k1", "v1");
320     kvs.put("k2", "v2");
321     kvs.put("k3", "v3");
322     htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
323         Coprocessor.PRIORITY_USER, kvs);
324 
325     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
326     if (admin.tableExists(tableName)) {
327       admin.disableTable(tableName);
328       admin.deleteTable(tableName);
329     }
330     admin.createTable(htd);
331     waitForTable(htd.getName());
332 
333     // verify that the coprocessor was loaded
334     boolean found_2 = false, found_1 = false, found_3 = false,
335         found_5 = false, found_6 = false;
336     boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
337         found6_k4 = false;
338 
339     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
340     for (HRegion region:
341         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
342       if (region.getRegionNameAsString().startsWith(tableName)) {
343         found_1 = found_1 ||
344             (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
345         found_2 = found_2 ||
346             (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
347         found_3 = found_3 ||
348             (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
349                 != null);
350         found_5 = found_5 ||
351             (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
352 
353         CoprocessorEnvironment env =
354             region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
355         if (env != null) {
356           found_6 = true;
357           Configuration conf = env.getConfiguration();
358           found6_k1 = conf.get("k1") != null;
359           found6_k2 = conf.get("k2") != null;
360           found6_k3 = conf.get("k3") != null;
361         }
362       }
363     }
364 
365     assertTrue("Class " + cpName1 + " was missing on a region", found_1);
366     assertTrue("Class " + cpName2 + " was missing on a region", found_2);
367     assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
368     assertTrue("Class " + cpName5 + " was missing on a region", found_5);
369     assertTrue("Class " + cpName6 + " was missing on a region", found_6);
370 
371     assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
372     assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
373     assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
374     assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
375   }
376 
377   @Test
378   public void testClassLoadingFromLibDirInJar() throws Exception {
379     loadingClassFromLibDirInJar("/lib/");
380   }
381 
382   @Test
383   public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
384     loadingClassFromLibDirInJar("lib/");
385   }
386 
387   void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
388     FileSystem fs = cluster.getFileSystem();
389 
390     File innerJarFile1 = buildCoprocessorJar(cpName1);
391     File innerJarFile2 = buildCoprocessorJar(cpName2);
392     File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
393 
394     byte buffer[] = new byte[BUFFER_SIZE];
395     // TODO: code here and elsewhere in this file is duplicated w/TestClassFinder.
396     //       Some refactoring may be in order...
397     // Open archive file
398     FileOutputStream stream = new FileOutputStream(outerJarFile);
399     JarOutputStream out = new JarOutputStream(stream, new Manifest());
400 
401     for (File jarFile: new File[] { innerJarFile1, innerJarFile2 }) {
402       // Add archive entry
403       JarEntry jarAdd = new JarEntry(libPrefix + jarFile.getName());
404       jarAdd.setTime(jarFile.lastModified());
405       out.putNextEntry(jarAdd);
406 
407       // Write file to archive
408       FileInputStream in = new FileInputStream(jarFile);
409       while (true) {
410         int nRead = in.read(buffer, 0, buffer.length);
411         if (nRead <= 0)
412           break;
413         out.write(buffer, 0, nRead);
414       }
415       in.close();
416     }
417     out.close();
418     stream.close();
419     LOG.info("Adding jar file to outer jar file completed");
420 
421     // copy the jars into dfs
422     fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
423       new Path(fs.getUri().toString() + Path.SEPARATOR));
424     String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
425       outerJarFile.getName();
426     assertTrue("Copy jar file to HDFS failed.",
427       fs.exists(new Path(jarFileOnHDFS)));
428     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
429 
430     // create a table that references the coprocessors
431     HTableDescriptor htd = new HTableDescriptor(tableName);
432     htd.addFamily(new HColumnDescriptor("test"));
433       // without configuration values
434     htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
435       "|" + Coprocessor.PRIORITY_USER);
436       // with configuration values
437     htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
438       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
439     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
440     if (admin.tableExists(tableName)) {
441       admin.disableTable(tableName);
442       admin.deleteTable(tableName);
443     }
444     admin.createTable(htd);
445     waitForTable(htd.getName());
446 
447     // verify that the coprocessors were loaded
448     boolean found1 = false, found2 = false, found2_k1 = false,
449         found2_k2 = false, found2_k3 = false;
450     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
451     for (HRegion region:
452         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
453       if (region.getRegionNameAsString().startsWith(tableName)) {
454         CoprocessorEnvironment env;
455         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
456         if (env != null) {
457           found1 = true;
458         }
459         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
460         if (env != null) {
461           found2 = true;
462           Configuration conf = env.getConfiguration();
463           found2_k1 = conf.get("k1") != null;
464           found2_k2 = conf.get("k2") != null;
465           found2_k3 = conf.get("k3") != null;
466         }
467       }
468     }
469     assertTrue("Class " + cpName1 + " was missing on a region", found1);
470     assertTrue("Class " + cpName2 + " was missing on a region", found2);
471     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
472     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
473     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
474   }
475 
476   @Test
477   public void testRegionServerCoprocessorsReported() throws Exception {
478     // This was a test for HBASE-4070.
479     // We are removing coprocessors from region load in HBASE-5258.
480     // Therefore, this test now only checks system coprocessors.
481 
482     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
483     assertAllRegionServers(regionServerSystemCoprocessors,null);
484   }
485 
486   /**
487    * return the subset of all regionservers
488    * (actually returns set of HServerLoads)
489    * which host some region in a given table.
490    * used by assertAllRegionServers() below to
491    * test reporting of loaded coprocessors.
492    * @param tableName : given table.
493    * @return subset of all servers.
494    */
495   Map<ServerName, HServerLoad> serversForTable(String tableName) {
496     Map<ServerName, HServerLoad> serverLoadHashMap =
497         new HashMap<ServerName, HServerLoad>();
498     for(Map.Entry<ServerName,HServerLoad> server:
499         TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
500             getOnlineServers().entrySet()) {
501       for(Map.Entry<byte[], HServerLoad.RegionLoad> region:
502           server.getValue().getRegionsLoad().entrySet()) {
503         if (region.getValue().getNameAsString().equals(tableName)) {
504           // this server server hosts a region of tableName: add this server..
505           serverLoadHashMap.put(server.getKey(),server.getValue());
506           // .. and skip the rest of the regions that it hosts.
507           break;
508         }
509       }
510     }
511     return serverLoadHashMap;
512   }
513 
514   void assertAllRegionServers(String[] expectedCoprocessors, String tableName)
515       throws InterruptedException {
516     Map<ServerName, HServerLoad> servers;
517     String[] actualCoprocessors = null;
518     boolean success = false;
519     for(int i = 0; i < 5; i++) {
520       if (tableName == null) {
521         //if no tableName specified, use all servers.
522         servers =
523             TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
524                 getOnlineServers();
525       } else {
526         servers = serversForTable(tableName);
527       }
528       boolean any_failed = false;
529       for(Map.Entry<ServerName,HServerLoad> server: servers.entrySet()) {
530         actualCoprocessors = server.getValue().getRsCoprocessors();
531         if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
532           LOG.debug("failed comparison: actual: " +
533               Arrays.toString(actualCoprocessors) +
534               " ; expected: " + Arrays.toString(expectedCoprocessors));
535           any_failed = true;
536           break;
537         }
538       }
539       if (any_failed == false) {
540         success = true;
541         break;
542       }
543       LOG.debug("retrying after failed comparison: " + i);
544       Thread.sleep(1000);
545     }
546     assertTrue(success);
547   }
548 
549   @Test
550   public void testMasterCoprocessorsReported() {
551     // HBASE 4070: Improve region server metrics to report loaded coprocessors
552     // to master: verify that the master is reporting the correct set of
553     // loaded coprocessors.
554     final String loadedMasterCoprocessorsVerify =
555         "[" + masterCoprocessor.getSimpleName() + "]";
556     String loadedMasterCoprocessors =
557         java.util.Arrays.toString(
558             TEST_UTIL.getHBaseCluster().getMaster().getCoprocessors());
559     assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
560   }
561 
562   private void waitForTable(byte[] name) throws InterruptedException, IOException {
563     // First wait until all regions are online
564     TEST_UTIL.waitTableEnabled(name, 5000);
565     // Now wait a bit longer for the coprocessor hosts to load the CPs
566     Thread.sleep(1000);
567   }
568 
569   @org.junit.Rule
570   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
571     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
572 }
573