View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.coprocessor;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.hbase.*;
26  import org.apache.hadoop.hbase.client.HBaseAdmin;
27  import org.apache.hadoop.hbase.regionserver.HRegion;
28  import org.apache.hadoop.hbase.regionserver.TestServerCustomProtocol;
29  import org.apache.hadoop.hbase.util.ClassLoaderTestHelper;
30  import org.apache.hadoop.hbase.util.CoprocessorClassLoader;
31  import org.apache.hadoop.hdfs.MiniDFSCluster;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.ServerLoad;
35  import org.apache.hadoop.hbase.RegionLoad;
36  
37  import java.io.*;
38  import java.util.*;
39  
40  import org.junit.*;
41  import org.junit.experimental.categories.Category;
42  
43  import static org.junit.Assert.assertEquals;
44  import static org.junit.Assert.assertNotNull;
45  import static org.junit.Assert.assertTrue;
46  import static org.junit.Assert.assertFalse;
47  
48  /**
49   * Test coprocessors class loading.
50   */
51  @Category(MediumTests.class)
52  public class TestClassLoading {
53    private static final Log LOG = LogFactory.getLog(TestClassLoading.class);
54    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55  
56    private static MiniDFSCluster cluster;
57  
58    static final String tableName = "TestClassLoading";
59    static final String cpName1 = "TestCP1";
60    static final String cpName2 = "TestCP2";
61    static final String cpName3 = "TestCP3";
62    static final String cpName4 = "TestCP4";
63    static final String cpName5 = "TestCP5";
64    static final String cpName6 = "TestCP6";
65    static final String cpNameInvalid = "TestCPInvalid";
66  
67    private static Class<?> regionCoprocessor1 = ColumnAggregationEndpoint.class;
68    // TOOD: Fix the import of this handler.  It is coming in from a package that is far away.
69    private static Class<?> regionCoprocessor2 = TestServerCustomProtocol.PingHandler.class;
70    private static Class<?> regionServerCoprocessor = SampleRegionWALObserver.class;
71    private static Class<?> masterCoprocessor = BaseMasterObserver.class;
72  
73    private static final String[] regionServerSystemCoprocessors =
74        new String[]{
75        regionServerCoprocessor.getSimpleName()
76    };
77  
78    @BeforeClass
79    public static void setUpBeforeClass() throws Exception {
80      Configuration conf = TEST_UTIL.getConfiguration();
81  
82      // regionCoprocessor1 will be loaded on all regionservers, since it is
83      // loaded for any tables (user or meta).
84      conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
85          regionCoprocessor1.getName());
86  
87      // regionCoprocessor2 will be loaded only on regionservers that serve a
88      // user table region. Therefore, if there are no user tables loaded,
89      // this coprocessor will not be loaded on any regionserver.
90      conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
91          regionCoprocessor2.getName());
92  
93      conf.setStrings(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
94          regionServerCoprocessor.getName());
95      conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
96          masterCoprocessor.getName());
97      TEST_UTIL.startMiniCluster(1);
98      cluster = TEST_UTIL.getDFSCluster();
99    }
100 
101   @AfterClass
102   public static void tearDownAfterClass() throws Exception {
103     TEST_UTIL.shutdownMiniCluster();
104   }
105 
106   static File buildCoprocessorJar(String className) throws Exception {
107     String code = "import org.apache.hadoop.hbase.coprocessor.*;" +
108       "public class " + className + " extends BaseRegionObserver {}";
109     return ClassLoaderTestHelper.buildJar(
110       TEST_UTIL.getDataTestDir().toString(), className, code);
111   }
112 
113   @Test
114   // HBASE-3516: Test CP Class loading from HDFS
115   public void testClassLoadingFromHDFS() throws Exception {
116     FileSystem fs = cluster.getFileSystem();
117 
118     File jarFile1 = buildCoprocessorJar(cpName1);
119     File jarFile2 = buildCoprocessorJar(cpName2);
120 
121     // copy the jars into dfs
122     fs.copyFromLocalFile(new Path(jarFile1.getPath()),
123       new Path(fs.getUri().toString() + Path.SEPARATOR));
124     String jarFileOnHDFS1 = fs.getUri().toString() + Path.SEPARATOR +
125       jarFile1.getName();
126     Path pathOnHDFS1 = new Path(jarFileOnHDFS1);
127     assertTrue("Copy jar file to HDFS failed.",
128       fs.exists(pathOnHDFS1));
129     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS1);
130 
131     fs.copyFromLocalFile(new Path(jarFile2.getPath()),
132         new Path(fs.getUri().toString() + Path.SEPARATOR));
133     String jarFileOnHDFS2 = fs.getUri().toString() + Path.SEPARATOR +
134       jarFile2.getName();
135     Path pathOnHDFS2 = new Path(jarFileOnHDFS2);
136     assertTrue("Copy jar file to HDFS failed.",
137       fs.exists(pathOnHDFS2));
138     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS2);
139 
140     // create a table that references the coprocessors
141     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
142     htd.addFamily(new HColumnDescriptor("test"));
143       // without configuration values
144     htd.setValue("COPROCESSOR$1", jarFileOnHDFS1.toString() + "|" + cpName1 +
145       "|" + Coprocessor.PRIORITY_USER);
146       // with configuration values
147     htd.setValue("COPROCESSOR$2", jarFileOnHDFS2.toString() + "|" + cpName2 +
148       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
149     // same jar but invalid class name (should fail to load this class)
150     htd.setValue("COPROCESSOR$3", jarFileOnHDFS2.toString() + "|" + cpNameInvalid +
151       "|" + Coprocessor.PRIORITY_USER);
152     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
153     if (admin.tableExists(tableName)) {
154       if (admin.isTableEnabled(tableName)) {
155         admin.disableTable(tableName);
156       }
157       admin.deleteTable(tableName);
158     }
159     CoprocessorClassLoader.clearCache();
160     byte[] startKey = {10, 63};
161     byte[] endKey = {12, 43};
162     admin.createTable(htd, startKey, endKey, 4);
163     waitForTable(htd.getTableName());
164 
165     // verify that the coprocessors were loaded
166     boolean foundTableRegion=false;
167     boolean found_invalid = true, found1 = true, found2 = true, found2_k1 = true,
168         found2_k2 = true, found2_k3 = true;
169     Map<HRegion, Set<ClassLoader>> regionsActiveClassLoaders =
170         new HashMap<HRegion, Set<ClassLoader>>();
171     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
172     for (HRegion region:
173         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
174       if (region.getRegionNameAsString().startsWith(tableName)) {
175         foundTableRegion = true;
176         CoprocessorEnvironment env;
177         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
178         found1 = found1 && (env != null);
179         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
180         found2 = found2 && (env != null);
181         if (env != null) {
182           Configuration conf = env.getConfiguration();
183           found2_k1 = found2_k1 && (conf.get("k1") != null);
184           found2_k2 = found2_k2 && (conf.get("k2") != null);
185           found2_k3 = found2_k3 && (conf.get("k3") != null);
186         } else {
187           found2_k1 = found2_k2 = found2_k3 = false;
188         }
189         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpNameInvalid);
190         found_invalid = found_invalid && (env != null);
191 
192         regionsActiveClassLoaders
193             .put(region, ((CoprocessorHost) region.getCoprocessorHost()).getExternalClassLoaders());
194       }
195     }
196 
197     assertTrue("No region was found for table " + tableName, foundTableRegion);
198     assertTrue("Class " + cpName1 + " was missing on a region", found1);
199     assertTrue("Class " + cpName2 + " was missing on a region", found2);
200     //an invalid CP class name is defined for this table, validate that it is not loaded
201     assertFalse("Class " + cpNameInvalid + " was found on a region", found_invalid);
202     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
203     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
204     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
205     // check if CP classloaders are cached
206     assertNotNull(jarFileOnHDFS1 + " was not cached",
207       CoprocessorClassLoader.getIfCached(pathOnHDFS1));
208     assertNotNull(jarFileOnHDFS2 + " was not cached",
209       CoprocessorClassLoader.getIfCached(pathOnHDFS2));
210     //two external jar used, should be one classloader per jar
211     assertEquals("The number of cached classloaders should be equal to the number" +
212       " of external jar files",
213       2, CoprocessorClassLoader.getAllCached().size());
214     //check if region active classloaders are shared across all RS regions
215     Set<ClassLoader> externalClassLoaders = new HashSet<ClassLoader>(
216       CoprocessorClassLoader.getAllCached());
217     for (Map.Entry<HRegion, Set<ClassLoader>> regionCP : regionsActiveClassLoaders.entrySet()) {
218       assertTrue("Some CP classloaders for region " + regionCP.getKey() + " are not cached."
219         + " ClassLoader Cache:" + externalClassLoaders
220         + " Region ClassLoaders:" + regionCP.getValue(),
221         externalClassLoaders.containsAll(regionCP.getValue()));
222     }
223   }
224 
225   private String getLocalPath(File file) {
226     return new Path(file.toURI()).toString();
227   }
228 
229   @Test
230   // HBASE-3516: Test CP Class loading from local file system
231   public void testClassLoadingFromLocalFS() throws Exception {
232     File jarFile = buildCoprocessorJar(cpName3);
233 
234     // create a table that references the jar
235     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName3));
236     htd.addFamily(new HColumnDescriptor("test"));
237     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
238       Coprocessor.PRIORITY_USER);
239     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
240     admin.createTable(htd);
241     waitForTable(htd.getTableName());
242 
243     // verify that the coprocessor was loaded
244     boolean found = false;
245     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
246     for (HRegion region:
247         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
248       if (region.getRegionNameAsString().startsWith(cpName3)) {
249         found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
250       }
251     }
252     assertTrue("Class " + cpName3 + " was missing on a region", found);
253   }
254 
255   @Test
256   // HBASE-6308: Test CP classloader is the CoprocessorClassLoader
257   public void testPrivateClassLoader() throws Exception {
258     File jarFile = buildCoprocessorJar(cpName4);
259 
260     // create a table that references the jar
261     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(cpName4));
262     htd.addFamily(new HColumnDescriptor("test"));
263     htd.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName4 + "|" +
264       Coprocessor.PRIORITY_USER);
265     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
266     admin.createTable(htd);
267     waitForTable(htd.getTableName());
268 
269     // verify that the coprocessor was loaded correctly
270     boolean found = false;
271     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
272     for (HRegion region:
273         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
274       if (region.getRegionNameAsString().startsWith(cpName4)) {
275         Coprocessor cp = region.getCoprocessorHost().findCoprocessor(cpName4);
276         if (cp != null) {
277           found = true;
278           assertEquals("Class " + cpName4 + " was not loaded by CoprocessorClassLoader",
279             cp.getClass().getClassLoader().getClass(), CoprocessorClassLoader.class);
280         }
281       }
282     }
283     assertTrue("Class " + cpName4 + " was missing on a region", found);
284   }
285 
286   @Test
287   // HBase-3810: Registering a Coprocessor at HTableDescriptor should be
288   // less strict
289   public void testHBase3810() throws Exception {
290     // allowed value pattern: [path] | class name | [priority] | [key values]
291 
292     File jarFile1 = buildCoprocessorJar(cpName1);
293     File jarFile2 = buildCoprocessorJar(cpName2);
294     File jarFile5 = buildCoprocessorJar(cpName5);
295     File jarFile6 = buildCoprocessorJar(cpName6);
296 
297     String cpKey1 = "COPROCESSOR$1";
298     String cpKey2 = " Coprocessor$2 ";
299     String cpKey3 = " coprocessor$03 ";
300 
301     String cpValue1 = getLocalPath(jarFile1) + "|" + cpName1 + "|" +
302         Coprocessor.PRIORITY_USER;
303     String cpValue2 = getLocalPath(jarFile2) + " | " + cpName2 + " | ";
304     // load from default class loader
305     String cpValue3 =
306         " | org.apache.hadoop.hbase.coprocessor.SimpleRegionObserver | | k=v ";
307 
308     // create a table that references the jar
309     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
310     htd.addFamily(new HColumnDescriptor("test"));
311 
312     // add 3 coprocessors by setting htd attributes directly.
313     htd.setValue(cpKey1, cpValue1);
314     htd.setValue(cpKey2, cpValue2);
315     htd.setValue(cpKey3, cpValue3);
316 
317     // add 2 coprocessor by using new htd.addCoprocessor() api
318     htd.addCoprocessor(cpName5, new Path(getLocalPath(jarFile5)),
319         Coprocessor.PRIORITY_USER, null);
320     Map<String, String> kvs = new HashMap<String, String>();
321     kvs.put("k1", "v1");
322     kvs.put("k2", "v2");
323     kvs.put("k3", "v3");
324     htd.addCoprocessor(cpName6, new Path(getLocalPath(jarFile6)),
325         Coprocessor.PRIORITY_USER, kvs);
326 
327     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
328     if (admin.tableExists(tableName)) {
329       if (admin.isTableEnabled(tableName)) {
330         admin.disableTable(tableName);
331       }
332       admin.deleteTable(tableName);
333     }
334     admin.createTable(htd);
335     waitForTable(htd.getTableName());
336 
337     // verify that the coprocessor was loaded
338     boolean found_2 = false, found_1 = false, found_3 = false,
339         found_5 = false, found_6 = false;
340     boolean found6_k1 = false, found6_k2 = false, found6_k3 = false,
341         found6_k4 = false;
342 
343     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
344     for (HRegion region:
345         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
346       if (region.getRegionNameAsString().startsWith(tableName)) {
347         found_1 = found_1 ||
348             (region.getCoprocessorHost().findCoprocessor(cpName1) != null);
349         found_2 = found_2 ||
350             (region.getCoprocessorHost().findCoprocessor(cpName2) != null);
351         found_3 = found_3 ||
352             (region.getCoprocessorHost().findCoprocessor("SimpleRegionObserver")
353                 != null);
354         found_5 = found_5 ||
355             (region.getCoprocessorHost().findCoprocessor(cpName5) != null);
356 
357         CoprocessorEnvironment env =
358             region.getCoprocessorHost().findCoprocessorEnvironment(cpName6);
359         if (env != null) {
360           found_6 = true;
361           Configuration conf = env.getConfiguration();
362           found6_k1 = conf.get("k1") != null;
363           found6_k2 = conf.get("k2") != null;
364           found6_k3 = conf.get("k3") != null;
365         }
366       }
367     }
368 
369     assertTrue("Class " + cpName1 + " was missing on a region", found_1);
370     assertTrue("Class " + cpName2 + " was missing on a region", found_2);
371     assertTrue("Class SimpleRegionObserver was missing on a region", found_3);
372     assertTrue("Class " + cpName5 + " was missing on a region", found_5);
373     assertTrue("Class " + cpName6 + " was missing on a region", found_6);
374 
375     assertTrue("Configuration key 'k1' was missing on a region", found6_k1);
376     assertTrue("Configuration key 'k2' was missing on a region", found6_k2);
377     assertTrue("Configuration key 'k3' was missing on a region", found6_k3);
378     assertFalse("Configuration key 'k4' wasn't configured", found6_k4);
379   }
380 
381   @Test
382   public void testClassLoadingFromLibDirInJar() throws Exception {
383     loadingClassFromLibDirInJar("/lib/");
384   }
385 
386   @Test
387   public void testClassLoadingFromRelativeLibDirInJar() throws Exception {
388     loadingClassFromLibDirInJar("lib/");
389   }
390 
391   void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
392     FileSystem fs = cluster.getFileSystem();
393 
394     File innerJarFile1 = buildCoprocessorJar(cpName1);
395     File innerJarFile2 = buildCoprocessorJar(cpName2);
396     File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");
397 
398     ClassLoaderTestHelper.addJarFilesToJar(
399       outerJarFile, libPrefix, innerJarFile1, innerJarFile2);
400 
401     // copy the jars into dfs
402     fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
403       new Path(fs.getUri().toString() + Path.SEPARATOR));
404     String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
405       outerJarFile.getName();
406     assertTrue("Copy jar file to HDFS failed.",
407       fs.exists(new Path(jarFileOnHDFS)));
408     LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);
409 
410     // create a table that references the coprocessors
411     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
412     htd.addFamily(new HColumnDescriptor("test"));
413       // without configuration values
414     htd.setValue("COPROCESSOR$1", jarFileOnHDFS.toString() + "|" + cpName1 +
415       "|" + Coprocessor.PRIORITY_USER);
416       // with configuration values
417     htd.setValue("COPROCESSOR$2", jarFileOnHDFS.toString() + "|" + cpName2 +
418       "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
419     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
420     if (admin.tableExists(tableName)) {
421       if (admin.isTableEnabled(tableName)) {
422         admin.disableTable(tableName);
423       }
424       admin.deleteTable(tableName);
425     }
426     admin.createTable(htd);
427     waitForTable(htd.getTableName());
428 
429     // verify that the coprocessors were loaded
430     boolean found1 = false, found2 = false, found2_k1 = false,
431         found2_k2 = false, found2_k3 = false;
432     MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
433     for (HRegion region:
434         hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
435       if (region.getRegionNameAsString().startsWith(tableName)) {
436         CoprocessorEnvironment env;
437         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
438         if (env != null) {
439           found1 = true;
440         }
441         env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
442         if (env != null) {
443           found2 = true;
444           Configuration conf = env.getConfiguration();
445           found2_k1 = conf.get("k1") != null;
446           found2_k2 = conf.get("k2") != null;
447           found2_k3 = conf.get("k3") != null;
448         }
449       }
450     }
451     assertTrue("Class " + cpName1 + " was missing on a region", found1);
452     assertTrue("Class " + cpName2 + " was missing on a region", found2);
453     assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
454     assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
455     assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
456   }
457 
458   @Test
459   public void testRegionServerCoprocessorsReported() throws Exception {
460     // This was a test for HBASE-4070.
461     // We are removing coprocessors from region load in HBASE-5258.
462     // Therefore, this test now only checks system coprocessors.
463 
464     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
465     assertAllRegionServers(regionServerSystemCoprocessors,null);
466   }
467 
468   /**
469    * return the subset of all regionservers
470    * (actually returns set of ServerLoads)
471    * which host some region in a given table.
472    * used by assertAllRegionServers() below to
473    * test reporting of loaded coprocessors.
474    * @param tableName : given table.
475    * @return subset of all servers.
476    */
477   Map<ServerName, ServerLoad> serversForTable(String tableName) {
478     Map<ServerName, ServerLoad> serverLoadHashMap =
479         new HashMap<ServerName, ServerLoad>();
480     for(Map.Entry<ServerName,ServerLoad> server:
481         TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
482             getOnlineServers().entrySet()) {
483       for( Map.Entry<byte[], RegionLoad> region:
484           server.getValue().getRegionsLoad().entrySet()) {
485         if (region.getValue().getNameAsString().equals(tableName)) {
486           // this server hosts a region of tableName: add this server..
487           serverLoadHashMap.put(server.getKey(),server.getValue());
488           // .. and skip the rest of the regions that it hosts.
489           break;
490         }
491       }
492     }
493     return serverLoadHashMap;
494   }
495 
496   void assertAllRegionServers(String[] expectedCoprocessors, String tableName)
497       throws InterruptedException {
498     Map<ServerName, ServerLoad> servers;
499     String[] actualCoprocessors = null;
500     boolean success = false;
501     for(int i = 0; i < 5; i++) {
502       if (tableName == null) {
503         //if no tableName specified, use all servers.
504         servers =
505             TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
506                 getOnlineServers();
507       } else {
508         servers = serversForTable(tableName);
509       }
510       boolean any_failed = false;
511       for(Map.Entry<ServerName,ServerLoad> server: servers.entrySet()) {
512         actualCoprocessors = server.getValue().getRsCoprocessors();
513         if (!Arrays.equals(actualCoprocessors, expectedCoprocessors)) {
514           LOG.debug("failed comparison: actual: " +
515               Arrays.toString(actualCoprocessors) +
516               " ; expected: " + Arrays.toString(expectedCoprocessors));
517           any_failed = true;
518           break;
519         }
520       }
521       if (any_failed == false) {
522         success = true;
523         break;
524       }
525       LOG.debug("retrying after failed comparison: " + i);
526       Thread.sleep(1000);
527     }
528     assertTrue(success);
529   }
530 
531   @Test
532   public void testMasterCoprocessorsReported() {
533     // HBASE 4070: Improve region server metrics to report loaded coprocessors
534     // to master: verify that the master is reporting the correct set of
535     // loaded coprocessors.
536     final String loadedMasterCoprocessorsVerify =
537         "[" + masterCoprocessor.getSimpleName() + "]";
538     String loadedMasterCoprocessors =
539         java.util.Arrays.toString(
540             TEST_UTIL.getHBaseCluster().getMaster().getCoprocessors());
541     assertEquals(loadedMasterCoprocessorsVerify, loadedMasterCoprocessors);
542   }
543 
544   private void waitForTable(TableName name) throws InterruptedException, IOException {
545     // First wait until all regions are online
546     TEST_UTIL.waitTableEnabled(name.getName());
547     // Now wait a bit longer for the coprocessor hosts to load the CPs
548     Thread.sleep(1000);
549   }
550 
551 }
552