View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertNull;
25  import static org.junit.Assert.assertTrue;
26  import static org.junit.Assert.fail;
27  
28  import java.io.IOException;
29  import java.lang.reflect.InvocationTargetException;
30  import java.lang.reflect.Method;
31  import java.util.List;
32  import java.util.Map;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.client.Get;
38  import org.apache.hadoop.hbase.client.HBaseAdmin;
39  import org.apache.hadoop.hbase.client.HConnection;
40  import org.apache.hadoop.hbase.client.HConnectionManager;
41  import org.apache.hadoop.hbase.client.HTable;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.client.Result;
44  import org.apache.hadoop.hbase.client.ResultScanner;
45  import org.apache.hadoop.hbase.client.Scan;
46  import org.apache.hadoop.hbase.master.HMaster;
47  import org.apache.hadoop.hbase.master.LoadBalancer;
48  import org.apache.hadoop.hbase.master.balancer.DefaultLoadBalancer;
49  import org.apache.hadoop.hbase.util.Bytes;
50  import org.apache.hadoop.hbase.util.Threads;
51  import org.apache.hadoop.hbase.zookeeper.EmptyWatcher;
52  import org.apache.hadoop.hbase.zookeeper.ZKAssign;
53  import org.apache.hadoop.hbase.zookeeper.ZKConfig;
54  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
55  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
56  import org.apache.zookeeper.CreateMode;
57  import org.apache.zookeeper.KeeperException;
58  import org.apache.zookeeper.ZooDefs;
59  import org.apache.zookeeper.ZooKeeper;
60  import org.apache.zookeeper.ZooKeeper.States;
61  import org.apache.zookeeper.data.ACL;
62  import org.apache.zookeeper.data.Stat;
63  import org.junit.AfterClass;
64  import org.junit.Assert;
65  import org.junit.Before;
66  import org.junit.BeforeClass;
67  import org.junit.Test;
68  import org.junit.experimental.categories.Category;
69  
70  
71  
72  @Category(LargeTests.class)
73  public class TestZooKeeper {
74    private final Log LOG = LogFactory.getLog(this.getClass());
75  
76    private final static HBaseTestingUtility
77        TEST_UTIL = new HBaseTestingUtility();
78  
79    /**
80     * @throws java.lang.Exception
81     */
82    @BeforeClass
83    public static void setUpBeforeClass() throws Exception {
84      // Test we can first start the ZK cluster by itself
85      Configuration conf = TEST_UTIL.getConfiguration();
86      TEST_UTIL.startMiniZKCluster();
87      conf.setBoolean("dfs.support.append", true);
88      conf.setInt(HConstants.ZK_SESSION_TIMEOUT, 1000);
89      conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, MockLoadBalancer.class,
90          LoadBalancer.class);
91      TEST_UTIL.startMiniCluster(2);
92    }
93  
94    /**
95     * @throws java.lang.Exception
96     */
97    @AfterClass
98    public static void tearDownAfterClass() throws Exception {
99      TEST_UTIL.shutdownMiniCluster();
100   }
101 
102   /**
103    * @throws java.lang.Exception
104    */
105   @Before
106   public void setUp() throws Exception {
107     TEST_UTIL.ensureSomeRegionServersAvailable(2);
108   }
109 
110 
111   private ZooKeeperWatcher getZooKeeperWatcher(HConnection c)
112   throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
113     Method getterZK = c.getClass().getDeclaredMethod("getKeepAliveZooKeeperWatcher");
114     getterZK.setAccessible(true);
115     return (ZooKeeperWatcher) getterZK.invoke(c);
116   }
117 
118 
119   /**
120    * See HBASE-1232 and http://wiki.apache.org/hadoop/ZooKeeper/FAQ#4.
121    * @throws IOException
122    * @throws InterruptedException
123    */
124   // fails frequently, disabled for now, see HBASE-6406
125   //@Test
126   public void testClientSessionExpired() throws Exception {
127     Configuration c = new Configuration(TEST_UTIL.getConfiguration());
128 
129     // We don't want to share the connection as we will check its state
130     c.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "1111");
131 
132     HConnection connection = HConnectionManager.getConnection(c);
133 
134     ZooKeeperWatcher connectionZK = getZooKeeperWatcher(connection);
135     LOG.info("ZooKeeperWatcher= 0x"+ Integer.toHexString(
136       connectionZK.hashCode()));
137     LOG.info("getRecoverableZooKeeper= 0x"+ Integer.toHexString(
138       connectionZK.getRecoverableZooKeeper().hashCode()));
139     LOG.info("session="+Long.toHexString(
140       connectionZK.getRecoverableZooKeeper().getSessionId()));
141 
142     TEST_UTIL.expireSession(connectionZK);
143 
144     LOG.info("Before using zkw state=" +
145       connectionZK.getRecoverableZooKeeper().getState());
146     // provoke session expiration by doing something with ZK
147     try {
148       connectionZK.getRecoverableZooKeeper().getZooKeeper().exists(
149         "/1/1", false);
150     } catch (KeeperException ignored) {
151     }
152 
153     // Check that the old ZK connection is closed, means we did expire
154     States state = connectionZK.getRecoverableZooKeeper().getState();
155     LOG.info("After using zkw state=" + state);
156     LOG.info("session="+Long.toHexString(
157       connectionZK.getRecoverableZooKeeper().getSessionId()));
158 
159     // It's asynchronous, so we may have to wait a little...
160     final long limit1 = System.currentTimeMillis() + 3000;
161     while (System.currentTimeMillis() < limit1 && state != States.CLOSED){
162       state = connectionZK.getRecoverableZooKeeper().getState();
163     }
164     LOG.info("After using zkw loop=" + state);
165     LOG.info("ZooKeeper should have timed out");
166     LOG.info("session="+Long.toHexString(
167       connectionZK.getRecoverableZooKeeper().getSessionId()));
168 
169     // It's surprising but sometimes we can still be in connected state.
170     // As it's known (even if not understood) we don't make the the test fail
171     // for this reason.)
172     // Assert.assertTrue("state=" + state, state == States.CLOSED);
173 
174     // Check that the client recovered
175     ZooKeeperWatcher newConnectionZK = getZooKeeperWatcher(connection);
176 
177     States state2 = newConnectionZK.getRecoverableZooKeeper().getState();
178     LOG.info("After new get state=" +state2);
179 
180     // As it's an asynchronous event we may got the same ZKW, if it's not
181     //  yet invalidated. Hence this loop.
182     final long limit2 = System.currentTimeMillis() + 3000;
183     while (System.currentTimeMillis() < limit2 &&
184       state2 != States.CONNECTED && state2 != States.CONNECTING) {
185 
186       newConnectionZK = getZooKeeperWatcher(connection);
187       state2 = newConnectionZK.getRecoverableZooKeeper().getState();
188     }
189     LOG.info("After new get state loop=" + state2);
190 
191     Assert.assertTrue(
192       state2 == States.CONNECTED || state2 == States.CONNECTING);
193 
194     connection.close();
195   }
196 
197   @Test (timeout = 60000)
198   public void testRegionServerSessionExpired() throws Exception {
199     LOG.info("Starting testRegionServerSessionExpired");
200     int metaIndex = TEST_UTIL.getMiniHBaseCluster().getServerWithMeta();
201     TEST_UTIL.expireRegionServerSession(metaIndex);
202     testSanity("testRegionServerSessionExpired");
203   }
204 
205   // @Test Disabled because seems to make no sense expiring master session
206   // and then trying to create table (down in testSanity); on master side
207   // it will fail because the master's session has expired -- St.Ack 07/24/2012
208   public void testMasterSessionExpired() throws Exception {
209     LOG.info("Starting testMasterSessionExpired");
210     TEST_UTIL.expireMasterSession();
211     testSanity("testMasterSessionExpired");
212   }
213 
214   /**
215    * Master recovery when the znode already exists. Internally, this
216    *  test differs from {@link #testMasterSessionExpired} because here
217    *  the master znode will exist in ZK.
218    */
219   @Test(timeout = 60000)
220   public void testMasterZKSessionRecoveryFailure() throws Exception {
221     LOG.info("Starting testMasterZKSessionRecoveryFailure");
222     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
223     HMaster m = cluster.getMaster();
224     m.abort("Test recovery from zk session expired",
225       new KeeperException.SessionExpiredException());
226     assertFalse(m.isStopped());
227     testSanity("testMasterZKSessionRecoveryFailure");
228   }
229 
230   /**
231    * Make sure we can use the cluster
232    * @throws Exception
233    */
234   private void testSanity(final String testName) throws Exception{
235     String tableName = testName + "_" + System.currentTimeMillis();
236     HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
237     HColumnDescriptor family = new HColumnDescriptor("fam");
238     desc.addFamily(family);
239     LOG.info("Creating table " + tableName);
240     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
241     try {
242       admin.createTable(desc);
243     } finally {
244       admin.close();
245     }
246 
247     HTable table =
248       new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
249     Put put = new Put(Bytes.toBytes("testrow"));
250     put.add(Bytes.toBytes("fam"),
251         Bytes.toBytes("col"), Bytes.toBytes("testdata"));
252     LOG.info("Putting table " + tableName);
253     table.put(put);
254     table.close();
255   }
256 
257   @Test
258   public void testMultipleZK()
259   throws IOException, NoSuchMethodException, InvocationTargetException, IllegalAccessException {
260     HTable localMeta =
261       new HTable(new Configuration(TEST_UTIL.getConfiguration()), TableName.META_TABLE_NAME);
262     Configuration otherConf = new Configuration(TEST_UTIL.getConfiguration());
263     otherConf.set(HConstants.ZOOKEEPER_QUORUM, "127.0.0.1");
264     HTable ipMeta = new HTable(otherConf, TableName.META_TABLE_NAME);
265 
266     // dummy, just to open the connection
267     final byte [] row = new byte [] {'r'};
268     localMeta.exists(new Get(row));
269     ipMeta.exists(new Get(row));
270 
271     // make sure they aren't the same
272     ZooKeeperWatcher z1 =
273       getZooKeeperWatcher(HConnectionManager.getConnection(localMeta.getConfiguration()));
274     ZooKeeperWatcher z2 =
275       getZooKeeperWatcher(HConnectionManager.getConnection(otherConf));
276     assertFalse(z1 == z2);
277     assertFalse(z1.getQuorum().equals(z2.getQuorum()));
278 
279     localMeta.close();
280     ipMeta.close();
281   }
282 
283   /**
284    * Create a znode with data
285    * @throws Exception
286    */
287   @Test
288   public void testCreateWithParents() throws Exception {
289     ZooKeeperWatcher zkw =
290         new ZooKeeperWatcher(new Configuration(TEST_UTIL.getConfiguration()),
291             TestZooKeeper.class.getName(), null);
292     byte[] expectedData = new byte[] { 1, 2, 3 };
293     ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4/testCreateWithParents", expectedData);
294     byte[] data = ZKUtil.getData(zkw, "/l1/l2/l3/l4/testCreateWithParents");
295     assertTrue(Bytes.equals(expectedData, data));
296     ZKUtil.deleteNodeRecursively(zkw, "/l1");
297 
298     ZKUtil.createWithParents(zkw, "/testCreateWithParents", expectedData);
299     data = ZKUtil.getData(zkw, "/testCreateWithParents");
300     assertTrue(Bytes.equals(expectedData, data));
301     ZKUtil.deleteNodeRecursively(zkw, "/testCreateWithParents");
302   }
303 
304   /**
305    * Create a bunch of znodes in a hierarchy, try deleting one that has childs (it will fail), then
306    * delete it recursively, then delete the last znode
307    * @throws Exception
308    */
309   @Test
310   public void testZNodeDeletes() throws Exception {
311     ZooKeeperWatcher zkw = new ZooKeeperWatcher(
312       new Configuration(TEST_UTIL.getConfiguration()),
313       TestZooKeeper.class.getName(), null);
314     ZKUtil.createWithParents(zkw, "/l1/l2/l3/l4");
315     try {
316       ZKUtil.deleteNode(zkw, "/l1/l2");
317       fail("We should not be able to delete if znode has childs");
318     } catch (KeeperException ex) {
319       assertNotNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
320     }
321     ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
322     // make sure it really is deleted
323     assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2/l3/l4", null));
324 
325     // do the same delete again and make sure it doesn't crash
326     ZKUtil.deleteNodeRecursively(zkw, "/l1/l2");
327 
328     ZKUtil.deleteNode(zkw, "/l1");
329     assertNull(ZKUtil.getDataNoWatch(zkw, "/l1/l2", null));
330   }
331 
332   @Test
333   public void testClusterKey() throws Exception {
334     testKey("server", "2181", "hbase");
335     testKey("server1,server2,server3", "2181", "hbase");
336     try {
337       ZKUtil.transformClusterKey("2181:hbase");
338     } catch (IOException ex) {
339       // OK
340     }
341   }
342 
343   private void testKey(String ensemble, String port, String znode)
344       throws IOException {
345     Configuration conf = new Configuration();
346     String key = ensemble+":"+port+":"+znode;
347     String[] parts = ZKUtil.transformClusterKey(key);
348     assertEquals(ensemble, parts[0]);
349     assertEquals(port, parts[1]);
350     assertEquals(znode, parts[2]);
351     ZKUtil.applyClusterKeyToConf(conf, key);
352     assertEquals(parts[0], conf.get(HConstants.ZOOKEEPER_QUORUM));
353     assertEquals(parts[1], conf.get(HConstants.ZOOKEEPER_CLIENT_PORT));
354     assertEquals(parts[2], conf.get(HConstants.ZOOKEEPER_ZNODE_PARENT));
355     String reconstructedKey = ZKUtil.getZooKeeperClusterKey(conf);
356     assertEquals(key, reconstructedKey);
357   }
358 
359   /**
360    * A test for HBASE-3238
361    * @throws IOException A connection attempt to zk failed
362    * @throws InterruptedException One of the non ZKUtil actions was interrupted
363    * @throws KeeperException Any of the zookeeper connections had a
364    * KeeperException
365    */
366   @Test
367   public void testCreateSilentIsReallySilent() throws InterruptedException,
368       KeeperException, IOException {
369     Configuration c = TEST_UTIL.getConfiguration();
370 
371     String aclZnode = "/aclRoot";
372     String quorumServers = ZKConfig.getZKQuorumServersString(c);
373     int sessionTimeout = 5 * 1000; // 5 seconds
374     ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
375     zk.addAuthInfo("digest", "hbase:rox".getBytes());
376 
377     // Assumes the  root of the ZooKeeper space is writable as it creates a node
378     // wherever the cluster home is defined.
379     ZooKeeperWatcher zk2 = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
380       "testMasterAddressManagerFromZK", null);
381 
382     // Save the previous ACL
383     Stat s =  null;
384     List<ACL> oldACL = null;
385     while (true) {
386       try {
387         s = new Stat();
388         oldACL = zk.getACL("/", s);
389         break;
390       } catch (KeeperException e) {
391         switch (e.code()) {
392           case CONNECTIONLOSS:
393           case SESSIONEXPIRED:
394           case OPERATIONTIMEOUT:
395             LOG.warn("Possibly transient ZooKeeper exception", e);
396             Threads.sleep(100);
397             break;
398          default:
399             throw e;
400         }
401       }
402     }
403 
404     // I set this acl after the attempted creation of the cluster home node.
405     // Add retries in case of retryable zk exceptions.
406     while (true) {
407       try {
408         zk.setACL("/", ZooDefs.Ids.CREATOR_ALL_ACL, -1);
409         break;
410       } catch (KeeperException e) {
411         switch (e.code()) {
412           case CONNECTIONLOSS:
413           case SESSIONEXPIRED:
414           case OPERATIONTIMEOUT:
415             LOG.warn("Possibly transient ZooKeeper exception: " + e);
416             Threads.sleep(100);
417             break;
418          default:
419             throw e;
420         }
421       }
422     }
423 
424     while (true) {
425       try {
426         zk.create(aclZnode, null, ZooDefs.Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT);
427         break;
428       } catch (KeeperException e) {
429         switch (e.code()) {
430           case CONNECTIONLOSS:
431           case SESSIONEXPIRED:
432           case OPERATIONTIMEOUT:
433             LOG.warn("Possibly transient ZooKeeper exception: " + e);
434             Threads.sleep(100);
435             break;
436          default:
437             throw e;
438         }
439       }
440     }
441     zk.close();
442     ZKUtil.createAndFailSilent(zk2, aclZnode);
443 
444     // Restore the ACL
445     ZooKeeper zk3 = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
446     zk3.addAuthInfo("digest", "hbase:rox".getBytes());
447     try {
448       zk3.setACL("/", oldACL, -1);
449     } finally {
450       zk3.close();
451     }
452  }
453 
454   /**
455    * Test should not fail with NPE when getChildDataAndWatchForNewChildren
456    * invoked with wrongNode
457    */
458   @Test
459   @SuppressWarnings("deprecation")
460   public void testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE()
461       throws Exception {
462     ZooKeeperWatcher zkw = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(),
463         "testGetChildDataAndWatchForNewChildrenShouldNotThrowNPE", null);
464     ZKUtil.getChildDataAndWatchForNewChildren(zkw, "/wrongNode");
465   }
466 
467   /**
468    * Tests that the master does not call retainAssignment after recovery from expired zookeeper
469    * session. Without the HBASE-6046 fix master always tries to assign all the user regions by
470    * calling retainAssignment.
471    */
472   @Test
473   public void testRegionAssignmentAfterMasterRecoveryDueToZKExpiry() throws Exception {
474     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
475     cluster.startRegionServer();
476     HMaster m = cluster.getMaster();
477     ZooKeeperWatcher zkw = m.getZooKeeperWatcher();
478     int expectedNumOfListeners = zkw.getNumberOfListeners();
479     // now the cluster is up. So assign some regions.
480     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
481     try {
482       byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
483         Bytes.toBytes("c"), Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
484         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j") };
485       String tableName = "testRegionAssignmentAfterMasterRecoveryDueToZKExpiry";
486       admin.createTable(new HTableDescriptor(TableName.valueOf(tableName)), SPLIT_KEYS);
487       ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
488       ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
489       m.getZooKeeperWatcher().close();
490       MockLoadBalancer.retainAssignCalled = false;
491       m.abort("Test recovery from zk session expired",
492         new KeeperException.SessionExpiredException());
493       assertFalse(m.isStopped());
494       // The recovered master should not call retainAssignment, as it is not a
495       // clean startup.
496       assertFalse("Retain assignment should not be called", MockLoadBalancer.retainAssignCalled);
497       // number of listeners should be same as the value before master aborted
498       assertEquals(expectedNumOfListeners, zkw.getNumberOfListeners());
499     } finally {
500       admin.close();
501     }
502   }
503 
504   /**
505    * Tests whether the logs are split when master recovers from a expired zookeeper session and an
506    * RS goes down.
507    */
508   @Test(timeout = 240000)
509   public void testLogSplittingAfterMasterRecoveryDueToZKExpiry() throws IOException,
510       KeeperException, InterruptedException {
511     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
512     cluster.startRegionServer();
513     HMaster m = cluster.getMaster();
514     // now the cluster is up. So assign some regions.
515     HBaseAdmin admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
516     HTable table = null;
517     try {
518       byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("1"), Bytes.toBytes("2"),
519         Bytes.toBytes("3"), Bytes.toBytes("4"), Bytes.toBytes("5") };
520 
521       String tableName = "testLogSplittingAfterMasterRecoveryDueToZKExpiry";
522       HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
523       HColumnDescriptor hcd = new HColumnDescriptor("col");
524       htd.addFamily(hcd);
525       admin.createTable(htd, SPLIT_KEYS);
526       ZooKeeperWatcher zooKeeperWatcher = HBaseTestingUtility.getZooKeeperWatcher(TEST_UTIL);
527       ZKAssign.blockUntilNoRIT(zooKeeperWatcher);
528       table = new HTable(TEST_UTIL.getConfiguration(), tableName);
529       Put p;
530       int numberOfPuts;
531       for (numberOfPuts = 0; numberOfPuts < 6; numberOfPuts++) {
532         p = new Put(Bytes.toBytes(numberOfPuts));
533         p.add(Bytes.toBytes("col"), Bytes.toBytes("ql"), Bytes.toBytes("value" + numberOfPuts));
534         table.put(p);
535       }
536       m.getZooKeeperWatcher().close();
537       m.abort("Test recovery from zk session expired",
538         new KeeperException.SessionExpiredException());
539       assertFalse(m.isStopped());
540       cluster.getRegionServer(0).abort("Aborting");
541       // Without patch for HBASE-6046 this test case will always timeout
542       // with patch the test case should pass.
543       Scan scan = new Scan();
544       int numberOfRows = 0;
545       ResultScanner scanner = table.getScanner(scan);
546       Result[] result = scanner.next(1);
547       while (result != null && result.length > 0) {
548         numberOfRows++;
549         result = scanner.next(1);
550       }
551       assertEquals("Number of rows should be equal to number of puts.", numberOfPuts,
552         numberOfRows);
553     } finally {
554       if (table != null) table.close();
555       admin.close();
556     }
557   }
558 
559   static class MockLoadBalancer extends DefaultLoadBalancer {
560     static boolean retainAssignCalled = false;
561 
562     @Override
563     public Map<ServerName, List<HRegionInfo>> retainAssignment(
564         Map<HRegionInfo, ServerName> regions, List<ServerName> servers) {
565       retainAssignCalled = true;
566       return super.retainAssignment(regions, servers);
567     }
568   }
569 
570 }
571