1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.lang.reflect.InvocationTargetException;
27  import java.net.BindException;
28  import java.util.Collection;
29  import java.util.Set;
30  import java.util.concurrent.CopyOnWriteArraySet;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.HMsg;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.HServerAddress;
40  import org.apache.hadoop.hbase.HServerInfo;
41  import org.apache.hadoop.hbase.MiniHBaseCluster;
42  import org.apache.hadoop.hbase.MiniHBaseCluster.MiniHBaseClusterRegionServer;
43  import org.apache.hadoop.hbase.client.Get;
44  import org.apache.hadoop.hbase.client.HTable;
45  import org.apache.hadoop.hbase.client.Put;
46  import org.apache.hadoop.hbase.client.Result;
47  import org.apache.hadoop.hbase.client.ResultScanner;
48  import org.apache.hadoop.hbase.client.Scan;
49  import org.apache.hadoop.hbase.regionserver.HRegion;
50  import org.apache.hadoop.hbase.regionserver.HRegionServer;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.apache.hadoop.hbase.util.Threads;
53  import org.apache.hadoop.hbase.util.Writables;
54  import org.junit.AfterClass;
55  import org.junit.Assert;
56  import org.junit.Before;
57  import org.junit.BeforeClass;
58  import org.junit.Test;
59  
60  /**
61   * Test transitions of state across the master.  Sets up the cluster once and
62   * then runs a couple of tests.
63   */
64  public class TestMasterTransitions {
65    private static final Log LOG = LogFactory.getLog(TestMasterTransitions.class);
66    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
67    private static final String TABLENAME = "master_transitions";
68    private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
69      Bytes.toBytes("b"), Bytes.toBytes("c")};
70  
71    /**
72     * Start up a mini cluster and put a small table of many empty regions into it.
73     * @throws Exception
74     */
75    @BeforeClass public static void beforeAllTests() throws Exception {
76      TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
77      // Parcel out the regions, don't give them out in big lumps.  We've only
78      // a few in this test.  Let a couple of cycles pass is more realistic and
79      // gives stuff a chance to work.
80      TEST_UTIL.getConfiguration().setInt("hbase.regions.percheckin", 2);
81      // Start a cluster of two regionservers.
82      TEST_UTIL.startMiniCluster(2);
83      // Create a table of three families.  This will assign a region.
84      TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
85      HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
86      int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
87      TEST_UTIL.waitUntilAllRegionsAssigned(countOfRegions);
88      addToEachStartKey(countOfRegions);
89    }
90  
91    @AfterClass public static void afterAllTests() throws IOException {
92      TEST_UTIL.shutdownMiniCluster();
93    }
94  
95    @Before public void setup() throws IOException {
96      TEST_UTIL.ensureSomeRegionServersAvailable(2);
97    }
98  
99    /**
100    * Listener for regionserver events testing hbase-2428 (Infinite loop of
101    * region closes if META region is offline).  In particular, listen
102    * for the close of the 'metaServer' and when it comes in, requeue it with a
103    * delay as though there were an issue processing the shutdown.  As part of
104    * the requeuing,  send over a close of a region on 'otherServer' so it comes
105    * into a master that has its meta region marked as offline.
106    */
107   static class HBase2428Listener implements RegionServerOperationListener {
108     // Map of what we've delayed so we don't do do repeated delays.
109     private final Set<RegionServerOperation> postponed =
110       new CopyOnWriteArraySet<RegionServerOperation>();
111     private boolean done = false;;
112     private boolean metaShutdownReceived = false;
113     private final HServerAddress metaAddress;
114     private final MiniHBaseCluster cluster;
115     private final int otherServerIndex;
116     private final HRegionInfo hri;
117     private int closeCount = 0;
118     static final int SERVER_DURATION = 3 * 1000;
119     static final int CLOSE_DURATION = 1 * 1000;
120  
121     HBase2428Listener(final MiniHBaseCluster c, final HServerAddress metaAddress,
122         final HRegionInfo closingHRI, final int otherServerIndex) {
123       this.cluster = c;
124       this.metaAddress = metaAddress;
125       this.hri = closingHRI;
126       this.otherServerIndex = otherServerIndex;
127     }
128 
129     @Override
130     public boolean process(final RegionServerOperation op) throws IOException {
131       // If a regionserver shutdown and its of the meta server, then we want to
132       // delay the processing of the shutdown and send off a close of a region on
133       // the 'otherServer.
134       boolean result = true;
135       if (op instanceof ProcessServerShutdown) {
136         ProcessServerShutdown pss = (ProcessServerShutdown)op;
137         if (pss.getDeadServerAddress().equals(this.metaAddress)) {
138           // Don't postpone more than once.
139           if (!this.postponed.contains(pss)) {
140             // Close some region.
141             this.cluster.addMessageToSendRegionServer(this.otherServerIndex,
142               new HMsg(HMsg.Type.MSG_REGION_CLOSE, hri,
143               Bytes.toBytes("Forcing close in test")));
144             this.postponed.add(pss);
145             // Put off the processing of the regionserver shutdown processing.
146             pss.setDelay(SERVER_DURATION);
147             this.metaShutdownReceived = true;
148             // Return false.  This will add this op to the delayed queue.
149             result = false;
150           }
151         }
152       } else {
153         // Have the close run frequently.
154         if (isWantedCloseOperation(op) != null) {
155           op.setDelay(CLOSE_DURATION);
156           // Count how many times it comes through here.
157           this.closeCount++;
158         }
159       }
160       return result;
161     }
162 
163     public void processed(final RegionServerOperation op) {
164       if (isWantedCloseOperation(op) != null) return;
165       this.done = true;
166     }
167 
168     /*
169      * @param op
170      * @return Null if not the wanted ProcessRegionClose, else <code>op</code>
171      * cast as a ProcessRegionClose.
172      */
173     private ProcessRegionClose isWantedCloseOperation(final RegionServerOperation op) {
174       // Count every time we get a close operation.
175       if (op instanceof ProcessRegionClose) {
176         ProcessRegionClose c = (ProcessRegionClose)op;
177         if (c.regionInfo.equals(hri)) {
178           return c;
179         }
180       }
181       return null;
182     }
183 
184     boolean isDone() {
185       return this.done;
186     }
187 
188     boolean isMetaShutdownReceived() {
189       return metaShutdownReceived;
190     }
191 
192     int getCloseCount() {
193       return this.closeCount;
194     }
195 
196     @Override
197     public boolean process(HServerInfo serverInfo, HMsg incomingMsg) {
198       return true;
199     }
200   }
201 
202   /**
203    * In 2428, the meta region has just been set offline and then a close comes
204    * in.
205    * @see <a href="https://issues.apache.org/jira/browse/HBASE-2428">HBASE-2428</a> 
206    */
207   @Test (timeout=300000) public void testRegionCloseWhenNoMetaHBase2428()
208   throws Exception {
209     LOG.info("Running testRegionCloseWhenNoMetaHBase2428");
210     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
211     final HMaster master = cluster.getMaster();
212     int metaIndex = cluster.getServerWithMeta();
213     // Figure the index of the server that is not server the .META.
214     int otherServerIndex = -1;
215     for (int i = 0; i < cluster.getRegionServerThreads().size(); i++) {
216       if (i == metaIndex) continue;
217       otherServerIndex = i;
218       break;
219     }
220     final HRegionServer otherServer = cluster.getRegionServer(otherServerIndex);
221     final HRegionServer metaHRS = cluster.getRegionServer(metaIndex);
222 
223     // Get a region out on the otherServer.
224     final HRegionInfo hri =
225       otherServer.getOnlineRegions().iterator().next().getRegionInfo();
226  
227     // Add our RegionServerOperationsListener
228     HBase2428Listener listener = new HBase2428Listener(cluster,
229       metaHRS.getHServerInfo().getServerAddress(), hri, otherServerIndex);
230     master.getRegionServerOperationQueue().
231       registerRegionServerOperationListener(listener);
232     try {
233       // Now close the server carrying meta.
234       cluster.abortRegionServer(metaIndex);
235 
236       // First wait on receipt of meta server shutdown message.
237       while(!listener.metaShutdownReceived) Threads.sleep(100);
238       while(!listener.isDone()) Threads.sleep(10);
239       // We should not have retried the close more times than it took for the
240       // server shutdown message to exit the delay queue and get processed
241       // (Multiple by two to add in some slop in case of GC or something).
242       assertTrue(listener.getCloseCount() > 1);
243       assertTrue(listener.getCloseCount() <
244         ((HBase2428Listener.SERVER_DURATION/HBase2428Listener.CLOSE_DURATION) * 2));
245 
246       // Assert the closed region came back online
247       assertRegionIsBackOnline(hri);
248     } finally {
249       master.getRegionServerOperationQueue().
250         unregisterRegionServerOperationListener(listener);
251     }
252   }
253 
254   /**
255    * Test adding in a new server before old one on same host+port is dead.
256    * Make the test more onerous by having the server under test carry the meta.
257    * If confusion between old and new, purportedly meta never comes back.  Test
258    * that meta gets redeployed.
259    */
260   @Test (timeout=300000) public void testAddingServerBeforeOldIsDead2413()
261   throws IOException {
262     LOG.info("Running testAddingServerBeforeOldIsDead2413");
263     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
264     int count = count();
265     int metaIndex = cluster.getServerWithMeta();
266     MiniHBaseClusterRegionServer metaHRS =
267       (MiniHBaseClusterRegionServer)cluster.getRegionServer(metaIndex);
268     int port = metaHRS.getServerInfo().getServerAddress().getPort();
269     Configuration c = TEST_UTIL.getConfiguration();
270     String oldPort = c.get(HConstants.REGIONSERVER_PORT, "0");
271     try {
272       LOG.info("KILLED=" + metaHRS);
273       metaHRS.kill();
274       c.set(HConstants.REGIONSERVER_PORT, Integer.toString(port));
275       // Try and start new regionserver.  It might clash with the old
276       // regionserver port so keep trying to get past the BindException.
277       HRegionServer hrs = null;
278       while (true) {
279         try {
280           hrs = cluster.startRegionServer().getRegionServer();
281           break;
282         } catch (IOException e) {
283           if (e.getCause() != null && e.getCause() instanceof InvocationTargetException) {
284             InvocationTargetException ee = (InvocationTargetException)e.getCause();
285             if (ee.getCause() != null && ee.getCause() instanceof BindException) {
286               LOG.info("BindException; retrying: " + e.toString());
287             }
288           }
289         }
290       }
291       LOG.info("STARTED=" + hrs);
292       // Wait until he's been given at least 3 regions before we go on to try
293       // and count rows in table.
294       while (hrs.getOnlineRegions().size() < 3) Threads.sleep(100);
295       LOG.info(hrs.toString() + " has " + hrs.getOnlineRegions().size() +
296         " regions");
297       assertEquals(count, count());
298     } finally {
299       c.set(HConstants.REGIONSERVER_PORT, oldPort);
300     }
301   }
302 
303 
304   /**
305    * HBase2482 is about outstanding region openings.  If any are outstanding
306    * when a regionserver goes down, then they'll never deploy.  They'll be
307    * stuck in the regions-in-transition list for ever.  This listener looks
308    * for a region opening HMsg and if its from the server passed on construction,
309    * then we kill it.  It also looks out for a close message on the victim
310    * server because that signifies start of the fireworks.
311    */
312   static class HBase2482Listener implements RegionServerOperationListener {
313     private final HRegionServer victim;
314     private boolean abortSent = false;
315     // We closed regions on new server.
316     private volatile boolean closed = false;
317     // Copy of regions on new server
318     private final Collection<HRegion> copyOfOnlineRegions;
319     // This is the region that was in transition on the server we aborted. Test
320     // passes if this region comes back online successfully.
321     private HRegionInfo regionToFind;
322 
323     HBase2482Listener(final HRegionServer victim) {
324       this.victim = victim;
325       // Copy regions currently open on this server so I can notice when
326       // there is a close.
327       this.copyOfOnlineRegions =
328         this.victim.getCopyOfOnlineRegionsSortedBySize().values();
329     }
330  
331     @Override
332     public boolean process(HServerInfo serverInfo, HMsg incomingMsg) {
333       if (!victim.getServerInfo().equals(serverInfo) ||
334           this.abortSent || !this.closed) {
335         return true;
336       }
337       if (!incomingMsg.isType(HMsg.Type.MSG_REPORT_PROCESS_OPEN)) return true;
338       // Save the region that is in transition so can test later it came back.
339       this.regionToFind = incomingMsg.getRegionInfo();
340       String msg = "ABORTING " + this.victim + " because got a " +
341         HMsg.Type.MSG_REPORT_PROCESS_OPEN + " on this server for " +
342         incomingMsg.getRegionInfo().getRegionNameAsString();
343       this.victim.abort(msg);
344       this.abortSent = true;
345       return true;
346     }
347 
348     @Override
349     public boolean process(RegionServerOperation op) throws IOException {
350       return true;
351     }
352 
353     @Override
354     public void processed(RegionServerOperation op) {
355       if (this.closed || !(op instanceof ProcessRegionClose)) return;
356       ProcessRegionClose close = (ProcessRegionClose)op;
357       for (HRegion r: this.copyOfOnlineRegions) {
358         if (r.getRegionInfo().equals(close.regionInfo)) {
359           // We've closed one of the regions that was on the victim server.
360           // Now can start testing for when all regions are back online again
361           LOG.info("Found close of " +
362             r.getRegionInfo().getRegionNameAsString() +
363             "; setting close happened flag");
364           this.closed = true;
365           break;
366         }
367       }
368     }
369   }
370 
371   /**
372    * In 2482, a RS with an opening region on it dies.  The said region is then
373    * stuck in the master's regions-in-transition and never leaves it.  This
374    * test works by bringing up a new regionserver, waiting for the load
375    * balancer to give it some regions.  Then, we close all on the new server.
376    * After sending all the close messages, we send the new regionserver the
377    * special blocking message so it can not process any more messages.
378    * Meantime reopening of the just-closed regions is backed up on the new
379    * server.  Soon as master gets an opening region from the new regionserver,
380    * we kill it.  We then wait on all regions to come back on line.  If bug
381    * is fixed, this should happen soon as the processing of the killed server is
382    * done.
383    * @see <a href="https://issues.apache.org/jira/browse/HBASE-2482">HBASE-2482</a> 
384    */
385   @Test (timeout=300000) public void testKillRSWithOpeningRegion2482()
386   throws Exception {
387     LOG.info("Running testKillRSWithOpeningRegion2482");
388     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
389     if (cluster.getLiveRegionServerThreads().size() < 2) {
390       // Need at least two servers.
391       cluster.startRegionServer();
392     }
393     // Count how many regions are online.  They need to be all back online for
394     // this test to succeed.
395     int countOfMetaRegions = countOfMetaRegions();
396     // Add a listener on the server.
397     HMaster m = cluster.getMaster();
398     // Start new regionserver.
399     MiniHBaseClusterRegionServer hrs =
400       (MiniHBaseClusterRegionServer)cluster.startRegionServer().getRegionServer();
401     LOG.info("Started new regionserver: " + hrs.toString());
402     // Wait until has some regions before proceeding.  Balancer will give it some.
403     int minimumRegions =
404       countOfMetaRegions/(cluster.getRegionServerThreads().size() * 2);
405     while (hrs.getOnlineRegions().size() < minimumRegions) Threads.sleep(100);
406     // Set the listener only after some regions have been opened on new server.
407     HBase2482Listener listener = new HBase2482Listener(hrs);
408     m.getRegionServerOperationQueue().
409       registerRegionServerOperationListener(listener);
410     try {
411       // Go close all non-catalog regions on this new server
412       closeAllNonCatalogRegions(cluster, hrs);
413       // After all closes, add blocking message before the region opens start to
414       // come in.
415       cluster.addMessageToSendRegionServer(hrs,
416         new HMsg(HMsg.Type.TESTING_MSG_BLOCK_RS));
417       // Wait till one of the above close messages has an effect before we start
418       // wait on all regions back online.
419       while (!listener.closed) Threads.sleep(100);
420       LOG.info("Past close");
421       // Make sure the abort server message was sent.
422       while(!listener.abortSent) Threads.sleep(100);
423       LOG.info("Past abort send; waiting on all regions to redeploy");
424       // Now wait for regions to come back online.
425       assertRegionIsBackOnline(listener.regionToFind);
426     } finally {
427       m.getRegionServerOperationQueue().
428         unregisterRegionServerOperationListener(listener);
429     }
430   }
431 
432   /*
433    * @return Count of all non-catalog regions on the designated server
434    */
435   private int closeAllNonCatalogRegions(final MiniHBaseCluster cluster,
436     final MiniHBaseCluster.MiniHBaseClusterRegionServer hrs)
437   throws IOException {
438     int countOfRegions = 0;
439     for (HRegion r: hrs.getOnlineRegions()) {
440       if (r.getRegionInfo().isMetaRegion()) continue;
441       cluster.addMessageToSendRegionServer(hrs,
442         new HMsg(HMsg.Type.MSG_REGION_CLOSE, r.getRegionInfo()));
443       LOG.info("Sent close of " + r.getRegionInfo().getRegionNameAsString() +
444         " on " + hrs.toString());
445       countOfRegions++;
446     }
447     return countOfRegions;
448   }
449 
450   private void assertRegionIsBackOnline(final HRegionInfo hri)
451   throws IOException {
452     // Region should have an entry in its startkey because of addRowToEachRegion.
453     byte [] row = getStartKey(hri);
454     HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
455     Get g =  new Get(row);
456     assertTrue((t.get(g)).size() > 0);
457   }
458 
459   /*
460    * @return Count of regions in meta table.
461    * @throws IOException
462    */
463   private static int countOfMetaRegions()
464   throws IOException {
465     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
466       HConstants.META_TABLE_NAME);
467     int rows = 0;
468     Scan scan = new Scan();
469     scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
470     ResultScanner s = meta.getScanner(scan);
471     for (Result r = null; (r = s.next()) != null;) {
472       byte [] b =
473         r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
474       if (b == null || b.length <= 0) break;
475       rows++;
476     }
477     s.close();
478     return rows;
479   }
480 
481   /*
482    * Add to each of the regions in .META. a value.  Key is the startrow of the
483    * region (except its 'aaa' for first region).  Actual value is the row name.
484    * @param expected
485    * @return
486    * @throws IOException
487    */
488   private static int addToEachStartKey(final int expected) throws IOException {
489     HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
490     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
491         HConstants.META_TABLE_NAME);
492     int rows = 0;
493     Scan scan = new Scan();
494     scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
495     ResultScanner s = meta.getScanner(scan);
496     for (Result r = null; (r = s.next()) != null;) {
497       byte [] b =
498         r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
499       if (b == null || b.length <= 0) break;
500       HRegionInfo hri = Writables.getHRegionInfo(b);
501       // If start key, add 'aaa'.
502       byte [] row = getStartKey(hri);
503       Put p = new Put(row);
504       p.add(getTestFamily(), getTestQualifier(), row);
505       t.put(p);
506       rows++;
507     }
508     s.close();
509     Assert.assertEquals(expected, rows);
510     return rows;
511   }
512 
513   /*
514    * @return Count of rows in TABLENAME
515    * @throws IOException
516    */
517   private static int count() throws IOException {
518     HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
519     int rows = 0;
520     Scan scan = new Scan();
521     ResultScanner s = t.getScanner(scan);
522     for (Result r = null; (r = s.next()) != null;) {
523       rows++;
524     }
525     s.close();
526     LOG.info("Counted=" + rows);
527     return rows;
528   }
529 
530   /*
531    * @param hri
532    * @return Start key for hri (If start key is '', then return 'aaa'.
533    */
534   private static byte [] getStartKey(final HRegionInfo hri) {
535     return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
536         Bytes.toBytes("aaa"): hri.getStartKey();
537   }
538 
539   private static byte [] getTestFamily() {
540     return FAMILIES[0];
541   }
542 
543   private static byte [] getTestQualifier() {
544     return getTestFamily();
545   }
546 }