1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  
23  import java.io.IOException;
24  import java.util.Collection;
25  import java.util.concurrent.atomic.AtomicBoolean;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.MiniHBaseCluster;
34  import org.apache.hadoop.hbase.client.HTable;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.Result;
37  import org.apache.hadoop.hbase.client.ResultScanner;
38  import org.apache.hadoop.hbase.client.Scan;
39  import org.apache.hadoop.hbase.executor.EventHandler;
40  import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
41  import org.apache.hadoop.hbase.executor.EventHandler.EventType;
42  import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo;
43  import org.apache.hadoop.hbase.regionserver.HRegionServer;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.Threads;
46  import org.apache.hadoop.hbase.util.Writables;
47  import org.junit.AfterClass;
48  import org.junit.Assert;
49  import org.junit.Before;
50  import org.junit.BeforeClass;
51  import org.junit.Test;
52  import static org.junit.Assert.assertTrue;
53  
54  /**
55   * Test open and close of regions using zk.
56   */
57  public class TestZKBasedOpenCloseRegion {
58    private static final Log LOG = LogFactory.getLog(TestZKBasedOpenCloseRegion.class);
59    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
60    private static final String TABLENAME = "TestZKBasedOpenCloseRegion";
61    private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
62      Bytes.toBytes("b"), Bytes.toBytes("c")};
63    private static int countOfRegions;
64  
65    @BeforeClass public static void beforeAllTests() throws Exception {
66      Configuration c = TEST_UTIL.getConfiguration();
67      c.setBoolean("dfs.support.append", true);
68      c.setInt("hbase.regionserver.info.port", 0);
69      TEST_UTIL.startMiniCluster(2);
70      TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
71      HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
72      countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
73      waitUntilAllRegionsAssigned();
74      addToEachStartKey(countOfRegions);
75    }
76  
77    @AfterClass public static void afterAllTests() throws IOException {
78      TEST_UTIL.shutdownMiniCluster();
79    }
80  
81    @Before public void setup() throws IOException {
82      if (TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size() < 2) {
83        // Need at least two servers.
84        LOG.info("Started new server=" +
85          TEST_UTIL.getHBaseCluster().startRegionServer());
86  
87      }
88      waitUntilAllRegionsAssigned();
89    }
90  
91    /**
92     * Test we reopen a region once closed.
93     * @throws Exception
94     */
95    @Test (timeout=300000) public void testReOpenRegion()
96    throws Exception {
97      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
98      LOG.info("Number of region servers = " +
99        cluster.getLiveRegionServerThreads().size());
100 
101     int rsIdx = 0;
102     HRegionServer regionServer =
103       TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
104     HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
105     LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
106 
107     AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
108     AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
109 
110     EventHandlerListener closeListener =
111       new ReopenEventListener(hri.getRegionNameAsString(),
112           closeEventProcessed, EventType.RS_ZK_REGION_CLOSED);
113     cluster.getMaster().executorService.
114       registerListener(EventType.RS_ZK_REGION_CLOSED, closeListener);
115 
116     EventHandlerListener openListener =
117       new ReopenEventListener(hri.getRegionNameAsString(),
118           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
119     cluster.getMaster().executorService.
120       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
121 
122     LOG.info("Unassign " + hri.getRegionNameAsString());
123     cluster.getMaster().assignmentManager.unassign(hri);
124 
125     while (!closeEventProcessed.get()) {
126       Threads.sleep(100);
127     }
128 
129     while (!reopenEventProcessed.get()) {
130       Threads.sleep(100);
131     }
132 
133     LOG.info("Done with testReOpenRegion");
134   }
135 
136   private HRegionInfo getNonMetaRegion(final Collection<HRegionInfo> regions) {
137     HRegionInfo hri = null;
138     for (HRegionInfo i: regions) {
139       LOG.info(i.getRegionNameAsString());
140       if (!i.isMetaRegion()) {
141         hri = i;
142         break;
143       }
144     }
145     return hri;
146   }
147 
148   public static class ReopenEventListener implements EventHandlerListener {
149     private static final Log LOG = LogFactory.getLog(ReopenEventListener.class);
150     String regionName;
151     AtomicBoolean eventProcessed;
152     EventType eventType;
153 
154     public ReopenEventListener(String regionName,
155         AtomicBoolean eventProcessed, EventType eventType) {
156       this.regionName = regionName;
157       this.eventProcessed = eventProcessed;
158       this.eventType = eventType;
159     }
160 
161     @Override
162     public void beforeProcess(EventHandler event) {
163       if(event.getEventType() == eventType) {
164         LOG.info("Received " + eventType + " and beginning to process it");
165       }
166     }
167 
168     @Override
169     public void afterProcess(EventHandler event) {
170       LOG.info("afterProcess(" + event + ")");
171       if(event.getEventType() == eventType) {
172         LOG.info("Finished processing " + eventType);
173         String regionName = "";
174         if(eventType == EventType.RS_ZK_REGION_OPENED) {
175           TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
176           regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
177         } else if(eventType == EventType.RS_ZK_REGION_CLOSED) {
178           TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
179           regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
180         }
181         if(this.regionName.equals(regionName)) {
182           eventProcessed.set(true);
183         }
184         synchronized(eventProcessed) {
185           eventProcessed.notifyAll();
186         }
187       }
188     }
189   }
190 
191   @Test (timeout=300000) public void testCloseRegion()
192   throws Exception {
193     LOG.info("Running testCloseRegion");
194     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
195     LOG.info("Number of region servers = " + cluster.getLiveRegionServerThreads().size());
196 
197     int rsIdx = 0;
198     HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
199     HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
200     LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
201 
202     AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
203     EventHandlerListener listener =
204       new CloseRegionEventListener(hri.getRegionNameAsString(),
205           closeEventProcessed);
206     cluster.getMaster().executorService.registerListener(EventType.RS_ZK_REGION_CLOSED, listener);
207 
208     cluster.getMaster().assignmentManager.unassign(hri);
209 
210     while (!closeEventProcessed.get()) {
211       Threads.sleep(100);
212     }
213     LOG.info("Done with testCloseRegion");
214   }
215 
216   public static class CloseRegionEventListener implements EventHandlerListener {
217     private static final Log LOG = LogFactory.getLog(CloseRegionEventListener.class);
218     String regionToClose;
219     AtomicBoolean closeEventProcessed;
220 
221     public CloseRegionEventListener(String regionToClose,
222         AtomicBoolean closeEventProcessed) {
223       this.regionToClose = regionToClose;
224       this.closeEventProcessed = closeEventProcessed;
225     }
226 
227     @Override
228     public void afterProcess(EventHandler event) {
229       LOG.info("afterProcess(" + event + ")");
230       if(event.getEventType() == EventType.RS_ZK_REGION_CLOSED) {
231         LOG.info("Finished processing CLOSE REGION");
232         TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
233         if (regionToClose.equals(hriCarrier.getHRegionInfo().getRegionNameAsString())) {
234           LOG.info("Setting closeEventProcessed flag");
235           closeEventProcessed.set(true);
236         } else {
237           LOG.info("Region to close didn't match");
238         }
239       }
240     }
241 
242     @Override
243     public void beforeProcess(EventHandler event) {
244       if(event.getEventType() == EventType.M_RS_CLOSE_REGION) {
245         LOG.info("Received CLOSE RPC and beginning to process it");
246       }
247     }
248   }
249 
250   /**
251    * This test shows how a region won't be able to be assigned to a RS
252    * if it's already "processing" it.
253    * @throws Exception
254    */
255   @Test
256   public void testRSAlreadyProcessingRegion() throws Exception {
257     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
258 
259     HRegionServer hr0 =
260         cluster.getLiveRegionServerThreads().get(0).getRegionServer();
261     HRegionServer hr1 =
262         cluster.getLiveRegionServerThreads().get(1).getRegionServer();
263     HRegionInfo hri = getNonMetaRegion(hr0.getOnlineRegions());
264 
265     // fake that hr1 is processing the region
266     hr1.getRegionsInTransitionInRS().add(hri.getEncodedNameAsBytes());
267 
268     AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
269     EventHandlerListener openListener =
270       new ReopenEventListener(hri.getRegionNameAsString(),
271           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
272     cluster.getMaster().executorService.
273       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
274 
275     // now ask the master to move the region to hr1, will fail
276     TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
277         Bytes.toBytes(hr1.getServerName()));
278 
279     while (!reopenEventProcessed.get()) {
280       Threads.sleep(100);
281     }
282 
283     // make sure the region came back
284     assertTrue(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()) == null);
285 
286     // remove the block and reset the boolean
287     hr1.getRegionsInTransitionInRS().remove(hri.getEncodedNameAsBytes());
288     reopenEventProcessed.set(false);
289 
290     // move the region again, but this time it will work
291     TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
292         Bytes.toBytes(hr1.getServerName()));
293 
294     while (!reopenEventProcessed.get()) {
295       Threads.sleep(100);
296     }
297 
298     // make sure the region has moved from the original RS
299     assertTrue(hr0.getOnlineRegion(hri.getEncodedNameAsBytes()) == null);
300 
301   }
302 
303   private static void waitUntilAllRegionsAssigned()
304   throws IOException {
305     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
306       HConstants.META_TABLE_NAME);
307     while (true) {
308       int rows = 0;
309       Scan scan = new Scan();
310       scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
311       ResultScanner s = meta.getScanner(scan);
312       for (Result r = null; (r = s.next()) != null;) {
313         byte [] b =
314           r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
315         if (b == null || b.length <= 0) {
316           break;
317         }
318         rows++;
319       }
320       s.close();
321       // If I get to here and all rows have a Server, then all have been assigned.
322       if (rows >= countOfRegions) {
323         break;
324       }
325       LOG.info("Found=" + rows);
326       Threads.sleep(1000);
327     }
328   }
329 
330   /*
331    * Add to each of the regions in .META. a value.  Key is the startrow of the
332    * region (except its 'aaa' for first region).  Actual value is the row name.
333    * @param expected
334    * @return
335    * @throws IOException
336    */
337   private static int addToEachStartKey(final int expected) throws IOException {
338     HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
339     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
340         HConstants.META_TABLE_NAME);
341     int rows = 0;
342     Scan scan = new Scan();
343     scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
344     ResultScanner s = meta.getScanner(scan);
345     for (Result r = null; (r = s.next()) != null;) {
346       byte [] b =
347         r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
348       if (b == null || b.length <= 0) {
349         break;
350       }
351       HRegionInfo hri = Writables.getHRegionInfo(b);
352       // If start key, add 'aaa'.
353       byte [] row = getStartKey(hri);
354       Put p = new Put(row);
355       p.add(getTestFamily(), getTestQualifier(), row);
356       t.put(p);
357       rows++;
358     }
359     s.close();
360     Assert.assertEquals(expected, rows);
361     return rows;
362   }
363 
364   private static byte [] getStartKey(final HRegionInfo hri) {
365     return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
366         Bytes.toBytes("aaa"): hri.getStartKey();
367   }
368 
369   private static byte [] getTestFamily() {
370     return FAMILIES[0];
371   }
372 
373   private static byte [] getTestQualifier() {
374     return getTestFamily();
375   }
376 
377   public static void main(String args[]) throws Exception {
378     TestZKBasedOpenCloseRegion.beforeAllTests();
379 
380     TestZKBasedOpenCloseRegion test = new TestZKBasedOpenCloseRegion();
381     test.setup();
382     test.testCloseRegion();
383 
384     TestZKBasedOpenCloseRegion.afterAllTests();
385   }
386 }