1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  
23  import java.io.IOException;
24  import java.util.Collection;
25  import java.util.concurrent.atomic.AtomicBoolean;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.*;
31  import org.apache.hadoop.hbase.client.Durability;
32  import org.apache.hadoop.hbase.client.HTable;
33  import org.apache.hadoop.hbase.client.Put;
34  import org.apache.hadoop.hbase.client.Result;
35  import org.apache.hadoop.hbase.client.ResultScanner;
36  import org.apache.hadoop.hbase.client.Scan;
37  import org.apache.hadoop.hbase.executor.EventHandler;
38  import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
39  import org.apache.hadoop.hbase.executor.EventHandler.EventType;
40  import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo;
41  import org.apache.hadoop.hbase.regionserver.HRegionServer;
42  import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.Threads;
45  import org.apache.hadoop.hbase.util.Writables;
46  import org.junit.AfterClass;
47  import org.junit.Assert;
48  import org.junit.Before;
49  import org.junit.BeforeClass;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  import org.mockito.Mockito;
53  import org.mockito.internal.util.reflection.Whitebox;
54  
55  import static org.junit.Assert.assertEquals;
56  import static org.junit.Assert.assertTrue;
57  import static org.junit.Assert.fail;
58  import static org.junit.Assert.assertFalse;
59  
60  /**
61   * Test open and close of regions using zk.
62   */
63  @Category(MediumTests.class)
64  public class TestZKBasedOpenCloseRegion {
65    private static final Log LOG = LogFactory.getLog(TestZKBasedOpenCloseRegion.class);
66    private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
67    private static final String TABLENAME = "TestZKBasedOpenCloseRegion";
68    private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
69      Bytes.toBytes("b"), Bytes.toBytes("c")};
70    private static int countOfRegions;
71  
72    @BeforeClass public static void beforeAllTests() throws Exception {
73      Configuration c = TEST_UTIL.getConfiguration();
74      c.setClass(HConstants.REGION_SERVER_IMPL, TestZKBasedOpenCloseRegionRegionServer.class,
75                HRegionServer.class);
76      c.setBoolean("dfs.support.append", true);
77      c.setInt("hbase.regionserver.info.port", 0);
78      TEST_UTIL.startMiniCluster(2);
79      TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
80      HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
81      countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
82      waitUntilAllRegionsAssigned();
83      addToEachStartKey(countOfRegions);
84      t.close();
85    }
86  
87    @AfterClass public static void afterAllTests() throws Exception {
88      TEST_UTIL.shutdownMiniCluster();
89    }
90  
91    @Before public void setup() throws IOException {
92      if (TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size() < 2) {
93        // Need at least two servers.
94        LOG.info("Started new server=" +
95          TEST_UTIL.getHBaseCluster().startRegionServer());
96  
97      }
98      waitUntilAllRegionsAssigned();
99    }
100 
101   /**
102    * Special HRegionServer used in these tests that allows access to
103    * {@link #addRegionsInTransition(HRegionInfo, String)}.
104    */
105   public static class TestZKBasedOpenCloseRegionRegionServer extends HRegionServer {
106     public TestZKBasedOpenCloseRegionRegionServer(Configuration conf)
107         throws IOException, InterruptedException {
108       super(conf);
109     }
110     @Override
111     public boolean addRegionsInTransition(HRegionInfo region,
112         String currentAction) throws RegionAlreadyInTransitionException {
113       return super.addRegionsInTransition(region, currentAction);
114     }
115   }
116 
117   /**
118    * Test we reopen a region once closed.
119    * @throws Exception
120    */
121   @Test (timeout=300000) public void testReOpenRegion()
122   throws Exception {
123     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
124     LOG.info("Number of region servers = " +
125       cluster.getLiveRegionServerThreads().size());
126 
127     int rsIdx = 0;
128     HRegionServer regionServer =
129       TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
130     HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
131     LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
132 
133     AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
134     AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
135 
136     EventHandlerListener closeListener =
137       new ReopenEventListener(hri.getRegionNameAsString(),
138           closeEventProcessed, EventType.RS_ZK_REGION_CLOSED);
139     cluster.getMaster().executorService.
140       registerListener(EventType.RS_ZK_REGION_CLOSED, closeListener);
141 
142     EventHandlerListener openListener =
143       new ReopenEventListener(hri.getRegionNameAsString(),
144           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
145     cluster.getMaster().executorService.
146       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
147 
148     LOG.info("Unassign " + hri.getRegionNameAsString());
149     cluster.getMaster().assignmentManager.unassign(hri);
150 
151     while (!closeEventProcessed.get()) {
152       Threads.sleep(100);
153     }
154 
155     while (!reopenEventProcessed.get()) {
156       Threads.sleep(100);
157     }
158 
159     LOG.info("Done with testReOpenRegion");
160   }
161 
162   private HRegionInfo getNonMetaRegion(final Collection<HRegionInfo> regions) {
163     HRegionInfo hri = null;
164     for (HRegionInfo i: regions) {
165       LOG.info(i.getRegionNameAsString());
166       if (!i.isMetaRegion()) {
167         hri = i;
168         break;
169       }
170     }
171     return hri;
172   }
173 
174   public static class ReopenEventListener implements EventHandlerListener {
175     private static final Log LOG = LogFactory.getLog(ReopenEventListener.class);
176     String regionName;
177     AtomicBoolean eventProcessed;
178     EventType eventType;
179 
180     public ReopenEventListener(String regionName,
181         AtomicBoolean eventProcessed, EventType eventType) {
182       this.regionName = regionName;
183       this.eventProcessed = eventProcessed;
184       this.eventType = eventType;
185     }
186 
187     @Override
188     public void beforeProcess(EventHandler event) {
189       if(event.getEventType() == eventType) {
190         LOG.info("Received " + eventType + " and beginning to process it");
191       }
192     }
193 
194     @Override
195     public void afterProcess(EventHandler event) {
196       LOG.info("afterProcess(" + event + ")");
197       if(event.getEventType() == eventType) {
198         LOG.info("Finished processing " + eventType);
199         String regionName = "";
200         if(eventType == EventType.RS_ZK_REGION_OPENED) {
201           TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
202           regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
203         } else if(eventType == EventType.RS_ZK_REGION_CLOSED) {
204           TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
205           regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
206         }
207         if(this.regionName.equals(regionName)) {
208           eventProcessed.set(true);
209         }
210         synchronized(eventProcessed) {
211           eventProcessed.notifyAll();
212         }
213       }
214     }
215   }
216 
217   /**
218    * This test shows how a region won't be able to be assigned to a RS
219    * if it's already "processing" it.
220    * @throws Exception
221    */
222   @Test
223   public void testRSAlreadyProcessingRegion() throws Exception {
224     LOG.info("starting testRSAlreadyProcessingRegion");
225     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
226 
227     HRegionServer hr0 =
228         cluster.getLiveRegionServerThreads().get(0).getRegionServer();
229     HRegionServer hr1 =
230         cluster.getLiveRegionServerThreads().get(1).getRegionServer();
231     HRegionInfo hri = getNonMetaRegion(hr0.getOnlineRegions());
232 
233     // Fake that hr1 is processing the region. At top of this test we made a
234     // regionserver that gave access addRegionsInTransition. Need to cast as
235     // TestZKBasedOpenCloseRegionRegionServer.
236     ((TestZKBasedOpenCloseRegionRegionServer) hr1).addRegionsInTransition(hri, "OPEN");
237 
238     AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
239     EventHandlerListener openListener =
240       new ReopenEventListener(hri.getRegionNameAsString(),
241           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
242     cluster.getMaster().executorService.
243       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
244 
245     // now ask the master to move the region to hr1, will fail
246     TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
247         Bytes.toBytes(hr1.getServerName().toString()));
248 
249     // make sure the region came back
250     assertEquals(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()), null);
251 
252     // remove the block and reset the boolean
253     hr1.removeFromRegionsInTransition(hri);
254     reopenEventProcessed.set(false);
255     
256     // now try moving a region when there is no region in transition.
257     hri = getNonMetaRegion(hr1.getOnlineRegions());
258 
259     openListener =
260       new ReopenEventListener(hri.getRegionNameAsString(),
261           reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
262 
263     cluster.getMaster().executorService.
264       registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
265     
266     TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
267         Bytes.toBytes(hr0.getServerName().toString()));
268 
269     while (!reopenEventProcessed.get()) {
270       Threads.sleep(100);
271     }
272 
273     // make sure the region has moved from the original RS
274     assertTrue(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()) == null);
275 
276   }
277 
278   /**
279    * If region open fails with IOException in openRegion() while doing tableDescriptors.get()
280    * the region should not add into regionsInTransitionInRS map
281    * @throws Exception
282    */
283   @Test
284   public void testRegionOpenFailsDueToIOException() throws Exception {
285     HRegionInfo REGIONINFO = new HRegionInfo(Bytes.toBytes("t"),
286         HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
287     HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0);
288     TableDescriptors htd = Mockito.mock(TableDescriptors.class);
289     Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors");
290     Whitebox.setInternalState(regionServer, "tableDescriptors", htd);
291     Mockito.doThrow(new IOException()).when(htd).get((byte[]) Mockito.any());
292     try {
293       regionServer.openRegion(REGIONINFO);
294       fail("It should throw IOException ");
295     } catch (IOException e) {
296     }
297     Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState);
298     assertFalse("Region should not be in RIT",
299         regionServer.containsKeyInRegionsInTransition(REGIONINFO));
300   }
301   
302   private static void waitUntilAllRegionsAssigned()
303   throws IOException {
304     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
305       HConstants.META_TABLE_NAME);
306     while (true) {
307       int rows = 0;
308       Scan scan = new Scan();
309       scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
310       ResultScanner s = meta.getScanner(scan);
311       for (Result r = null; (r = s.next()) != null;) {
312         byte [] b =
313           r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
314         if (b == null || b.length <= 0) {
315           break;
316         }
317         rows++;
318       }
319       s.close();
320       // If I get to here and all rows have a Server, then all have been assigned.
321       if (rows >= countOfRegions) {
322         break;
323       }
324       LOG.info("Found=" + rows);
325       Threads.sleep(1000);
326     }
327     meta.close();
328   }
329 
330   /*
331    * Add to each of the regions in .META. a value.  Key is the startrow of the
332    * region (except its 'aaa' for first region).  Actual value is the row name.
333    * @param expected
334    * @return
335    * @throws IOException
336    */
337   private static int addToEachStartKey(final int expected) throws IOException {
338     HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
339     HTable meta = new HTable(TEST_UTIL.getConfiguration(),
340         HConstants.META_TABLE_NAME);
341     int rows = 0;
342     Scan scan = new Scan();
343     scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
344     ResultScanner s = meta.getScanner(scan);
345     for (Result r = null; (r = s.next()) != null;) {
346       byte [] b =
347         r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
348       if (b == null || b.length <= 0) {
349         break;
350       }
351       HRegionInfo hri = Writables.getHRegionInfo(b);
352       // If start key, add 'aaa'.
353       byte [] row = getStartKey(hri);
354       Put p = new Put(row);
355       p.setDurability(Durability.SKIP_WAL);
356       p.add(getTestFamily(), getTestQualifier(), row);
357       t.put(p);
358       rows++;
359     }
360     s.close();
361     Assert.assertEquals(expected, rows);
362     t.close();
363     meta.close();
364     return rows;
365   }
366 
367   private static byte [] getStartKey(final HRegionInfo hri) {
368     return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
369         Bytes.toBytes("aaa"): hri.getStartKey();
370   }
371 
372   private static byte [] getTestFamily() {
373     return FAMILIES[0];
374   }
375 
376   private static byte [] getTestQualifier() {
377     return getTestFamily();
378   }
379 
380   @org.junit.Rule
381   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
382     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
383 }
384