1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master;
21
22
23 import java.io.IOException;
24 import java.util.Collection;
25 import java.util.concurrent.atomic.AtomicBoolean;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.*;
31 import org.apache.hadoop.hbase.client.Durability;
32 import org.apache.hadoop.hbase.client.HTable;
33 import org.apache.hadoop.hbase.client.Put;
34 import org.apache.hadoop.hbase.client.Result;
35 import org.apache.hadoop.hbase.client.ResultScanner;
36 import org.apache.hadoop.hbase.client.Scan;
37 import org.apache.hadoop.hbase.executor.EventHandler;
38 import org.apache.hadoop.hbase.executor.EventHandler.EventHandlerListener;
39 import org.apache.hadoop.hbase.executor.EventHandler.EventType;
40 import org.apache.hadoop.hbase.master.handler.TotesHRegionInfo;
41 import org.apache.hadoop.hbase.regionserver.HRegionServer;
42 import org.apache.hadoop.hbase.regionserver.RegionAlreadyInTransitionException;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.Threads;
45 import org.apache.hadoop.hbase.util.Writables;
46 import org.junit.AfterClass;
47 import org.junit.Assert;
48 import org.junit.Before;
49 import org.junit.BeforeClass;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52 import org.mockito.Mockito;
53 import org.mockito.internal.util.reflection.Whitebox;
54
55 import static org.junit.Assert.assertEquals;
56 import static org.junit.Assert.assertTrue;
57 import static org.junit.Assert.fail;
58 import static org.junit.Assert.assertFalse;
59
60
61
62
63 @Category(MediumTests.class)
64 public class TestZKBasedOpenCloseRegion {
65 private static final Log LOG = LogFactory.getLog(TestZKBasedOpenCloseRegion.class);
66 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
67 private static final String TABLENAME = "TestZKBasedOpenCloseRegion";
68 private static final byte [][] FAMILIES = new byte [][] {Bytes.toBytes("a"),
69 Bytes.toBytes("b"), Bytes.toBytes("c")};
70 private static int countOfRegions;
71
72 @BeforeClass public static void beforeAllTests() throws Exception {
73 Configuration c = TEST_UTIL.getConfiguration();
74 c.setClass(HConstants.REGION_SERVER_IMPL, TestZKBasedOpenCloseRegionRegionServer.class,
75 HRegionServer.class);
76 c.setBoolean("dfs.support.append", true);
77 c.setInt("hbase.regionserver.info.port", 0);
78 TEST_UTIL.startMiniCluster(2);
79 TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
80 HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
81 countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
82 waitUntilAllRegionsAssigned();
83 addToEachStartKey(countOfRegions);
84 t.close();
85 }
86
87 @AfterClass public static void afterAllTests() throws Exception {
88 TEST_UTIL.shutdownMiniCluster();
89 }
90
91 @Before public void setup() throws IOException {
92 if (TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().size() < 2) {
93
94 LOG.info("Started new server=" +
95 TEST_UTIL.getHBaseCluster().startRegionServer());
96
97 }
98 waitUntilAllRegionsAssigned();
99 }
100
101
102
103
104
105 public static class TestZKBasedOpenCloseRegionRegionServer extends HRegionServer {
106 public TestZKBasedOpenCloseRegionRegionServer(Configuration conf)
107 throws IOException, InterruptedException {
108 super(conf);
109 }
110 @Override
111 public boolean addRegionsInTransition(HRegionInfo region,
112 String currentAction) throws RegionAlreadyInTransitionException {
113 return super.addRegionsInTransition(region, currentAction);
114 }
115 }
116
117
118
119
120
121 @Test (timeout=300000) public void testReOpenRegion()
122 throws Exception {
123 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
124 LOG.info("Number of region servers = " +
125 cluster.getLiveRegionServerThreads().size());
126
127 int rsIdx = 0;
128 HRegionServer regionServer =
129 TEST_UTIL.getHBaseCluster().getRegionServer(rsIdx);
130 HRegionInfo hri = getNonMetaRegion(regionServer.getOnlineRegions());
131 LOG.debug("Asking RS to close region " + hri.getRegionNameAsString());
132
133 AtomicBoolean closeEventProcessed = new AtomicBoolean(false);
134 AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
135
136 EventHandlerListener closeListener =
137 new ReopenEventListener(hri.getRegionNameAsString(),
138 closeEventProcessed, EventType.RS_ZK_REGION_CLOSED);
139 cluster.getMaster().executorService.
140 registerListener(EventType.RS_ZK_REGION_CLOSED, closeListener);
141
142 EventHandlerListener openListener =
143 new ReopenEventListener(hri.getRegionNameAsString(),
144 reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
145 cluster.getMaster().executorService.
146 registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
147
148 LOG.info("Unassign " + hri.getRegionNameAsString());
149 cluster.getMaster().assignmentManager.unassign(hri);
150
151 while (!closeEventProcessed.get()) {
152 Threads.sleep(100);
153 }
154
155 while (!reopenEventProcessed.get()) {
156 Threads.sleep(100);
157 }
158
159 LOG.info("Done with testReOpenRegion");
160 }
161
162 private HRegionInfo getNonMetaRegion(final Collection<HRegionInfo> regions) {
163 HRegionInfo hri = null;
164 for (HRegionInfo i: regions) {
165 LOG.info(i.getRegionNameAsString());
166 if (!i.isMetaRegion()) {
167 hri = i;
168 break;
169 }
170 }
171 return hri;
172 }
173
174 public static class ReopenEventListener implements EventHandlerListener {
175 private static final Log LOG = LogFactory.getLog(ReopenEventListener.class);
176 String regionName;
177 AtomicBoolean eventProcessed;
178 EventType eventType;
179
180 public ReopenEventListener(String regionName,
181 AtomicBoolean eventProcessed, EventType eventType) {
182 this.regionName = regionName;
183 this.eventProcessed = eventProcessed;
184 this.eventType = eventType;
185 }
186
187 @Override
188 public void beforeProcess(EventHandler event) {
189 if(event.getEventType() == eventType) {
190 LOG.info("Received " + eventType + " and beginning to process it");
191 }
192 }
193
194 @Override
195 public void afterProcess(EventHandler event) {
196 LOG.info("afterProcess(" + event + ")");
197 if(event.getEventType() == eventType) {
198 LOG.info("Finished processing " + eventType);
199 String regionName = "";
200 if(eventType == EventType.RS_ZK_REGION_OPENED) {
201 TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
202 regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
203 } else if(eventType == EventType.RS_ZK_REGION_CLOSED) {
204 TotesHRegionInfo hriCarrier = (TotesHRegionInfo)event;
205 regionName = hriCarrier.getHRegionInfo().getRegionNameAsString();
206 }
207 if(this.regionName.equals(regionName)) {
208 eventProcessed.set(true);
209 }
210 synchronized(eventProcessed) {
211 eventProcessed.notifyAll();
212 }
213 }
214 }
215 }
216
217
218
219
220
221
222 @Test
223 public void testRSAlreadyProcessingRegion() throws Exception {
224 LOG.info("starting testRSAlreadyProcessingRegion");
225 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
226
227 HRegionServer hr0 =
228 cluster.getLiveRegionServerThreads().get(0).getRegionServer();
229 HRegionServer hr1 =
230 cluster.getLiveRegionServerThreads().get(1).getRegionServer();
231 HRegionInfo hri = getNonMetaRegion(hr0.getOnlineRegions());
232
233
234
235
236 ((TestZKBasedOpenCloseRegionRegionServer) hr1).addRegionsInTransition(hri, "OPEN");
237
238 AtomicBoolean reopenEventProcessed = new AtomicBoolean(false);
239 EventHandlerListener openListener =
240 new ReopenEventListener(hri.getRegionNameAsString(),
241 reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
242 cluster.getMaster().executorService.
243 registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
244
245
246 TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
247 Bytes.toBytes(hr1.getServerName().toString()));
248
249
250 assertEquals(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()), null);
251
252
253 hr1.removeFromRegionsInTransition(hri);
254 reopenEventProcessed.set(false);
255
256
257 hri = getNonMetaRegion(hr1.getOnlineRegions());
258
259 openListener =
260 new ReopenEventListener(hri.getRegionNameAsString(),
261 reopenEventProcessed, EventType.RS_ZK_REGION_OPENED);
262
263 cluster.getMaster().executorService.
264 registerListener(EventType.RS_ZK_REGION_OPENED, openListener);
265
266 TEST_UTIL.getHBaseAdmin().move(hri.getEncodedNameAsBytes(),
267 Bytes.toBytes(hr0.getServerName().toString()));
268
269 while (!reopenEventProcessed.get()) {
270 Threads.sleep(100);
271 }
272
273
274 assertTrue(hr1.getOnlineRegion(hri.getEncodedNameAsBytes()) == null);
275
276 }
277
278
279
280
281
282
283 @Test
284 public void testRegionOpenFailsDueToIOException() throws Exception {
285 HRegionInfo REGIONINFO = new HRegionInfo(Bytes.toBytes("t"),
286 HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
287 HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(0);
288 TableDescriptors htd = Mockito.mock(TableDescriptors.class);
289 Object orizinalState = Whitebox.getInternalState(regionServer,"tableDescriptors");
290 Whitebox.setInternalState(regionServer, "tableDescriptors", htd);
291 Mockito.doThrow(new IOException()).when(htd).get((byte[]) Mockito.any());
292 try {
293 regionServer.openRegion(REGIONINFO);
294 fail("It should throw IOException ");
295 } catch (IOException e) {
296 }
297 Whitebox.setInternalState(regionServer, "tableDescriptors", orizinalState);
298 assertFalse("Region should not be in RIT",
299 regionServer.containsKeyInRegionsInTransition(REGIONINFO));
300 }
301
302 private static void waitUntilAllRegionsAssigned()
303 throws IOException {
304 HTable meta = new HTable(TEST_UTIL.getConfiguration(),
305 HConstants.META_TABLE_NAME);
306 while (true) {
307 int rows = 0;
308 Scan scan = new Scan();
309 scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
310 ResultScanner s = meta.getScanner(scan);
311 for (Result r = null; (r = s.next()) != null;) {
312 byte [] b =
313 r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
314 if (b == null || b.length <= 0) {
315 break;
316 }
317 rows++;
318 }
319 s.close();
320
321 if (rows >= countOfRegions) {
322 break;
323 }
324 LOG.info("Found=" + rows);
325 Threads.sleep(1000);
326 }
327 meta.close();
328 }
329
330
331
332
333
334
335
336
337 private static int addToEachStartKey(final int expected) throws IOException {
338 HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
339 HTable meta = new HTable(TEST_UTIL.getConfiguration(),
340 HConstants.META_TABLE_NAME);
341 int rows = 0;
342 Scan scan = new Scan();
343 scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
344 ResultScanner s = meta.getScanner(scan);
345 for (Result r = null; (r = s.next()) != null;) {
346 byte [] b =
347 r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
348 if (b == null || b.length <= 0) {
349 break;
350 }
351 HRegionInfo hri = Writables.getHRegionInfo(b);
352
353 byte [] row = getStartKey(hri);
354 Put p = new Put(row);
355 p.setDurability(Durability.SKIP_WAL);
356 p.add(getTestFamily(), getTestQualifier(), row);
357 t.put(p);
358 rows++;
359 }
360 s.close();
361 Assert.assertEquals(expected, rows);
362 t.close();
363 meta.close();
364 return rows;
365 }
366
367 private static byte [] getStartKey(final HRegionInfo hri) {
368 return Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())?
369 Bytes.toBytes("aaa"): hri.getStartKey();
370 }
371
372 private static byte [] getTestFamily() {
373 return FAMILIES[0];
374 }
375
376 private static byte [] getTestQualifier() {
377 return getTestFamily();
378 }
379
380 @org.junit.Rule
381 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
382 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
383 }
384