1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.handler;
21
22 import java.io.IOException;
23 import java.util.concurrent.atomic.AtomicBoolean;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.HRegionInfo;
28 import org.apache.hadoop.hbase.HTableDescriptor;
29 import org.apache.hadoop.hbase.Server;
30 import org.apache.hadoop.hbase.executor.EventHandler;
31 import org.apache.hadoop.hbase.regionserver.HRegion;
32 import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
33 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
34 import org.apache.hadoop.hbase.regionserver.wal.HLog;
35 import org.apache.hadoop.hbase.util.CancelableProgressable;
36 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
37 import org.apache.zookeeper.KeeperException;
38
39
40
41
42
43
44 public class OpenRegionHandler extends EventHandler {
45 private static final Log LOG = LogFactory.getLog(OpenRegionHandler.class);
46
47 protected final RegionServerServices rsServices;
48
49 private final HRegionInfo regionInfo;
50 private final HTableDescriptor htd;
51
52
53
54
55 private volatile int version = -1;
56
57
58 public OpenRegionHandler(final Server server,
59 final RegionServerServices rsServices, HRegionInfo regionInfo,
60 HTableDescriptor htd) {
61 this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION, -1);
62 }
63 public OpenRegionHandler(final Server server,
64 final RegionServerServices rsServices, HRegionInfo regionInfo,
65 HTableDescriptor htd, int version) {
66 this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION,
67 version);
68 }
69
70 protected OpenRegionHandler(final Server server,
71 final RegionServerServices rsServices, final HRegionInfo regionInfo,
72 final HTableDescriptor htd, EventType eventType,
73 final int version) {
74 super(server, eventType);
75 this.rsServices = rsServices;
76 this.regionInfo = regionInfo;
77 this.htd = htd;
78 this.version = version;
79 }
80
81 public HRegionInfo getRegionInfo() {
82 return regionInfo;
83 }
84
85 @Override
86 public void process() throws IOException {
87 boolean transitionToFailedOpen = false;
88 boolean openSuccessful = false;
89 try {
90 final String name = regionInfo.getRegionNameAsString();
91 if (this.server.isStopped() || this.rsServices.isStopping()) {
92 return;
93 }
94 final String encodedName = regionInfo.getEncodedName();
95
96
97 HRegion region = this.rsServices.getFromOnlineRegions(encodedName);
98
99
100
101 region = openRegion();
102 if (region == null) {
103 tryTransitionToFailedOpen(regionInfo);
104 transitionToFailedOpen = true;
105 return;
106 }
107 boolean failed = true;
108 if (tickleOpening("post_region_open")) {
109 if (updateMeta(region)) {
110 failed = false;
111 }
112 }
113 if (failed || this.server.isStopped() ||
114 this.rsServices.isStopping()) {
115 cleanupFailedOpen(region);
116 tryTransitionToFailedOpen(regionInfo);
117 transitionToFailedOpen = true;
118 return;
119 }
120
121 if (!transitionToOpened(region)) {
122
123
124
125
126
127
128 cleanupFailedOpen(region);
129 transitionToFailedOpen = true;
130 return;
131 }
132
133 this.rsServices.addToOnlineRegions(region);
134 openSuccessful = true;
135
136 LOG.debug("Opened " + name + " on server:" +
137 this.server.getServerName());
138 } finally {
139 this.rsServices.removeFromRegionsInTransition(this.regionInfo);
140 if (!openSuccessful && !transitionToFailedOpen) {
141 tryTransitionToFailedOpen(regionInfo);
142 }
143 }
144 }
145
146
147
148
149
150
151
152
153 boolean updateMeta(final HRegion r) {
154 if (this.server.isStopped() || this.rsServices.isStopping()) {
155 return false;
156 }
157
158
159 final AtomicBoolean signaller = new AtomicBoolean(false);
160 PostOpenDeployTasksThread t = new PostOpenDeployTasksThread(r,
161 this.server, this.rsServices, signaller);
162 t.start();
163 int assignmentTimeout = this.server.getConfiguration().
164 getInt("hbase.master.assignment.timeoutmonitor.period", 10000);
165
166
167 long timeout = assignmentTimeout * 10;
168 long now = System.currentTimeMillis();
169 long endTime = now + timeout;
170
171
172 long period = Math.max(1, assignmentTimeout/ 3);
173 long lastUpdate = now;
174 boolean tickleOpening = true;
175 while (!signaller.get() && t.isAlive() && !this.server.isStopped() &&
176 !this.rsServices.isStopping() && (endTime > now)) {
177 long elapsed = now - lastUpdate;
178 if (elapsed > period) {
179
180 lastUpdate = now;
181 tickleOpening = tickleOpening("post_open_deploy");
182 }
183 synchronized (signaller) {
184 try {
185 signaller.wait(period);
186 } catch (InterruptedException e) {
187
188 }
189 }
190 now = System.currentTimeMillis();
191 }
192
193
194 if (t.isAlive()) {
195 if (!signaller.get()) {
196
197 LOG.debug("Interrupting thread " + t);
198 t.interrupt();
199 }
200 try {
201 t.join();
202 } catch (InterruptedException ie) {
203 LOG.warn("Interrupted joining " +
204 r.getRegionInfo().getRegionNameAsString(), ie);
205 Thread.currentThread().interrupt();
206 }
207 }
208
209
210
211
212 return ((!Thread.interrupted() && t.getException() == null) && tickleOpening);
213 }
214
215
216
217
218
219
220
221 static class PostOpenDeployTasksThread extends Thread {
222 private Exception exception = null;
223 private final Server server;
224 private final RegionServerServices services;
225 private final HRegion region;
226 private final AtomicBoolean signaller;
227
228 PostOpenDeployTasksThread(final HRegion region, final Server server,
229 final RegionServerServices services, final AtomicBoolean signaller) {
230 super("PostOpenDeployTasks:" + region.getRegionInfo().getEncodedName());
231 this.setDaemon(true);
232 this.server = server;
233 this.services = services;
234 this.region = region;
235 this.signaller = signaller;
236 }
237
238 public void run() {
239 try {
240 this.services.postOpenDeployTasks(this.region,
241 this.server.getCatalogTracker(), false);
242 } catch (KeeperException e) {
243 server.abort("Exception running postOpenDeployTasks; region=" +
244 this.region.getRegionInfo().getEncodedName(), e);
245 } catch (Exception e) {
246 LOG.warn("Exception running postOpenDeployTasks; region=" +
247 this.region.getRegionInfo().getEncodedName(), e);
248 this.exception = e;
249 }
250
251 this.signaller.set(true);
252 synchronized (this.signaller) {
253 this.signaller.notify();
254 }
255 }
256
257
258
259
260 Exception getException() {
261 return this.exception;
262 }
263 }
264
265
266
267
268
269
270
271 private boolean transitionToOpened(final HRegion r) throws IOException {
272 boolean result = false;
273 HRegionInfo hri = r.getRegionInfo();
274 final String name = hri.getRegionNameAsString();
275
276 try {
277 if (ZKAssign.transitionNodeOpened(this.server.getZooKeeper(), hri,
278 this.server.getServerName(), this.version) == -1) {
279 LOG.warn("Completed the OPEN of region " + name +
280 " but when transitioning from " +
281 " OPENING to OPENED got a version mismatch, someone else clashed " +
282 "so now unassigning -- closing region on server: " +
283 this.server.getServerName());
284 } else {
285 LOG.debug("region transitioned to opened in zookeeper: " +
286 r.getRegionInfo() + ", server: " + this.server.getServerName());
287 result = true;
288 }
289 } catch (KeeperException e) {
290 LOG.error("Failed transitioning node " + name +
291 " from OPENING to OPENED -- closing region", e);
292 }
293 return result;
294 }
295
296
297
298
299
300
301 private boolean tryTransitionToFailedOpen(final HRegionInfo hri) {
302 boolean result = false;
303 final String name = hri.getRegionNameAsString();
304 try {
305 LOG.info("Opening of region " + hri + " failed, marking as FAILED_OPEN in ZK");
306 if (ZKAssign.transitionNode(
307 this.server.getZooKeeper(), hri,
308 this.server.getServerName(),
309 EventType.RS_ZK_REGION_OPENING,
310 EventType.RS_ZK_REGION_FAILED_OPEN,
311 this.version) == -1) {
312 LOG.warn("Unable to mark region " + hri + " as FAILED_OPEN. " +
313 "It's likely that the master already timed out this open " +
314 "attempt, and thus another RS already has the region.");
315 } else {
316 result = true;
317 }
318 } catch (KeeperException e) {
319 LOG.error("Failed transitioning node " + name +
320 " from OPENING to FAILED_OPEN", e);
321 }
322 return result;
323 }
324
325
326
327
328 HRegion openRegion() {
329 HRegion region = null;
330 try {
331
332
333 region = HRegion.openHRegion(this.regionInfo, this.htd,
334 this.rsServices.getWAL(this.regionInfo),
335 this.server.getConfiguration(),
336 this.rsServices,
337 new CancelableProgressable() {
338 public boolean progress() {
339
340
341
342 return tickleOpening("open_region_progress");
343 }
344 });
345 } catch (Throwable t) {
346
347
348
349 LOG.error(
350 "Failed open of region=" + this.regionInfo.getRegionNameAsString()
351 + ", starting to roll back the global memstore size.", t);
352
353 if (this.rsServices != null) {
354 RegionServerAccounting rsAccounting =
355 this.rsServices.getRegionServerAccounting();
356 if (rsAccounting != null) {
357 rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName());
358 }
359 }
360 }
361 return region;
362 }
363
364 void cleanupFailedOpen(final HRegion region) throws IOException {
365 if (region != null) region.close();
366 }
367
368
369
370
371
372
373
374
375 boolean tickleOpening(final String context) {
376
377 if (!isGoodVersion()) return false;
378 String encodedName = this.regionInfo.getEncodedName();
379 try {
380 this.version =
381 ZKAssign.retransitionNodeOpening(server.getZooKeeper(),
382 this.regionInfo, this.server.getServerName(), this.version);
383 } catch (KeeperException e) {
384 server.abort("Exception refreshing OPENING; region=" + encodedName +
385 ", context=" + context, e);
386 this.version = -1;
387 }
388 boolean b = isGoodVersion();
389 if (!b) {
390 LOG.warn("Failed refreshing OPENING; region=" + encodedName +
391 ", context=" + context);
392 }
393 return b;
394 }
395
396 private boolean isGoodVersion() {
397 return this.version != -1;
398 }
399 }