1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.handler;
21
22 import java.io.IOException;
23 import java.util.concurrent.atomic.AtomicBoolean;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.hbase.HRegionInfo;
28 import org.apache.hadoop.hbase.HTableDescriptor;
29 import org.apache.hadoop.hbase.Server;
30 import org.apache.hadoop.hbase.executor.EventHandler;
31 import org.apache.hadoop.hbase.regionserver.HRegion;
32 import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
33 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
34 import org.apache.hadoop.hbase.regionserver.wal.HLog;
35 import org.apache.hadoop.hbase.util.CancelableProgressable;
36 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
37 import org.apache.zookeeper.KeeperException;
38
39
40
41
42
43
44 public class OpenRegionHandler extends EventHandler {
45 private static final Log LOG = LogFactory.getLog(OpenRegionHandler.class);
46
47 protected final RegionServerServices rsServices;
48
49 private final HRegionInfo regionInfo;
50 private final HTableDescriptor htd;
51
52
53
54
55 private volatile int version = -1;
56
57
58 public OpenRegionHandler(final Server server,
59 final RegionServerServices rsServices, HRegionInfo regionInfo,
60 HTableDescriptor htd) {
61 this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION, -1);
62 }
63 public OpenRegionHandler(final Server server,
64 final RegionServerServices rsServices, HRegionInfo regionInfo,
65 HTableDescriptor htd, int version) {
66 this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION,
67 version);
68 }
69
70 protected OpenRegionHandler(final Server server,
71 final RegionServerServices rsServices, final HRegionInfo regionInfo,
72 final HTableDescriptor htd, EventType eventType,
73 final int version) {
74 super(server, eventType);
75 this.rsServices = rsServices;
76 this.regionInfo = regionInfo;
77 this.htd = htd;
78 this.version = version;
79 }
80
81 public HRegionInfo getRegionInfo() {
82 return regionInfo;
83 }
84
85 @Override
86 public void process() throws IOException {
87 boolean transitionToFailedOpen = false;
88 boolean openSuccessful = false;
89 try {
90 final String name = regionInfo.getRegionNameAsString();
91 if (this.server.isStopped() || this.rsServices.isStopping()) {
92 return;
93 }
94 final String encodedName = regionInfo.getEncodedName();
95
96
97 HRegion region = this.rsServices.getFromOnlineRegions(encodedName);
98
99
100
101 region = openRegion();
102 if (region == null) {
103 tryTransitionToFailedOpen(regionInfo);
104 transitionToFailedOpen = true;
105 return;
106 }
107 boolean failed = true;
108 if (tickleOpening("post_region_open")) {
109 if (updateMeta(region)) {
110 failed = false;
111 }
112 }
113 if (failed || this.server.isStopped() ||
114 this.rsServices.isStopping()) {
115 cleanupFailedOpen(region);
116 tryTransitionToFailedOpen(regionInfo);
117 transitionToFailedOpen = true;
118 return;
119 }
120
121 if (!transitionToOpened(region)) {
122
123
124
125
126
127
128 cleanupFailedOpen(region);
129 transitionToFailedOpen = true;
130 return;
131 }
132
133 this.rsServices.addToOnlineRegions(region);
134 openSuccessful = true;
135
136 LOG.debug("Opened " + name + " on server:" +
137 this.server.getServerName());
138 } finally {
139 this.rsServices.removeFromRegionsInTransition(this.regionInfo);
140 if (!openSuccessful && !transitionToFailedOpen) {
141 tryTransitionToFailedOpen(regionInfo);
142 }
143 }
144 }
145
146
147
148
149
150
151
152
153 boolean updateMeta(final HRegion r) {
154 if (this.server.isStopped() || this.rsServices.isStopping()) {
155 return false;
156 }
157
158
159 final AtomicBoolean signaller = new AtomicBoolean(false);
160 PostOpenDeployTasksThread t = new PostOpenDeployTasksThread(r,
161 this.server, this.rsServices, signaller);
162 t.start();
163 int assignmentTimeout = this.server.getConfiguration().
164 getInt("hbase.master.assignment.timeoutmonitor.period", 10000);
165
166
167 long timeout = assignmentTimeout * 10;
168 long now = System.currentTimeMillis();
169 long endTime = now + timeout;
170
171
172 long period = Math.max(1, assignmentTimeout/ 3);
173 long lastUpdate = now;
174 boolean tickleOpening = true;
175 while (!signaller.get() && t.isAlive() && !this.server.isStopped() &&
176 !this.rsServices.isStopping() && (endTime > now)) {
177 long elapsed = now - lastUpdate;
178 if (elapsed > period) {
179
180 lastUpdate = now;
181 tickleOpening = tickleOpening("post_open_deploy");
182 }
183 synchronized (signaller) {
184 try {
185 signaller.wait(period);
186 } catch (InterruptedException e) {
187
188 }
189 }
190 now = System.currentTimeMillis();
191 }
192
193
194 if (t.isAlive()) {
195 if (!signaller.get()) {
196
197 LOG.debug("Interrupting thread " + t);
198 t.interrupt();
199 }
200 try {
201 t.join();
202 } catch (InterruptedException ie) {
203 LOG.warn("Interrupted joining " +
204 r.getRegionInfo().getRegionNameAsString(), ie);
205 Thread.currentThread().interrupt();
206 }
207 }
208
209
210
211
212 return ((!Thread.interrupted() && t.getException() == null) && tickleOpening);
213 }
214
215
216
217
218
219
220
221 static class PostOpenDeployTasksThread extends Thread {
222 private Exception exception = null;
223 private final Server server;
224 private final RegionServerServices services;
225 private final HRegion region;
226 private final AtomicBoolean signaller;
227
228 PostOpenDeployTasksThread(final HRegion region, final Server server,
229 final RegionServerServices services, final AtomicBoolean signaller) {
230 super("PostOpenDeployTasks:" + region.getRegionInfo().getEncodedName());
231 this.setDaemon(true);
232 this.server = server;
233 this.services = services;
234 this.region = region;
235 this.signaller = signaller;
236 }
237
238 public void run() {
239 try {
240 this.services.postOpenDeployTasks(this.region,
241 this.server.getCatalogTracker(), false);
242 } catch (Exception e) {
243 LOG.warn("Exception running postOpenDeployTasks; region=" +
244 this.region.getRegionInfo().getEncodedName(), e);
245 this.exception = e;
246 }
247
248 this.signaller.set(true);
249 synchronized (this.signaller) {
250 this.signaller.notify();
251 }
252 }
253
254
255
256
257 Exception getException() {
258 return this.exception;
259 }
260 }
261
262
263
264
265
266
267
268 private boolean transitionToOpened(final HRegion r) throws IOException {
269 boolean result = false;
270 HRegionInfo hri = r.getRegionInfo();
271 final String name = hri.getRegionNameAsString();
272
273 try {
274 if (ZKAssign.transitionNodeOpened(this.server.getZooKeeper(), hri,
275 this.server.getServerName(), this.version) == -1) {
276 LOG.warn("Completed the OPEN of region " + name +
277 " but when transitioning from " +
278 " OPENING to OPENED got a version mismatch, someone else clashed " +
279 "so now unassigning -- closing region on server: " +
280 this.server.getServerName());
281 } else {
282 LOG.debug("region transitioned to opened in zookeeper: " +
283 r.getRegionInfo() + ", server: " + this.server.getServerName());
284 result = true;
285 }
286 } catch (KeeperException e) {
287 LOG.error("Failed transitioning node " + name +
288 " from OPENING to OPENED -- closing region", e);
289 }
290 return result;
291 }
292
293
294
295
296
297
298 private boolean tryTransitionToFailedOpen(final HRegionInfo hri) {
299 boolean result = false;
300 final String name = hri.getRegionNameAsString();
301 try {
302 LOG.info("Opening of region " + hri + " failed, marking as FAILED_OPEN in ZK");
303 if (ZKAssign.transitionNode(
304 this.server.getZooKeeper(), hri,
305 this.server.getServerName(),
306 EventType.RS_ZK_REGION_OPENING,
307 EventType.RS_ZK_REGION_FAILED_OPEN,
308 this.version) == -1) {
309 LOG.warn("Unable to mark region " + hri + " as FAILED_OPEN. " +
310 "It's likely that the master already timed out this open " +
311 "attempt, and thus another RS already has the region.");
312 } else {
313 result = true;
314 }
315 } catch (KeeperException e) {
316 LOG.error("Failed transitioning node " + name +
317 " from OPENING to FAILED_OPEN", e);
318 }
319 return result;
320 }
321
322
323
324
325 HRegion openRegion() {
326 HRegion region = null;
327 try {
328
329
330 region = HRegion.openHRegion(this.regionInfo, this.htd,
331 this.rsServices.getWAL(this.regionInfo),
332 this.server.getConfiguration(),
333 this.rsServices,
334 new CancelableProgressable() {
335 public boolean progress() {
336
337
338
339 return tickleOpening("open_region_progress");
340 }
341 });
342 } catch (Throwable t) {
343
344
345
346 LOG.error(
347 "Failed open of region=" + this.regionInfo.getRegionNameAsString()
348 + ", starting to roll back the global memstore size.", t);
349
350 if (this.rsServices != null) {
351 RegionServerAccounting rsAccounting =
352 this.rsServices.getRegionServerAccounting();
353 if (rsAccounting != null) {
354 rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName());
355 }
356 }
357 }
358 return region;
359 }
360
361 void cleanupFailedOpen(final HRegion region) throws IOException {
362 if (region != null) region.close();
363 }
364
365
366
367
368
369
370
371
372 boolean tickleOpening(final String context) {
373
374 if (!isGoodVersion()) return false;
375 String encodedName = this.regionInfo.getEncodedName();
376 try {
377 this.version =
378 ZKAssign.retransitionNodeOpening(server.getZooKeeper(),
379 this.regionInfo, this.server.getServerName(), this.version);
380 } catch (KeeperException e) {
381 server.abort("Exception refreshing OPENING; region=" + encodedName +
382 ", context=" + context, e);
383 this.version = -1;
384 }
385 boolean b = isGoodVersion();
386 if (!b) {
387 LOG.warn("Failed refreshing OPENING; region=" + encodedName +
388 ", context=" + context);
389 }
390 return b;
391 }
392
393 private boolean isGoodVersion() {
394 return this.version != -1;
395 }
396 }