1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.handler;
20
21 import java.io.IOException;
22 import java.util.concurrent.atomic.AtomicBoolean;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.hbase.classification.InterfaceAudience;
27 import org.apache.hadoop.hbase.HRegionInfo;
28 import org.apache.hadoop.hbase.HTableDescriptor;
29 import org.apache.hadoop.hbase.Server;
30 import org.apache.hadoop.hbase.coordination.OpenRegionCoordination;
31 import org.apache.hadoop.hbase.executor.EventHandler;
32 import org.apache.hadoop.hbase.executor.EventType;
33 import org.apache.hadoop.hbase.protobuf.generated.RegionServerStatusProtos.RegionStateTransition.TransitionCode;
34 import org.apache.hadoop.hbase.regionserver.HRegion;
35 import org.apache.hadoop.hbase.regionserver.RegionServerAccounting;
36 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
37 import org.apache.hadoop.hbase.util.CancelableProgressable;
38 import org.apache.hadoop.hbase.util.ConfigUtil;
39
40
41
42
43
44 @InterfaceAudience.Private
45 public class OpenRegionHandler extends EventHandler {
46 private static final Log LOG = LogFactory.getLog(OpenRegionHandler.class);
47
48 protected final RegionServerServices rsServices;
49
50 private final HRegionInfo regionInfo;
51 private final HTableDescriptor htd;
52
53 private OpenRegionCoordination coordination;
54 private OpenRegionCoordination.OpenRegionDetails ord;
55
56 private final boolean useZKForAssignment;
57
58 public OpenRegionHandler(final Server server,
59 final RegionServerServices rsServices, HRegionInfo regionInfo,
60 HTableDescriptor htd, OpenRegionCoordination coordination,
61 OpenRegionCoordination.OpenRegionDetails ord) {
62 this(server, rsServices, regionInfo, htd, EventType.M_RS_OPEN_REGION,
63 coordination, ord);
64 }
65
66 protected OpenRegionHandler(final Server server,
67 final RegionServerServices rsServices, final HRegionInfo regionInfo,
68 final HTableDescriptor htd, EventType eventType,
69 OpenRegionCoordination coordination, OpenRegionCoordination.OpenRegionDetails ord) {
70 super(server, eventType);
71 this.rsServices = rsServices;
72 this.regionInfo = regionInfo;
73 this.htd = htd;
74 this.coordination = coordination;
75 this.ord = ord;
76 useZKForAssignment = ConfigUtil.useZKForAssignment(server.getConfiguration());
77 }
78
79 public HRegionInfo getRegionInfo() {
80 return regionInfo;
81 }
82
83 @Override
84 public void process() throws IOException {
85 boolean openSuccessful = false;
86 boolean transitionedToOpening = false;
87 final String regionName = regionInfo.getRegionNameAsString();
88 HRegion region = null;
89
90 try {
91 if (this.server.isStopped() || this.rsServices.isStopping()) {
92 return;
93 }
94 final String encodedName = regionInfo.getEncodedName();
95
96
97
98
99
100
101
102 if (this.rsServices.getFromOnlineRegions(encodedName) != null) {
103 LOG.error("Region " + encodedName +
104 " was already online when we started processing the opening. " +
105 "Marking this new attempt as failed");
106 return;
107 }
108
109
110
111
112 if (!isRegionStillOpening()){
113 LOG.error("Region " + encodedName + " opening cancelled");
114 return;
115 }
116
117 if (useZKForAssignment
118 && !coordination.transitionFromOfflineToOpening(regionInfo, ord)) {
119 LOG.warn("Region was hijacked? Opening cancelled for encodedName=" + encodedName);
120
121 return;
122 }
123 transitionedToOpening = true;
124
125
126 region = openRegion();
127 if (region == null) {
128 return;
129 }
130
131 boolean failed = true;
132 if (isRegionStillOpening() && (!useZKForAssignment ||
133 coordination.tickleOpening(ord, regionInfo, rsServices, "post_region_open"))) {
134 if (updateMeta(region)) {
135 failed = false;
136 }
137 }
138 if (failed || this.server.isStopped() ||
139 this.rsServices.isStopping()) {
140 return;
141 }
142
143 if (!isRegionStillOpening() ||
144 (useZKForAssignment && !coordination.transitionToOpened(region, ord))) {
145
146
147
148
149
150 return;
151 }
152
153
154
155
156
157
158
159
160
161
162
163 this.rsServices.addToOnlineRegions(region);
164 openSuccessful = true;
165
166
167 LOG.debug("Opened " + regionName + " on " +
168 this.server.getServerName());
169
170
171 } finally {
172
173 if (!openSuccessful) {
174 doCleanUpOnFailedOpen(region, transitionedToOpening, ord);
175 }
176 final Boolean current = this.rsServices.getRegionsInTransitionInRS().
177 remove(this.regionInfo.getEncodedNameAsBytes());
178
179
180
181
182
183
184
185
186 if (openSuccessful) {
187 if (current == null) {
188 LOG.error("Bad state: we've just opened a region that was NOT in transition. Region="
189 + regionName);
190 } else if (Boolean.FALSE.equals(current)) {
191
192 LOG.error("Race condition: we've finished to open a region, while a close was requested "
193 + " on region=" + regionName + ". It can be a critical error, as a region that"
194 + " should be closed is now opened. Closing it now");
195 cleanupFailedOpen(region);
196 }
197 }
198 }
199 }
200
201 private void doCleanUpOnFailedOpen(HRegion region, boolean transitionedToOpening,
202 OpenRegionCoordination.OpenRegionDetails ord)
203 throws IOException {
204 if (transitionedToOpening) {
205 try {
206 if (region != null) {
207 cleanupFailedOpen(region);
208 }
209 } finally {
210 if (!useZKForAssignment) {
211 rsServices.reportRegionStateTransition(TransitionCode.FAILED_OPEN, regionInfo);
212 } else {
213
214
215 coordination.tryTransitionFromOpeningToFailedOpen(regionInfo, ord);
216 }
217 }
218 } else if (!useZKForAssignment) {
219 rsServices.reportRegionStateTransition(TransitionCode.FAILED_OPEN, regionInfo);
220 } else {
221
222
223 coordination.tryTransitionFromOfflineToFailedOpen(this.rsServices, regionInfo, ord);
224 }
225 }
226
227
228
229
230
231
232
233
234 boolean updateMeta(final HRegion r) {
235 if (this.server.isStopped() || this.rsServices.isStopping()) {
236 return false;
237 }
238
239
240 final AtomicBoolean signaller = new AtomicBoolean(false);
241 PostOpenDeployTasksThread t = new PostOpenDeployTasksThread(r,
242 this.server, this.rsServices, signaller);
243 t.start();
244
245
246
247
248
249 long now = System.currentTimeMillis();
250 long lastUpdate = now;
251 boolean tickleOpening = true;
252 while (!signaller.get() && t.isAlive() && !this.server.isStopped() &&
253 !this.rsServices.isStopping() && isRegionStillOpening()) {
254 long elapsed = now - lastUpdate;
255 if (elapsed > 120000) {
256
257 lastUpdate = now;
258 if (useZKForAssignment) {
259 tickleOpening = coordination.tickleOpening(
260 ord, regionInfo, rsServices, "post_open_deploy");
261 }
262 }
263 synchronized (signaller) {
264 try {
265
266
267 if (!signaller.get()) signaller.wait(10000);
268 } catch (InterruptedException e) {
269
270 }
271 }
272 now = System.currentTimeMillis();
273 }
274
275
276 if (t.isAlive()) {
277 if (!signaller.get()) {
278
279 LOG.debug("Interrupting thread " + t);
280 t.interrupt();
281 }
282 try {
283 t.join();
284 } catch (InterruptedException ie) {
285 LOG.warn("Interrupted joining " +
286 r.getRegionInfo().getRegionNameAsString(), ie);
287 Thread.currentThread().interrupt();
288 }
289 }
290
291
292
293
294 return ((!Thread.interrupted() && t.getException() == null) && tickleOpening);
295 }
296
297
298
299
300
301
302 static class PostOpenDeployTasksThread extends Thread {
303 private Throwable exception = null;
304 private final Server server;
305 private final RegionServerServices services;
306 private final HRegion region;
307 private final AtomicBoolean signaller;
308
309 PostOpenDeployTasksThread(final HRegion region, final Server server,
310 final RegionServerServices services, final AtomicBoolean signaller) {
311 super("PostOpenDeployTasks:" + region.getRegionInfo().getEncodedName());
312 this.setDaemon(true);
313 this.server = server;
314 this.services = services;
315 this.region = region;
316 this.signaller = signaller;
317 }
318
319 public void run() {
320 try {
321 this.services.postOpenDeployTasks(this.region);
322 } catch (Throwable e) {
323 String msg = "Exception running postOpenDeployTasks; region=" +
324 this.region.getRegionInfo().getEncodedName();
325 this.exception = e;
326 if (e instanceof IOException
327 && isRegionStillOpening(region.getRegionInfo(), services)) {
328 server.abort(msg, e);
329 } else {
330 LOG.warn(msg, e);
331 }
332 }
333
334 this.signaller.set(true);
335 synchronized (this.signaller) {
336 this.signaller.notify();
337 }
338 }
339
340
341
342
343 Throwable getException() {
344 return this.exception;
345 }
346 }
347
348
349
350
351 HRegion openRegion() {
352 HRegion region = null;
353 try {
354
355
356 region = HRegion.openHRegion(this.regionInfo, this.htd,
357 this.rsServices.getWAL(this.regionInfo),
358 this.server.getConfiguration(),
359 this.rsServices,
360 new CancelableProgressable() {
361 public boolean progress() {
362 if (useZKForAssignment) {
363
364 return coordination.tickleOpening(ord, regionInfo,
365 rsServices, "open_region_progress");
366 }
367 if (!isRegionStillOpening()) {
368 LOG.warn("Open region aborted since it isn't opening any more");
369 return false;
370 }
371 return true;
372 }
373 });
374 } catch (Throwable t) {
375
376
377
378 LOG.error(
379 "Failed open of region=" + this.regionInfo.getRegionNameAsString()
380 + ", starting to roll back the global memstore size.", t);
381
382 if (this.rsServices != null) {
383 RegionServerAccounting rsAccounting =
384 this.rsServices.getRegionServerAccounting();
385 if (rsAccounting != null) {
386 rsAccounting.rollbackRegionReplayEditsSize(this.regionInfo.getRegionName());
387 }
388 }
389 }
390 return region;
391 }
392
393 void cleanupFailedOpen(final HRegion region) throws IOException {
394 if (region != null) {
395 byte[] encodedName = regionInfo.getEncodedNameAsBytes();
396 try {
397 rsServices.getRegionsInTransitionInRS().put(encodedName,Boolean.FALSE);
398 this.rsServices.removeFromOnlineRegions(region, null);
399 region.close();
400 } finally {
401 rsServices.getRegionsInTransitionInRS().remove(encodedName);
402 }
403 }
404 }
405
406 private static boolean isRegionStillOpening(
407 HRegionInfo regionInfo, RegionServerServices rsServices) {
408 byte[] encodedName = regionInfo.getEncodedNameAsBytes();
409 Boolean action = rsServices.getRegionsInTransitionInRS().get(encodedName);
410 return Boolean.TRUE.equals(action);
411 }
412
413 private boolean isRegionStillOpening() {
414 return isRegionStillOpening(regionInfo, rsServices);
415 }
416 }