1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master.handler;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.NavigableMap;
27 import java.util.Set;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.Server;
34 import org.apache.hadoop.hbase.ServerName;
35 import org.apache.hadoop.hbase.catalog.CatalogTracker;
36 import org.apache.hadoop.hbase.catalog.MetaEditor;
37 import org.apache.hadoop.hbase.catalog.MetaReader;
38 import org.apache.hadoop.hbase.client.Result;
39 import org.apache.hadoop.hbase.executor.EventHandler;
40 import org.apache.hadoop.hbase.master.AssignmentManager;
41 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
42 import org.apache.hadoop.hbase.master.DeadServer;
43 import org.apache.hadoop.hbase.master.MasterServices;
44 import org.apache.hadoop.hbase.master.ServerManager;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.Pair;
47 import org.apache.hadoop.hbase.util.Threads;
48 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
49 import org.apache.zookeeper.KeeperException;
50
51
52
53
54
55
56 public class ServerShutdownHandler extends EventHandler {
57 private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
58 protected final ServerName serverName;
59 protected final MasterServices services;
60 protected final DeadServer deadServers;
61 protected final boolean shouldSplitHlog;
62
63 public ServerShutdownHandler(final Server server, final MasterServices services,
64 final DeadServer deadServers, final ServerName serverName,
65 final boolean shouldSplitHlog) {
66 this(server, services, deadServers, serverName, EventType.M_SERVER_SHUTDOWN,
67 shouldSplitHlog);
68 }
69
70 ServerShutdownHandler(final Server server, final MasterServices services,
71 final DeadServer deadServers, final ServerName serverName, EventType type,
72 final boolean shouldSplitHlog) {
73 super(server, type);
74 this.serverName = serverName;
75 this.server = server;
76 this.services = services;
77 this.deadServers = deadServers;
78 if (!this.deadServers.contains(this.serverName)) {
79 LOG.warn(this.serverName + " is NOT in deadservers; it should be!");
80 }
81 this.shouldSplitHlog = shouldSplitHlog;
82 }
83
84 @Override
85 public String getInformativeName() {
86 if (serverName != null) {
87 return this.getClass().getSimpleName() + " for " + serverName;
88 } else {
89 return super.getInformativeName();
90 }
91 }
92
93
94
95
96 boolean isCarryingRoot() {
97 return false;
98 }
99
100
101
102
103 boolean isCarryingMeta() {
104 return false;
105 }
106
107 @Override
108 public String toString() {
109 String name = "UnknownServerName";
110 if(server != null && server.getServerName() != null) {
111 name = server.getServerName().toString();
112 }
113 return getClass().getSimpleName() + "-" + name + "-" + getSeqid();
114 }
115
116 @Override
117 public void process() throws IOException {
118 final ServerName serverName = this.serverName;
119 try {
120 if (this.server.isStopped()) {
121 throw new IOException("Server is stopped");
122 }
123
124 try {
125 if (this.shouldSplitHlog) {
126 LOG.info("Splitting logs for " + serverName);
127 this.services.getMasterFileSystem().splitLog(serverName);
128 } else {
129 LOG.info("Skipping log splitting for " + serverName);
130 }
131 } catch (IOException ioe) {
132
133
134 this.services.getExecutorService().submit((ServerShutdownHandler)this);
135 this.deadServers.add(serverName);
136 throw new IOException("failed log splitting for " +
137 serverName + ", will retry", ioe);
138 }
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157 if (isCarryingRoot() || isCarryingMeta()) {
158 this.services.getExecutorService().submit(new ServerShutdownHandler(
159 this.server, this.services, this.deadServers, serverName, false));
160 this.deadServers.add(serverName);
161 return;
162 }
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180 NavigableMap<HRegionInfo, Result> hris = null;
181 while (!this.server.isStopped()) {
182 try {
183 this.server.getCatalogTracker().waitForMeta();
184 hris = MetaReader.getServerUserRegions(this.server.getCatalogTracker(),
185 this.serverName);
186 break;
187 } catch (InterruptedException e) {
188 Thread.currentThread().interrupt();
189 throw new IOException("Interrupted", e);
190 } catch (IOException ioe) {
191 LOG.info("Received exception accessing META during server shutdown of " +
192 serverName + ", retrying META read", ioe);
193 }
194 }
195
196
197
198 Pair<Set<HRegionInfo>, List<RegionState>> p = this.services.getAssignmentManager()
199 .processServerShutdown(this.serverName);
200 Set<HRegionInfo> ritsGoingToServer = p.getFirst();
201 List<RegionState> ritsOnServer = p.getSecond();
202
203 List<HRegionInfo> regionsToAssign = getRegionsToAssign(hris, ritsOnServer, ritsGoingToServer);
204 for (HRegionInfo hri : ritsGoingToServer) {
205 if (!this.services.getAssignmentManager().isRegionAssigned(hri)) {
206 if (!regionsToAssign.contains(hri)) {
207 regionsToAssign.add(hri);
208 RegionState rit =
209 services.getAssignmentManager().getRegionsInTransition().get(hri.getEncodedName());
210 removeRITsOfRregionInDisablingOrDisabledTables(regionsToAssign, rit,
211 services.getAssignmentManager(), hri);
212 }
213 }
214 }
215
216
217 for (HRegionInfo hri : regionsToAssign) {
218 this.services.getAssignmentManager().assign(hri, true);
219 }
220 LOG.info(regionsToAssign.size() + " regions which were planned to open on " + this.serverName
221 + " have been re-assigned.");
222 } finally {
223 this.deadServers.finish(serverName);
224 }
225 LOG.info("Finished processing of shutdown of " + serverName);
226 }
227
228
229
230
231
232
233
234
235
236 private List<HRegionInfo> getRegionsToAssign(final NavigableMap<HRegionInfo, Result> metaHRIs,
237 final List<RegionState> ritsOnServer, Set<HRegionInfo> ritsGoingToServer) throws IOException {
238 List<HRegionInfo> toAssign = new ArrayList<HRegionInfo>();
239
240
241 if (metaHRIs == null || metaHRIs.isEmpty()) return toAssign;
242
243
244
245
246
247
248
249
250 for (RegionState rs : ritsOnServer) {
251 if (!rs.isClosing() && !rs.isPendingClose() && !rs.isSplitting()) {
252 LOG.debug("Removed " + rs.getRegion().getRegionNameAsString()
253 + " from list of regions to assign because region state: " + rs.getState());
254 metaHRIs.remove(rs.getRegion());
255 }
256 }
257
258 AssignmentManager assignmentManager = this.services.getAssignmentManager();
259 for (Map.Entry<HRegionInfo, Result> e : metaHRIs.entrySet()) {
260 RegionState rit =
261 assignmentManager.getRegionsInTransition().get(e.getKey().getEncodedName());
262
263 if (processDeadRegion(e.getKey(), e.getValue(), assignmentManager,
264 this.server.getCatalogTracker())) {
265 ServerName addressFromAM = assignmentManager.getRegionServerOfRegion(e.getKey());
266 if (rit != null && !rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()
267 && !ritsGoingToServer.contains(e.getKey())) {
268
269
270 LOG.info("Skip assigning region " + rit.toString());
271 } else if (addressFromAM != null && !addressFromAM.equals(this.serverName)) {
272 LOG.debug("Skip assigning region " + e.getKey().getRegionNameAsString()
273 + " because it has been opened in " + addressFromAM.getServerName());
274 ritsGoingToServer.remove(e.getKey());
275 } else {
276 if (rit != null) {
277
278 try {
279 LOG.info("Reassigning region with rs =" + rit + " and deleting zk node if exists");
280 ZKAssign.deleteNodeFailSilent(services.getZooKeeper(), e.getKey());
281 } catch (KeeperException ke) {
282 this.server.abort("Unexpected ZK exception deleting unassigned node " + e.getKey(),
283 ke);
284 return null;
285 }
286 }
287 toAssign.add(e.getKey());
288 }
289 } else if (rit != null && (rit.isSplitting() || rit.isSplit())) {
290
291
292
293
294
295 HRegionInfo region = rit.getRegion();
296 AssignmentManager am = assignmentManager;
297 am.regionOffline(region);
298 ritsGoingToServer.remove(region);
299 }
300
301
302
303
304
305
306 removeRITsOfRregionInDisablingOrDisabledTables(toAssign, rit, assignmentManager, e.getKey());
307 }
308
309 return toAssign;
310 }
311
312 private void removeRITsOfRregionInDisablingOrDisabledTables(List<HRegionInfo> toAssign,
313 RegionState rit, AssignmentManager assignmentManager, HRegionInfo hri) {
314
315 if (!assignmentManager.getZKTable().isDisablingOrDisabledTable(hri.getTableNameAsString())) {
316 return;
317 }
318
319
320 toAssign.remove(hri);
321
322 if (rit != null) {
323 assignmentManager.deleteNodeAndOfflineRegion(hri);
324 }
325 }
326
327
328
329
330
331
332
333
334
335
336
337 public static boolean processDeadRegion(HRegionInfo hri, Result result,
338 AssignmentManager assignmentManager, CatalogTracker catalogTracker)
339 throws IOException {
340 boolean tablePresent = assignmentManager.getZKTable().isTablePresent(
341 hri.getTableNameAsString());
342 if (!tablePresent) {
343 LOG.info("The table " + hri.getTableNameAsString()
344 + " was deleted. Hence not proceeding.");
345 return false;
346 }
347
348 boolean disabled = assignmentManager.getZKTable().isDisabledTable(
349 hri.getTableNameAsString());
350 if (disabled){
351 LOG.info("The table " + hri.getTableNameAsString()
352 + " was disabled. Hence not proceeding.");
353 return false;
354 }
355 if (hri.isOffline() && hri.isSplit()) {
356 LOG.debug("Offlined and split region " + hri.getRegionNameAsString() +
357 "; checking daughter presence");
358 if (MetaReader.getRegion(catalogTracker, hri.getRegionName()) == null) {
359 return false;
360 }
361 fixupDaughters(result, assignmentManager, catalogTracker);
362 return false;
363 }
364 boolean disabling = assignmentManager.getZKTable().isDisablingTable(
365 hri.getTableNameAsString());
366 if (disabling) {
367 LOG.info("The table " + hri.getTableNameAsString()
368 + " is disabled. Hence not assigning region" + hri.getEncodedName());
369 return false;
370 }
371 return true;
372 }
373
374
375
376
377
378
379
380
381 public static int fixupDaughters(final Result result,
382 final AssignmentManager assignmentManager,
383 final CatalogTracker catalogTracker)
384 throws IOException {
385 int fixedA = fixupDaughter(result, HConstants.SPLITA_QUALIFIER,
386 assignmentManager, catalogTracker);
387 int fixedB = fixupDaughter(result, HConstants.SPLITB_QUALIFIER,
388 assignmentManager, catalogTracker);
389 return fixedA + fixedB;
390 }
391
392
393
394
395
396
397
398
399 static int fixupDaughter(final Result result, final byte [] qualifier,
400 final AssignmentManager assignmentManager,
401 final CatalogTracker catalogTracker)
402 throws IOException {
403 HRegionInfo daughter =
404 MetaReader.parseHRegionInfoFromCatalogResult(result, qualifier);
405 if (daughter == null) return 0;
406 if (isDaughterMissing(catalogTracker, daughter)) {
407 LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString());
408 MetaEditor.addDaughter(catalogTracker, daughter, null);
409
410
411
412
413
414
415 assignmentManager.assign(daughter, true);
416 return 1;
417 } else {
418 LOG.debug("Daughter " + daughter.getRegionNameAsString() + " present");
419 }
420 return 0;
421 }
422
423
424
425
426
427
428
429
430 private static boolean isDaughterMissing(final CatalogTracker catalogTracker,
431 final HRegionInfo daughter) throws IOException {
432 FindDaughterVisitor visitor = new FindDaughterVisitor(daughter);
433
434
435
436
437
438
439 byte [] startrow = daughter.getRegionName();
440 MetaReader.fullScan(catalogTracker, visitor, startrow);
441 return !visitor.foundDaughter();
442 }
443
444
445
446
447
448 static class FindDaughterVisitor implements MetaReader.Visitor {
449 private final HRegionInfo daughter;
450 private boolean found = false;
451
452 FindDaughterVisitor(final HRegionInfo daughter) {
453 this.daughter = daughter;
454 }
455
456
457
458
459 boolean foundDaughter() {
460 return this.found;
461 }
462
463 @Override
464 public boolean visit(Result r) throws IOException {
465 HRegionInfo hri =
466 MetaReader.parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
467 if (hri == null) {
468 LOG.warn("No serialized HRegionInfo in " + r);
469 return true;
470 }
471 byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
472 HConstants.SERVER_QUALIFIER);
473
474 if (value == null) return false;
475
476
477 if (!Bytes.equals(daughter.getTableName(),
478 hri.getTableName())) {
479
480 return false;
481 }
482
483 if (!Bytes.equals(daughter.getStartKey(), hri.getStartKey())) {
484 return false;
485 }
486
487
488
489 this.found = true;
490 return false;
491 }
492 }
493 }