1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master.handler;
21
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.NavigableMap;
27 import java.util.Set;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.Server;
34 import org.apache.hadoop.hbase.ServerName;
35 import org.apache.hadoop.hbase.catalog.CatalogTracker;
36 import org.apache.hadoop.hbase.catalog.MetaEditor;
37 import org.apache.hadoop.hbase.catalog.MetaReader;
38 import org.apache.hadoop.hbase.client.Result;
39 import org.apache.hadoop.hbase.executor.EventHandler;
40 import org.apache.hadoop.hbase.master.AssignmentManager;
41 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
42 import org.apache.hadoop.hbase.master.DeadServer;
43 import org.apache.hadoop.hbase.master.MasterServices;
44 import org.apache.hadoop.hbase.master.ServerManager;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.Pair;
47 import org.apache.hadoop.hbase.util.Threads;
48 import org.apache.hadoop.hbase.zookeeper.ZKAssign;
49 import org.apache.zookeeper.KeeperException;
50
51
52
53
54
55
56 public class ServerShutdownHandler extends EventHandler {
57 private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
58 protected final ServerName serverName;
59 protected final MasterServices services;
60 protected final DeadServer deadServers;
61 protected final boolean shouldSplitHlog;
62
63 public ServerShutdownHandler(final Server server, final MasterServices services,
64 final DeadServer deadServers, final ServerName serverName,
65 final boolean shouldSplitHlog) {
66 this(server, services, deadServers, serverName, EventType.M_SERVER_SHUTDOWN,
67 shouldSplitHlog);
68 }
69
70 ServerShutdownHandler(final Server server, final MasterServices services,
71 final DeadServer deadServers, final ServerName serverName, EventType type,
72 final boolean shouldSplitHlog) {
73 super(server, type);
74 this.serverName = serverName;
75 this.server = server;
76 this.services = services;
77 this.deadServers = deadServers;
78 if (!this.deadServers.contains(this.serverName)) {
79 LOG.warn(this.serverName + " is NOT in deadservers; it should be!");
80 }
81 this.shouldSplitHlog = shouldSplitHlog;
82 }
83
84 @Override
85 public String getInformativeName() {
86 if (serverName != null) {
87 return this.getClass().getSimpleName() + " for " + serverName;
88 } else {
89 return super.getInformativeName();
90 }
91 }
92
93
94
95
96 boolean isCarryingRoot() {
97 return false;
98 }
99
100
101
102
103 boolean isCarryingMeta() {
104 return false;
105 }
106
107 @Override
108 public String toString() {
109 String name = "UnknownServerName";
110 if(server != null && server.getServerName() != null) {
111 name = server.getServerName().toString();
112 }
113 return getClass().getSimpleName() + "-" + name + "-" + getSeqid();
114 }
115
116 @Override
117 public void process() throws IOException {
118 final ServerName serverName = this.serverName;
119 try {
120 if (this.server.isStopped()) {
121 throw new IOException("Server is stopped");
122 }
123
124 try {
125 if (this.shouldSplitHlog) {
126 LOG.info("Splitting logs for " + serverName);
127 this.services.getMasterFileSystem().splitLog(serverName);
128 } else {
129 LOG.info("Skipping log splitting for " + serverName);
130 }
131 } catch (IOException ioe) {
132
133
134 this.services.getExecutorService().submit((ServerShutdownHandler)this);
135 this.deadServers.add(serverName);
136 throw new IOException("failed log splitting for " +
137 serverName + ", will retry", ioe);
138 }
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157 if (isCarryingRoot() || isCarryingMeta()) {
158 this.services.getExecutorService().submit(new ServerShutdownHandler(
159 this.server, this.services, this.deadServers, serverName, false));
160 this.deadServers.add(serverName);
161 return;
162 }
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180 NavigableMap<HRegionInfo, Result> hris = null;
181 while (!this.server.isStopped()) {
182 try {
183 this.server.getCatalogTracker().waitForMeta();
184
185 if (!this.server.isStopped()) {
186 hris = MetaReader.getServerUserRegions(this.server.getCatalogTracker(),
187 this.serverName);
188 }
189 break;
190 } catch (InterruptedException e) {
191 Thread.currentThread().interrupt();
192 throw new IOException("Interrupted", e);
193 } catch (IOException ioe) {
194 LOG.info("Received exception accessing META during server shutdown of " +
195 serverName + ", retrying META read", ioe);
196 }
197 }
198
199
200
201 Pair<Set<HRegionInfo>, List<RegionState>> p = this.services.getAssignmentManager()
202 .processServerShutdown(this.serverName);
203 Set<HRegionInfo> ritsGoingToServer = p.getFirst();
204 List<RegionState> ritsOnServer = p.getSecond();
205
206 List<HRegionInfo> regionsToAssign = getRegionsToAssign(hris, ritsOnServer, ritsGoingToServer);
207 for (HRegionInfo hri : ritsGoingToServer) {
208 if (!this.services.getAssignmentManager().isRegionAssigned(hri)) {
209 if (!regionsToAssign.contains(hri)) {
210 regionsToAssign.add(hri);
211 RegionState rit =
212 services.getAssignmentManager().getRegionsInTransition().get(hri.getEncodedName());
213 removeRITsOfRregionInDisablingOrDisabledTables(regionsToAssign, rit,
214 services.getAssignmentManager(), hri);
215 }
216 }
217 }
218
219
220 for (HRegionInfo hri : regionsToAssign) {
221 this.services.getAssignmentManager().assign(hri, true);
222 }
223 LOG.info(regionsToAssign.size() + " regions which were planned to open on " + this.serverName
224 + " have been re-assigned.");
225 } finally {
226 this.deadServers.finish(serverName);
227 }
228 LOG.info("Finished processing of shutdown of " + serverName);
229 }
230
231
232
233
234
235
236
237
238
239 private List<HRegionInfo> getRegionsToAssign(final NavigableMap<HRegionInfo, Result> metaHRIs,
240 final List<RegionState> ritsOnServer, Set<HRegionInfo> ritsGoingToServer) throws IOException {
241 List<HRegionInfo> toAssign = new ArrayList<HRegionInfo>();
242
243
244 if (metaHRIs == null || metaHRIs.isEmpty()) return toAssign;
245
246
247
248
249
250
251
252
253 for (RegionState rs : ritsOnServer) {
254 if (!rs.isClosing() && !rs.isPendingClose() && !rs.isSplitting()) {
255 LOG.debug("Removed " + rs.getRegion().getRegionNameAsString()
256 + " from list of regions to assign because region state: " + rs.getState());
257 metaHRIs.remove(rs.getRegion());
258 }
259 }
260
261 AssignmentManager assignmentManager = this.services.getAssignmentManager();
262 for (Map.Entry<HRegionInfo, Result> e : metaHRIs.entrySet()) {
263 RegionState rit =
264 assignmentManager.getRegionsInTransition().get(e.getKey().getEncodedName());
265
266 if (processDeadRegion(e.getKey(), e.getValue(), assignmentManager,
267 this.server.getCatalogTracker())) {
268 ServerName addressFromAM = assignmentManager.getRegionServerOfRegion(e.getKey());
269 if (rit != null && !rit.isClosing() && !rit.isPendingClose() && !rit.isSplitting()
270 && !ritsGoingToServer.contains(e.getKey())) {
271
272
273 LOG.info("Skip assigning region " + rit.toString());
274 } else if (addressFromAM != null && !addressFromAM.equals(this.serverName)) {
275 LOG.debug("Skip assigning region " + e.getKey().getRegionNameAsString()
276 + " because it has been opened in " + addressFromAM.getServerName());
277 ritsGoingToServer.remove(e.getKey());
278 } else {
279 if (rit != null) {
280
281 try {
282 LOG.info("Reassigning region with rs =" + rit + " and deleting zk node if exists");
283 ZKAssign.deleteNodeFailSilent(services.getZooKeeper(), e.getKey());
284 } catch (KeeperException ke) {
285 this.server.abort("Unexpected ZK exception deleting unassigned node " + e.getKey(),
286 ke);
287 return null;
288 }
289 }
290 toAssign.add(e.getKey());
291 }
292 } else if (rit != null && (rit.isSplitting() || rit.isSplit())) {
293
294
295
296
297
298 HRegionInfo region = rit.getRegion();
299 AssignmentManager am = assignmentManager;
300 am.regionOffline(region);
301 ritsGoingToServer.remove(region);
302 }
303
304
305
306
307
308
309 removeRITsOfRregionInDisablingOrDisabledTables(toAssign, rit, assignmentManager, e.getKey());
310 }
311
312 return toAssign;
313 }
314
315 private void removeRITsOfRregionInDisablingOrDisabledTables(List<HRegionInfo> toAssign,
316 RegionState rit, AssignmentManager assignmentManager, HRegionInfo hri) {
317
318 if (!assignmentManager.getZKTable().isDisablingOrDisabledTable(hri.getTableNameAsString())) {
319 return;
320 }
321
322
323 toAssign.remove(hri);
324
325 if (rit != null) {
326 assignmentManager.deleteNodeAndOfflineRegion(hri);
327 }
328 }
329
330
331
332
333
334
335
336
337
338
339
340 public static boolean processDeadRegion(HRegionInfo hri, Result result,
341 AssignmentManager assignmentManager, CatalogTracker catalogTracker)
342 throws IOException {
343 boolean tablePresent = assignmentManager.getZKTable().isTablePresent(
344 hri.getTableNameAsString());
345 if (!tablePresent) {
346 LOG.info("The table " + hri.getTableNameAsString()
347 + " was deleted. Hence not proceeding.");
348 return false;
349 }
350
351 boolean disabled = assignmentManager.getZKTable().isDisabledTable(
352 hri.getTableNameAsString());
353 if (disabled){
354 LOG.info("The table " + hri.getTableNameAsString()
355 + " was disabled. Hence not proceeding.");
356 return false;
357 }
358 if (hri.isOffline() && hri.isSplit()) {
359 LOG.debug("Offlined and split region " + hri.getRegionNameAsString() +
360 "; checking daughter presence");
361 if (MetaReader.getRegion(catalogTracker, hri.getRegionName()) == null) {
362 return false;
363 }
364 fixupDaughters(result, assignmentManager, catalogTracker);
365 return false;
366 }
367 boolean disabling = assignmentManager.getZKTable().isDisablingTable(
368 hri.getTableNameAsString());
369 if (disabling) {
370 LOG.info("The table " + hri.getTableNameAsString()
371 + " is disabled. Hence not assigning region" + hri.getEncodedName());
372 return false;
373 }
374 return true;
375 }
376
377
378
379
380
381
382
383
384 public static int fixupDaughters(final Result result,
385 final AssignmentManager assignmentManager,
386 final CatalogTracker catalogTracker)
387 throws IOException {
388 int fixedA = fixupDaughter(result, HConstants.SPLITA_QUALIFIER,
389 assignmentManager, catalogTracker);
390 int fixedB = fixupDaughter(result, HConstants.SPLITB_QUALIFIER,
391 assignmentManager, catalogTracker);
392 return fixedA + fixedB;
393 }
394
395
396
397
398
399
400
401
402 static int fixupDaughter(final Result result, final byte [] qualifier,
403 final AssignmentManager assignmentManager,
404 final CatalogTracker catalogTracker)
405 throws IOException {
406 HRegionInfo daughter =
407 MetaReader.parseHRegionInfoFromCatalogResult(result, qualifier);
408 if (daughter == null) return 0;
409 if (isDaughterMissing(catalogTracker, daughter)) {
410 LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString());
411 MetaEditor.addDaughter(catalogTracker, daughter, null);
412
413
414
415
416
417
418 assignmentManager.assign(daughter, true);
419 return 1;
420 } else {
421 LOG.debug("Daughter " + daughter.getRegionNameAsString() + " present");
422 }
423 return 0;
424 }
425
426
427
428
429
430
431
432
433 private static boolean isDaughterMissing(final CatalogTracker catalogTracker,
434 final HRegionInfo daughter) throws IOException {
435 FindDaughterVisitor visitor = new FindDaughterVisitor(daughter);
436
437
438
439
440
441
442 byte [] startrow = daughter.getRegionName();
443 MetaReader.fullScan(catalogTracker, visitor, startrow);
444 return !visitor.foundDaughter();
445 }
446
447
448
449
450
451 static class FindDaughterVisitor implements MetaReader.Visitor {
452 private final HRegionInfo daughter;
453 private boolean found = false;
454
455 FindDaughterVisitor(final HRegionInfo daughter) {
456 this.daughter = daughter;
457 }
458
459
460
461
462 boolean foundDaughter() {
463 return this.found;
464 }
465
466 @Override
467 public boolean visit(Result r) throws IOException {
468 HRegionInfo hri =
469 MetaReader.parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
470 if (hri == null) {
471 LOG.warn("No serialized HRegionInfo in " + r);
472 return true;
473 }
474 byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
475 HConstants.SERVER_QUALIFIER);
476
477 if (value == null) return false;
478
479
480 if (!Bytes.equals(daughter.getTableName(),
481 hri.getTableName())) {
482
483 return false;
484 }
485
486 if (!Bytes.equals(daughter.getStartKey(), hri.getStartKey())) {
487 return false;
488 }
489
490
491
492 this.found = true;
493 return false;
494 }
495 }
496 }