1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master.handler;
21
22 import java.io.IOException;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.NavigableMap;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.HRegionInfo;
31 import org.apache.hadoop.hbase.HServerInfo;
32 import org.apache.hadoop.hbase.Server;
33 import org.apache.hadoop.hbase.catalog.CatalogTracker;
34 import org.apache.hadoop.hbase.catalog.MetaEditor;
35 import org.apache.hadoop.hbase.catalog.MetaReader;
36 import org.apache.hadoop.hbase.client.Result;
37 import org.apache.hadoop.hbase.executor.EventHandler;
38 import org.apache.hadoop.hbase.master.AssignmentManager;
39 import org.apache.hadoop.hbase.master.AssignmentManager.RegionState;
40 import org.apache.hadoop.hbase.master.DeadServer;
41 import org.apache.hadoop.hbase.master.MasterServices;
42 import org.apache.hadoop.hbase.master.ServerManager;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.apache.hadoop.hbase.util.Writables;
45 import org.apache.zookeeper.KeeperException;
46
47
48
49
50
51
52 public class ServerShutdownHandler extends EventHandler {
53 private static final Log LOG = LogFactory.getLog(ServerShutdownHandler.class);
54 private final HServerInfo hsi;
55 private final Server server;
56 private final MasterServices services;
57 private final DeadServer deadServers;
58
59 public ServerShutdownHandler(final Server server, final MasterServices services,
60 final DeadServer deadServers, final HServerInfo hsi) {
61 this(server, services, deadServers, hsi, EventType.M_SERVER_SHUTDOWN);
62 }
63
64 ServerShutdownHandler(final Server server, final MasterServices services,
65 final DeadServer deadServers, final HServerInfo hsi, EventType type) {
66 super(server, type);
67 this.hsi = hsi;
68 this.server = server;
69 this.services = services;
70 this.deadServers = deadServers;
71 if (!this.deadServers.contains(hsi.getServerName())) {
72 LOG.warn(hsi.getServerName() + " is NOT in deadservers; it should be!");
73 }
74 }
75
76
77
78
79
80
81
82
83
84
85
86
87 private void verifyAndAssignRoot()
88 throws InterruptedException, IOException, KeeperException {
89 long timeout = this.server.getConfiguration().
90 getLong("hbase.catalog.verification.timeout", 1000);
91 if (!this.server.getCatalogTracker().verifyRootRegionLocation(timeout)) {
92 this.services.getAssignmentManager().assignRoot();
93 }
94 }
95
96
97
98
99
100 private void verifyAndAssignRootWithRetries() throws IOException {
101 int iTimes = this.server.getConfiguration().getInt(
102 "hbase.catalog.verification.retries", 10);
103
104 long waitTime = this.server.getConfiguration().getLong(
105 "hbase.catalog.verification.timeout", 1000);
106
107 int iFlag = 0;
108 while (true) {
109 try {
110 verifyAndAssignRoot();
111 break;
112 } catch (KeeperException e) {
113 this.server.abort("In server shutdown processing, assigning root", e);
114 throw new IOException("Aborting", e);
115 } catch (Exception e) {
116 if (iFlag >= iTimes) {
117 this.server.abort("verifyAndAssignRoot failed after" + iTimes
118 + " times retries, aborting", e);
119 throw new IOException("Aborting", e);
120 }
121 try {
122 Thread.sleep(waitTime);
123 } catch (InterruptedException e1) {
124 LOG.warn("Interrupted when is the thread sleep", e1);
125 Thread.currentThread().interrupt();
126 throw new IOException("Interrupted", e1);
127 }
128 iFlag++;
129 }
130 }
131 }
132
133
134
135
136 boolean isCarryingRoot() {
137 return false;
138 }
139
140
141
142
143 boolean isCarryingMeta() {
144 return false;
145 }
146
147 @Override
148 public void process() throws IOException {
149 final String serverName = this.hsi.getServerName();
150
151 LOG.info("Splitting logs for " + serverName);
152 this.services.getMasterFileSystem().splitLog(serverName);
153
154
155
156
157
158 List<RegionState> regionsInTransition =
159 this.services.getAssignmentManager().processServerShutdown(this.hsi);
160
161
162 if (isCarryingRoot()) {
163 verifyAndAssignRootWithRetries();
164 }
165
166
167 if (isCarryingMeta()) this.services.getAssignmentManager().assignMeta();
168
169
170
171
172 NavigableMap<HRegionInfo, Result> hris = null;
173 while (!this.server.isStopped()) {
174 try {
175 this.server.getCatalogTracker().waitForMeta();
176 hris = MetaReader.getServerUserRegions(this.server.getCatalogTracker(),
177 this.hsi);
178 break;
179 } catch (InterruptedException e) {
180 Thread.currentThread().interrupt();
181 throw new IOException("Interrupted", e);
182 } catch (IOException ioe) {
183 LOG.info("Received exception accessing META during server shutdown of " +
184 serverName + ", retrying META read", ioe);
185 }
186 }
187
188
189 for (RegionState rit : regionsInTransition) {
190 if (!rit.isClosing() && !rit.isPendingClose()) {
191 LOG.debug("Removed " + rit.getRegion().getRegionNameAsString() +
192 " from list of regions to assign because in RIT");
193 hris.remove(rit.getRegion());
194 }
195 }
196
197 LOG.info("Reassigning " + (hris == null? 0: hris.size()) +
198 " region(s) that " + serverName +
199 " was carrying (skipping " + regionsInTransition.size() +
200 " regions(s) that are already in transition)");
201
202
203 for (Map.Entry<HRegionInfo, Result> e: hris.entrySet()) {
204 if (processDeadRegion(e.getKey(), e.getValue(),
205 this.services.getAssignmentManager(),
206 this.server.getCatalogTracker())) {
207 this.services.getAssignmentManager().assign(e.getKey(), true);
208 }
209 }
210 this.deadServers.finish(serverName);
211 LOG.info("Finished processing of shutdown of " + serverName);
212 }
213
214
215
216
217
218
219
220
221
222
223
224 public static boolean processDeadRegion(HRegionInfo hri, Result result,
225 AssignmentManager assignmentManager, CatalogTracker catalogTracker)
226 throws IOException {
227
228 boolean disabled = assignmentManager.getZKTable().isDisabledTable(
229 hri.getTableDesc().getNameAsString());
230 if (disabled) return false;
231 if (hri.isOffline() && hri.isSplit()) {
232 LOG.debug("Offlined and split region " + hri.getRegionNameAsString() +
233 "; checking daughter presence");
234 fixupDaughters(result, assignmentManager, catalogTracker);
235 return false;
236 }
237 return true;
238 }
239
240
241
242
243
244
245
246 static void fixupDaughters(final Result result,
247 final AssignmentManager assignmentManager,
248 final CatalogTracker catalogTracker)
249 throws IOException {
250 fixupDaughter(result, HConstants.SPLITA_QUALIFIER, assignmentManager,
251 catalogTracker);
252 fixupDaughter(result, HConstants.SPLITB_QUALIFIER, assignmentManager,
253 catalogTracker);
254 }
255
256
257
258
259
260
261
262 static void fixupDaughter(final Result result, final byte [] qualifier,
263 final AssignmentManager assignmentManager,
264 final CatalogTracker catalogTracker)
265 throws IOException {
266 HRegionInfo daughter = getHRegionInfo(result, qualifier);
267 if (daughter == null) return;
268 if (isDaughterMissing(catalogTracker, daughter)) {
269 LOG.info("Fixup; missing daughter " + daughter.getRegionNameAsString());
270 MetaEditor.addDaughter(catalogTracker, daughter, null);
271
272 assignmentManager.assign(daughter, true);
273 } else {
274 LOG.debug("Daughter " + daughter.getRegionNameAsString() + " present");
275 }
276 }
277
278
279
280
281
282
283
284
285
286 private static HRegionInfo getHRegionInfo(final Result r, byte [] qualifier)
287 throws IOException {
288 byte [] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
289 if (bytes == null || bytes.length <= 0) return null;
290 return Writables.getHRegionInfoOrNull(bytes);
291 }
292
293
294
295
296
297
298
299
300 private static boolean isDaughterMissing(final CatalogTracker catalogTracker,
301 final HRegionInfo daughter) throws IOException {
302 FindDaughterVisitor visitor = new FindDaughterVisitor(daughter);
303
304
305
306
307
308
309 byte [] startrow = daughter.getRegionName();
310 MetaReader.fullScan(catalogTracker, visitor, startrow);
311 return !visitor.foundDaughter();
312 }
313
314
315
316
317
318 static class FindDaughterVisitor implements MetaReader.Visitor {
319 private final HRegionInfo daughter;
320 private boolean found = false;
321
322 FindDaughterVisitor(final HRegionInfo daughter) {
323 this.daughter = daughter;
324 }
325
326
327
328
329 boolean foundDaughter() {
330 return this.found;
331 }
332
333 @Override
334 public boolean visit(Result r) throws IOException {
335 HRegionInfo hri = getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
336 if (hri == null) {
337 LOG.warn("No serialized HRegionInfo in " + r);
338 return true;
339 }
340 byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
341 HConstants.SERVER_QUALIFIER);
342
343 if (value == null) return false;
344
345
346 if (!Bytes.equals(daughter.getTableDesc().getName(),
347 hri.getTableDesc().getName())) {
348
349 return false;
350 }
351
352 if (!Bytes.equals(daughter.getStartKey(), hri.getStartKey())) {
353 return false;
354 }
355
356
357
358 this.found = true;
359 return false;
360 }
361 }
362 }