1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.procedure;
19
20 import java.io.IOException;
21 import java.util.Arrays;
22 import java.util.List;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.classification.InterfaceAudience;
27 import org.apache.hadoop.classification.InterfaceStability;
28 import org.apache.hadoop.hbase.errorhandling.ForeignException;
29 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
30 import org.apache.hadoop.hbase.util.Bytes;
31 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
32 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
33 import org.apache.zookeeper.KeeperException;
34
35 import com.google.protobuf.InvalidProtocolBufferException;
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55 @InterfaceAudience.Public
56 @InterfaceStability.Evolving
57 public class ZKProcedureMemberRpcs implements ProcedureMemberRpcs {
58
59 private static final Log LOG = LogFactory.getLog(ZKProcedureMemberRpcs.class);
60 private final String memberName;
61
62 protected ProcedureMember member;
63 private ZKProcedureUtil zkController;
64
65
66
67
68
69
70
71
72
73 public ZKProcedureMemberRpcs(ZooKeeperWatcher watcher,
74 String procType, String memberName) throws KeeperException {
75 this.zkController = new ZKProcedureUtil(watcher, procType, memberName) {
76 @Override
77 public void nodeCreated(String path) {
78 if (!isInProcedurePath(path)) {
79 return;
80 }
81
82 LOG.info("Received created event:" + path);
83
84 if (isAcquiredNode(path)) {
85 waitForNewProcedures();
86 return;
87 } else if (isAbortNode(path)) {
88 watchForAbortedProcedures();
89 return;
90 }
91 String parent = ZKUtil.getParent(path);
92
93 if (isReachedNode(parent)) {
94 receivedReachedGlobalBarrier(path);
95 return;
96 } else if (isAbortNode(parent)) {
97 abort(path);
98 return;
99 } else if (isAcquiredNode(parent)) {
100 startNewSubprocedure(path);
101 } else {
102 LOG.debug("Ignoring created notification for node:" + path);
103 }
104 }
105
106 @Override
107 public void nodeChildrenChanged(String path) {
108 if (path.equals(this.acquiredZnode)) {
109 LOG.info("Received procedure start children changed event: " + path);
110 waitForNewProcedures();
111 } else if (path.equals(this.abortZnode)) {
112 LOG.info("Received procedure abort children changed event: " + path);
113 watchForAbortedProcedures();
114 }
115 }
116 };
117 this.memberName = memberName;
118 }
119
120 public ZKProcedureUtil getZkController() {
121 return zkController;
122 }
123
124 @Override
125 public String getMemberName() {
126 return memberName;
127 }
128
129
130
131
132
133 private void receivedReachedGlobalBarrier(String path) {
134 LOG.debug("Recieved reached global barrier:" + path);
135 String procName = ZKUtil.getNodeName(path);
136 this.member.receivedReachedGlobalBarrier(procName);
137 }
138
139 private void watchForAbortedProcedures() {
140 LOG.debug("Checking for aborted procedures on node: '" + zkController.getAbortZnode() + "'");
141 try {
142
143 for (String node : ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
144 zkController.getAbortZnode())) {
145 String abortNode = ZKUtil.joinZNode(zkController.getAbortZnode(), node);
146 abort(abortNode);
147 }
148 } catch (KeeperException e) {
149 member.controllerConnectionFailure("Failed to list children for abort node:"
150 + zkController.getAbortZnode(), new IOException(e));
151 }
152 }
153
154 private void waitForNewProcedures() {
155
156 LOG.debug("Looking for new procedures under znode:'" + zkController.getAcquiredBarrier() + "'");
157 List<String> runningProcedures = null;
158 try {
159 runningProcedures = ZKUtil.listChildrenAndWatchForNewChildren(zkController.getWatcher(),
160 zkController.getAcquiredBarrier());
161 if (runningProcedures == null) {
162 LOG.debug("No running procedures.");
163 return;
164 }
165 } catch (KeeperException e) {
166 member.controllerConnectionFailure("General failure when watching for new procedures",
167 new IOException(e));
168 }
169 if (runningProcedures == null) {
170 LOG.debug("No running procedures.");
171 return;
172 }
173 for (String procName : runningProcedures) {
174
175 String path = ZKUtil.joinZNode(zkController.getAcquiredBarrier(), procName);
176 startNewSubprocedure(path);
177 }
178 }
179
180
181
182
183
184
185
186
187 private synchronized void startNewSubprocedure(String path) {
188 LOG.debug("Found procedure znode: " + path);
189 String opName = ZKUtil.getNodeName(path);
190
191 String abortZNode = zkController.getAbortZNode(opName);
192 try {
193 if (ZKUtil.watchAndCheckExists(zkController.getWatcher(), abortZNode)) {
194 LOG.debug("Not starting:" + opName + " because we already have an abort notification.");
195 return;
196 }
197 } catch (KeeperException e) {
198 member.controllerConnectionFailure("Failed to get the abort znode (" + abortZNode
199 + ") for procedure :" + opName, new IOException(e));
200 return;
201 }
202
203
204 Subprocedure subproc = null;
205 try {
206 byte[] data = ZKUtil.getData(zkController.getWatcher(), path);
207 LOG.debug("start proc data length is " + data.length);
208 if (!ProtobufUtil.isPBMagicPrefix(data)) {
209 String msg = "Data in for starting procuedure " + opName +
210 " is illegally formatted (no pb magic). " +
211 "Killing the procedure: " + Bytes.toString(data);
212 LOG.error(msg);
213 throw new IllegalArgumentException(msg);
214 }
215 data = Arrays.copyOfRange(data, ProtobufUtil.lengthOfPBMagic(), data.length);
216 LOG.debug("Found data for znode:" + path);
217 subproc = member.createSubprocedure(opName, data);
218 member.submitSubprocedure(subproc);
219 } catch (IllegalArgumentException iae ) {
220 LOG.error("Illegal argument exception", iae);
221 sendMemberAborted(subproc, new ForeignException(getMemberName(), iae));
222 } catch (IllegalStateException ise) {
223 LOG.error("Illegal state exception ", ise);
224 sendMemberAborted(subproc, new ForeignException(getMemberName(), ise));
225 } catch (KeeperException e) {
226 member.controllerConnectionFailure("Failed to get data for new procedure:" + opName,
227 new IOException(e));
228 }
229 }
230
231
232
233
234
235
236
237 @Override
238 public void sendMemberAcquired(Subprocedure sub) throws IOException {
239 String procName = sub.getName();
240 try {
241 LOG.debug("Member: '" + memberName + "' joining acquired barrier for procedure (" + procName
242 + ") in zk");
243 String acquiredZNode = ZKUtil.joinZNode(ZKProcedureUtil.getAcquireBarrierNode(
244 zkController, procName), memberName);
245 ZKUtil.createAndFailSilent(zkController.getWatcher(), acquiredZNode);
246
247
248 String reachedBarrier = zkController.getReachedBarrierNode(procName);
249 LOG.debug("Watch for global barrier reached:" + reachedBarrier);
250 if (ZKUtil.watchAndCheckExists(zkController.getWatcher(), reachedBarrier)) {
251 receivedReachedGlobalBarrier(reachedBarrier);
252 }
253 } catch (KeeperException e) {
254 member.controllerConnectionFailure("Failed to acquire barrier for procedure: "
255 + procName + " and member: " + memberName, new IOException(e));
256 }
257 }
258
259
260
261
262 @Override
263 public void sendMemberCompleted(Subprocedure sub) throws IOException {
264 String procName = sub.getName();
265 LOG.debug("Marking procedure '" + procName + "' completed for member '" + memberName
266 + "' in zk");
267 String joinPath = ZKUtil.joinZNode(zkController.getReachedBarrierNode(procName), memberName);
268 try {
269 ZKUtil.createAndFailSilent(zkController.getWatcher(), joinPath);
270 } catch (KeeperException e) {
271 member.controllerConnectionFailure("Failed to post zk node:" + joinPath
272 + " to join procedure barrier.", new IOException(e));
273 }
274 }
275
276
277
278
279
280 @Override
281 public void sendMemberAborted(Subprocedure sub, ForeignException ee) {
282 if (sub == null) {
283 LOG.error("Failed due to null subprocedure", ee);
284 return;
285 }
286 String procName = sub.getName();
287 LOG.debug("Aborting procedure (" + procName + ") in zk");
288 String procAbortZNode = zkController.getAbortZNode(procName);
289 try {
290 String source = (ee.getSource() == null) ? memberName: ee.getSource();
291 byte[] errorInfo = ProtobufUtil.prependPBMagic(ForeignException.serialize(source, ee));
292 ZKUtil.createAndFailSilent(zkController.getWatcher(), procAbortZNode, errorInfo);
293 LOG.debug("Finished creating abort znode:" + procAbortZNode);
294 } catch (KeeperException e) {
295
296
297 zkController.logZKTree(zkController.getBaseZnode());
298 member.controllerConnectionFailure("Failed to post zk node:" + procAbortZNode
299 + " to abort procedure", new IOException(e));
300 }
301 }
302
303
304
305
306
307 protected void abort(String abortZNode) {
308 LOG.debug("Aborting procedure member for znode " + abortZNode);
309 String opName = ZKUtil.getNodeName(abortZNode);
310 try {
311 byte[] data = ZKUtil.getData(zkController.getWatcher(), abortZNode);
312
313
314 ForeignException ee;
315 try {
316 if (!ProtobufUtil.isPBMagicPrefix(data)) {
317 String msg = "Illegally formatted data in abort node for proc " + opName
318 + ". Killing the procedure.";
319 LOG.error(msg);
320
321 ee = new ForeignException(getMemberName(), new IllegalArgumentException(msg));
322 } else {
323 data = Arrays.copyOfRange(data, ProtobufUtil.lengthOfPBMagic(), data.length);
324 ee = ForeignException.deserialize(data);
325 }
326 } catch (InvalidProtocolBufferException e) {
327 LOG.warn("Got an error notification for op:" + opName
328 + " but we can't read the information. Killing the procedure.");
329
330 ee = new ForeignException(getMemberName(), e);
331 }
332
333 this.member.receiveAbortProcedure(opName, ee);
334 } catch (KeeperException e) {
335 member.controllerConnectionFailure("Failed to get data for abort znode:" + abortZNode
336 + zkController.getAbortZnode(), new IOException(e));
337 }
338 }
339
340 public void start(ProcedureMember listener) {
341 LOG.debug("Starting procedure member '" + this.memberName + "'");
342 this.member = listener;
343 watchForAbortedProcedures();
344 waitForNewProcedures();
345 }
346
347 @Override
348 public void close() throws IOException {
349 zkController.close();
350 }
351
352 }