View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.replication;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.Set;
26  import java.util.TreeMap;
27  import java.util.concurrent.ConcurrentHashMap;
28  import java.util.concurrent.ConcurrentMap;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.hbase.HBaseConfiguration;
33  import org.apache.hadoop.hbase.classification.InterfaceAudience;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.hbase.Abortable;
36  import org.apache.hadoop.hbase.CompoundConfiguration;
37  import org.apache.hadoop.hbase.exceptions.DeserializationException;
38  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
39  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.BytesBytesPair;
40  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
41  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
42  import org.apache.hadoop.hbase.replication.ReplicationPeer.PeerState;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.Pair;
45  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
46  import org.apache.hadoop.hbase.zookeeper.ZKUtil.ZKUtilOp;
47  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
48  import org.apache.zookeeper.KeeperException;
49  
50  import com.google.protobuf.ByteString;
51  
52  /**
53   * This class provides an implementation of the ReplicationPeers interface using Zookeeper. The
54   * peers znode contains a list of all peer replication clusters and the current replication state of
55   * those clusters. It has one child peer znode for each peer cluster. The peer znode is named with
56   * the cluster id provided by the user in the HBase shell. The value of the peer znode contains the
57   * peers cluster key provided by the user in the HBase Shell. The cluster key contains a list of
58   * zookeeper quorum peers, the client port for the zookeeper quorum, and the base znode for HBase.
59   * For example:
60   *
61   *  /hbase/replication/peers/1 [Value: zk1.host.com,zk2.host.com,zk3.host.com:2181:/hbase]
62   *  /hbase/replication/peers/2 [Value: zk5.host.com,zk6.host.com,zk7.host.com:2181:/hbase]
63   *
64   * Each of these peer znodes has a child znode that indicates whether or not replication is enabled
65   * on that peer cluster. These peer-state znodes do not have child znodes and simply contain a
66   * boolean value (i.e. ENABLED or DISABLED). This value is read/maintained by the
67   * ReplicationPeer.PeerStateTracker class. For example:
68   *
69   * /hbase/replication/peers/1/peer-state [Value: ENABLED]
70   *
71   * Each of these peer znodes has a child znode that indicates which data will be replicated
72   * to the peer cluster. These peer-tableCFs znodes do not have child znodes and only have a
73   * table/cf list config. This value is read/maintained by the ReplicationPeer.TableCFsTracker
74   * class. For example:
75   *
76   * /hbase/replication/peers/1/tableCFs [Value: "table1; table2:cf1,cf3; table3:cfx,cfy"]
77   */
78  @InterfaceAudience.Private
79  public class ReplicationPeersZKImpl extends ReplicationStateZKBase implements ReplicationPeers {
80  
81    // Map of peer clusters keyed by their id
82    private Map<String, ReplicationPeerZKImpl> peerClusters;
83    private final String tableCFsNodeName;
84  
85    private static final Log LOG = LogFactory.getLog(ReplicationPeersZKImpl.class);
86  
87    public ReplicationPeersZKImpl(final ZooKeeperWatcher zk, final Configuration conf,
88        Abortable abortable) {
89      super(zk, conf, abortable);
90      this.tableCFsNodeName = conf.get("zookeeper.znode.replication.peers.tableCFs", "tableCFs");
91      this.peerClusters = new ConcurrentHashMap<String, ReplicationPeerZKImpl>();
92    }
93  
94    @Override
95    public void init() throws ReplicationException {
96      try {
97        if (ZKUtil.checkExists(this.zookeeper, this.peersZNode) < 0) {
98          ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
99        }
100     } catch (KeeperException e) {
101       throw new ReplicationException("Could not initialize replication peers", e);
102     }
103     addExistingPeers();
104   }
105 
106   @Override
107   public void addPeer(String id, ReplicationPeerConfig peerConfig, String tableCFs)
108       throws ReplicationException {
109     try {
110       if (peerExists(id)) {
111         throw new IllegalArgumentException("Cannot add a peer with id=" + id
112             + " because that id already exists.");
113       }
114       
115       if(id.contains("-")){
116         throw new IllegalArgumentException("Found invalid peer name:" + id);
117       }
118       
119       ZKUtil.createWithParents(this.zookeeper, this.peersZNode);
120       List<ZKUtilOp> listOfOps = new ArrayList<ZKUtil.ZKUtilOp>();
121       ZKUtilOp op1 =
122           ZKUtilOp.createAndFailSilent(ZKUtil.joinZNode(this.peersZNode, id),
123             toByteArray(peerConfig));
124       // There is a race (if hbase.zookeeper.useMulti is false)
125       // b/w PeerWatcher and ReplicationZookeeper#add method to create the
126       // peer-state znode. This happens while adding a peer
127       // The peer state data is set as "ENABLED" by default.
128       ZKUtilOp op2 = ZKUtilOp.createAndFailSilent(getPeerStateNode(id), ENABLED_ZNODE_BYTES);
129       String tableCFsStr = (tableCFs == null) ? "" : tableCFs;
130       ZKUtilOp op3 = ZKUtilOp.createAndFailSilent(getTableCFsNode(id), Bytes.toBytes(tableCFsStr));
131       listOfOps.add(op1);
132       listOfOps.add(op2);
133       listOfOps.add(op3);
134       ZKUtil.multiOrSequential(this.zookeeper, listOfOps, false);
135     } catch (KeeperException e) {
136       throw new ReplicationException("Could not add peer with id=" + id + ", peerConfif="
137           + peerConfig, e);
138     }
139   }
140 
141   @Override
142   public void removePeer(String id) throws ReplicationException {
143     try {
144       if (!peerExists(id)) {
145         throw new IllegalArgumentException("Cannot remove peer with id=" + id
146             + " because that id does not exist.");
147       }
148       ZKUtil.deleteNodeRecursively(this.zookeeper, ZKUtil.joinZNode(this.peersZNode, id));
149     } catch (KeeperException e) {
150       throw new ReplicationException("Could not remove peer with id=" + id, e);
151     }
152   }
153 
154   @Override
155   public void enablePeer(String id) throws ReplicationException {
156     changePeerState(id, ZooKeeperProtos.ReplicationState.State.ENABLED);
157     LOG.info("peer " + id + " is enabled");
158   }
159 
160   @Override
161   public void disablePeer(String id) throws ReplicationException {
162     changePeerState(id, ZooKeeperProtos.ReplicationState.State.DISABLED);
163     LOG.info("peer " + id + " is disabled");
164   }
165 
166   @Override
167   public String getPeerTableCFsConfig(String id) throws ReplicationException {
168     try {
169       if (!peerExists(id)) {
170         throw new IllegalArgumentException("peer " + id + " doesn't exist");
171       }
172       try {
173         return Bytes.toString(ZKUtil.getData(this.zookeeper, getTableCFsNode(id)));
174       } catch (Exception e) {
175         throw new ReplicationException(e);
176       }
177     } catch (KeeperException e) {
178       throw new ReplicationException("Unable to get tableCFs of the peer with id=" + id, e);
179     }
180   }
181 
182   @Override
183   public void setPeerTableCFsConfig(String id, String tableCFsStr) throws ReplicationException {
184     try {
185       if (!peerExists(id)) {
186         throw new IllegalArgumentException("Cannot set peer tableCFs because id=" + id
187             + " does not exist.");
188       }
189       String tableCFsZKNode = getTableCFsNode(id);
190       byte[] tableCFs = Bytes.toBytes(tableCFsStr);
191       if (ZKUtil.checkExists(this.zookeeper, tableCFsZKNode) != -1) {
192         ZKUtil.setData(this.zookeeper, tableCFsZKNode, tableCFs);
193       } else {
194         ZKUtil.createAndWatch(this.zookeeper, tableCFsZKNode, tableCFs);
195       }
196       LOG.info("Peer tableCFs with id= " + id + " is now " + tableCFsStr);
197     } catch (KeeperException e) {
198       throw new ReplicationException("Unable to change tableCFs of the peer with id=" + id, e);
199     }
200   }
201 
202   @Override
203   public Map<String, List<String>> getTableCFs(String id) throws IllegalArgumentException {
204     ReplicationPeer replicationPeer = this.peerClusters.get(id);
205     if (replicationPeer == null) {
206       throw new IllegalArgumentException("Peer with id= " + id + " is not connected");
207     }
208     return replicationPeer.getTableCFs();
209   }
210 
211   @Override
212   public boolean getStatusOfPeer(String id) {
213     ReplicationPeer replicationPeer = this.peerClusters.get(id);
214     if (replicationPeer == null) {
215       throw new IllegalArgumentException("Peer with id= " + id + " is not connected");
216     } 
217     return this.peerClusters.get(id).getPeerState() == PeerState.ENABLED;
218   }
219 
220   @Override
221   public boolean getStatusOfPeerFromBackingStore(String id) throws ReplicationException {
222     try {
223       if (!peerExists(id)) {
224         throw new IllegalArgumentException("peer " + id + " doesn't exist");
225       }
226       String peerStateZNode = getPeerStateNode(id);
227       try {
228         return ReplicationPeerZKImpl.isStateEnabled(ZKUtil.getData(this.zookeeper, peerStateZNode));
229       } catch (KeeperException e) {
230         throw new ReplicationException(e);
231       } catch (DeserializationException e) {
232         throw new ReplicationException(e);
233       }
234     } catch (KeeperException e) {
235       throw new ReplicationException("Unable to get status of the peer with id=" + id +
236           " from backing store", e);
237     }
238   }
239 
240   @Override
241   public Map<String, ReplicationPeerConfig> getAllPeerConfigs() {
242     Map<String, ReplicationPeerConfig> peers = new TreeMap<String, ReplicationPeerConfig>();
243     List<String> ids = null;
244     try {
245       ids = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
246       for (String id : ids) {
247         ReplicationPeerConfig peerConfig = getReplicationPeerConfig(id);
248         if (peerConfig == null) {
249           LOG.warn("Failed to get replication peer configuration of clusterid=" + id
250               + " znode content, continuing.");
251           continue;
252         }
253         peers.put(id, peerConfig);
254       }
255     } catch (KeeperException e) {
256       this.abortable.abort("Cannot get the list of peers ", e);
257     } catch (ReplicationException e) {
258       this.abortable.abort("Cannot get the list of peers ", e);
259     }
260     return peers;
261   }
262 
263   @Override
264   public ReplicationPeerConfig getReplicationPeerConfig(String peerId) throws ReplicationException {
265     String znode = ZKUtil.joinZNode(this.peersZNode, peerId);
266     byte[] data = null;
267     try {
268       data = ZKUtil.getData(this.zookeeper, znode);
269     } catch (KeeperException e) {
270       throw new ReplicationException("Error getting configuration for peer with id=" + peerId, e);
271     }
272     if (data == null) {
273       LOG.error("Could not get configuration for peer because it doesn't exist. peerId=" + peerId);
274       return null;
275     }
276 
277     try {
278       return parsePeerFrom(data);
279     } catch (DeserializationException e) {
280       LOG.warn("Failed to parse cluster key from peerId=" + peerId
281           + ", specifically the content from the following znode: " + znode);
282       return null;
283     }
284   }
285   
286   @Override
287   public Pair<ReplicationPeerConfig, Configuration> getPeerConf(String peerId)
288       throws ReplicationException {
289     ReplicationPeerConfig peerConfig = getReplicationPeerConfig(peerId);
290 
291     if (peerConfig == null) {
292       return null;
293     }
294 
295     Configuration otherConf;
296     try {
297       otherConf = HBaseConfiguration.createClusterConf(this.conf, peerConfig.getClusterKey());
298     } catch (IOException e) {
299       LOG.error("Can't get peer configuration for peerId=" + peerId + " because:", e);
300       return null;
301     }
302 
303     if (!peerConfig.getConfiguration().isEmpty()) {
304       CompoundConfiguration compound = new CompoundConfiguration();
305       compound.add(otherConf);
306       compound.addStringMap(peerConfig.getConfiguration());
307       return new Pair<ReplicationPeerConfig, Configuration>(peerConfig, compound);
308     }
309 
310     return new Pair<ReplicationPeerConfig, Configuration>(peerConfig, otherConf);
311   }
312   
313   @Override
314   public Set<String> getPeerIds() {
315     return peerClusters.keySet(); // this is not thread-safe
316   }
317 
318   @Override
319   public ReplicationPeer getPeer(String peerId) {
320     return peerClusters.get(peerId);
321   }
322   
323   /**
324    * List all registered peer clusters and set a watch on their znodes.
325    */
326   @Override
327   public List<String> getAllPeerIds() {
328     List<String> ids = null;
329     try {
330       ids = ZKUtil.listChildrenAndWatchThem(this.zookeeper, this.peersZNode);
331     } catch (KeeperException e) {
332       this.abortable.abort("Cannot get the list of peers ", e);
333     }
334     return ids;
335   }
336 
337   /**
338    * A private method used during initialization. This method attempts to add all registered
339    * peer clusters. This method does not set a watch on the peer cluster znodes.
340    */
341   private void addExistingPeers() throws ReplicationException {
342     List<String> znodes = null;
343     try {
344       znodes = ZKUtil.listChildrenNoWatch(this.zookeeper, this.peersZNode);
345     } catch (KeeperException e) {
346       throw new ReplicationException("Error getting the list of peer clusters.", e);
347     }
348     if (znodes != null) {
349       for (String z : znodes) {
350         createAndAddPeer(z);
351       }
352     }
353   }
354 
355   @Override
356   public boolean peerAdded(String peerId) throws ReplicationException {
357     return createAndAddPeer(peerId);
358   }
359 
360   @Override
361   public void peerRemoved(String peerId) {
362     ReplicationPeer rp = this.peerClusters.get(peerId);
363     if (rp != null) {
364       this.peerClusters.remove(peerId);
365     }
366   }
367 
368   /**
369    * Attempt to connect to a new remote slave cluster.
370    * @param peerId a short that identifies the cluster
371    * @return true if a new connection was made, false if no new connection was made.
372    */
373   public boolean createAndAddPeer(String peerId) throws ReplicationException {
374     if (peerClusters == null) {
375       return false;
376     }
377     if (this.peerClusters.containsKey(peerId)) {
378       return false;
379     }
380 
381     ReplicationPeerZKImpl peer = null;
382     try {
383       peer = createPeer(peerId);
384     } catch (Exception e) {
385       throw new ReplicationException("Error adding peer with id=" + peerId, e);
386     }
387     if (peer == null) {
388       return false;
389     }
390     ReplicationPeerZKImpl previous = ((ConcurrentMap<String, ReplicationPeerZKImpl>) peerClusters)
391         .putIfAbsent(peerId, peer);
392     if (previous == null) {
393       LOG.info("Added new peer cluster=" + peer.getPeerConfig().getClusterKey());
394     } else {
395       LOG.info("Peer already present, " + previous.getPeerConfig().getClusterKey()
396           + ", new cluster=" + peer.getPeerConfig().getClusterKey());
397     }
398     return true;
399   }
400 
401   private String getTableCFsNode(String id) {
402     return ZKUtil.joinZNode(this.peersZNode, ZKUtil.joinZNode(id, this.tableCFsNodeName));
403   }
404 
405   private String getPeerStateNode(String id) {
406     return ZKUtil.joinZNode(this.peersZNode, ZKUtil.joinZNode(id, this.peerStateNodeName));
407   }
408 
409   /**
410    * Update the state znode of a peer cluster.
411    * @param id
412    * @param state
413    */
414   private void changePeerState(String id, ZooKeeperProtos.ReplicationState.State state)
415       throws ReplicationException {
416     try {
417       if (!peerExists(id)) {
418         throw new IllegalArgumentException("Cannot enable/disable peer because id=" + id
419             + " does not exist.");
420       }
421       String peerStateZNode = getPeerStateNode(id);
422       byte[] stateBytes =
423           (state == ZooKeeperProtos.ReplicationState.State.ENABLED) ? ENABLED_ZNODE_BYTES
424               : DISABLED_ZNODE_BYTES;
425       if (ZKUtil.checkExists(this.zookeeper, peerStateZNode) != -1) {
426         ZKUtil.setData(this.zookeeper, peerStateZNode, stateBytes);
427       } else {
428         ZKUtil.createAndWatch(this.zookeeper, peerStateZNode, stateBytes);
429       }
430       LOG.info("Peer with id= " + id + " is now " + state.name());
431     } catch (KeeperException e) {
432       throw new ReplicationException("Unable to change state of the peer with id=" + id, e);
433     }
434   }
435 
436   /**
437    * Helper method to connect to a peer
438    * @param peerId peer's identifier
439    * @return object representing the peer
440    * @throws ReplicationException
441    */
442   private ReplicationPeerZKImpl createPeer(String peerId) throws ReplicationException {
443     Pair<ReplicationPeerConfig, Configuration> pair = getPeerConf(peerId);
444     if (pair == null) {
445       return null;
446     }
447     Configuration peerConf = pair.getSecond();
448 
449     ReplicationPeerZKImpl peer = new ReplicationPeerZKImpl(peerConf, peerId, pair.getFirst());
450     try {
451       peer.startStateTracker(this.zookeeper, this.getPeerStateNode(peerId));
452     } catch (KeeperException e) {
453       throw new ReplicationException("Error starting the peer state tracker for peerId=" +
454           peerId, e);
455     }
456 
457     try {
458       peer.startTableCFsTracker(this.zookeeper, this.getTableCFsNode(peerId));
459     } catch (KeeperException e) {
460       throw new ReplicationException("Error starting the peer tableCFs tracker for peerId=" +
461           peerId, e);
462     }
463 
464     return peer;
465   }
466 
467   /**
468    * @param bytes Content of a peer znode.
469    * @return ClusterKey parsed from the passed bytes.
470    * @throws DeserializationException
471    */
472   private static ReplicationPeerConfig parsePeerFrom(final byte[] bytes)
473       throws DeserializationException {
474     if (ProtobufUtil.isPBMagicPrefix(bytes)) {
475       int pblen = ProtobufUtil.lengthOfPBMagic();
476       ZooKeeperProtos.ReplicationPeer.Builder builder =
477           ZooKeeperProtos.ReplicationPeer.newBuilder();
478       ZooKeeperProtos.ReplicationPeer peer;
479       try {
480         ProtobufUtil.mergeFrom(builder, bytes, pblen, bytes.length - pblen);
481         peer = builder.build();
482       } catch (IOException e) {
483         throw new DeserializationException(e);
484       }
485       return convert(peer);
486     } else {
487       if (bytes.length > 0) {
488         return new ReplicationPeerConfig().setClusterKey(Bytes.toString(bytes));
489       }
490       return new ReplicationPeerConfig().setClusterKey("");
491     }
492   }
493 
494   private static ReplicationPeerConfig convert(ZooKeeperProtos.ReplicationPeer peer) {
495     ReplicationPeerConfig peerConfig = new ReplicationPeerConfig();
496     if (peer.hasClusterkey()) {
497       peerConfig.setClusterKey(peer.getClusterkey());
498     }
499     if (peer.hasReplicationEndpointImpl()) {
500       peerConfig.setReplicationEndpointImpl(peer.getReplicationEndpointImpl());
501     }
502 
503     for (BytesBytesPair pair : peer.getDataList()) {
504       peerConfig.getPeerData().put(pair.getFirst().toByteArray(), pair.getSecond().toByteArray());
505     }
506 
507     for (NameStringPair pair : peer.getConfigurationList()) {
508       peerConfig.getConfiguration().put(pair.getName(), pair.getValue());
509     }
510     return peerConfig;
511   }
512 
513   private static ZooKeeperProtos.ReplicationPeer convert(ReplicationPeerConfig  peerConfig) {
514     ZooKeeperProtos.ReplicationPeer.Builder builder = ZooKeeperProtos.ReplicationPeer.newBuilder();
515     if (peerConfig.getClusterKey() != null) {
516       builder.setClusterkey(peerConfig.getClusterKey());
517     }
518     if (peerConfig.getReplicationEndpointImpl() != null) {
519       builder.setReplicationEndpointImpl(peerConfig.getReplicationEndpointImpl());
520     }
521 
522     for (Map.Entry<byte[], byte[]> entry : peerConfig.getPeerData().entrySet()) {
523       builder.addData(BytesBytesPair.newBuilder()
524         .setFirst(ByteString.copyFrom(entry.getKey()))
525         .setSecond(ByteString.copyFrom(entry.getValue()))
526           .build());
527     }
528 
529     for (Map.Entry<String, String> entry : peerConfig.getConfiguration().entrySet()) {
530       builder.addConfiguration(NameStringPair.newBuilder()
531         .setName(entry.getKey())
532         .setValue(entry.getValue())
533         .build());
534     }
535 
536     return builder.build();
537   }
538 
539   /**
540    * @param peerConfig
541    * @return Serialized protobuf of <code>peerConfig</code> with pb magic prefix prepended suitable
542    *         for use as content of a this.peersZNode; i.e. the content of PEER_ID znode under
543    *         /hbase/replication/peers/PEER_ID
544    */
545   private static byte[] toByteArray(final ReplicationPeerConfig peerConfig) {
546     byte[] bytes = convert(peerConfig).toByteArray();
547     return ProtobufUtil.prependPBMagic(bytes);
548   }
549 
550 
551 }