1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.util;
19
20 import java.io.IOException;
21 import java.util.List;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.conf.Configured;
27 import org.apache.hadoop.hbase.Abortable;
28 import org.apache.hadoop.hbase.HBaseConfiguration;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
31 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
32 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
33 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.ReplicationPeer;
34 import org.apache.hadoop.hbase.replication.ReplicationStateZKBase;
35 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
36 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
37 import org.apache.hadoop.util.Tool;
38 import org.apache.hadoop.util.ToolRunner;
39 import org.apache.zookeeper.KeeperException;
40 import org.apache.zookeeper.KeeperException.NoNodeException;
41
42
43
44
45 public class ZKDataMigrator extends Configured implements Tool {
46
47 private static final Log LOG = LogFactory.getLog(ZKDataMigrator.class);
48
49 @Override
50 public int run(String[] as) throws Exception {
51 Configuration conf = getConf();
52 ZooKeeperWatcher zkw = null;
53 try {
54 zkw = new ZooKeeperWatcher(getConf(), "Migrate ZK data to PB.",
55 new ZKDataMigratorAbortable());
56 if (ZKUtil.checkExists(zkw, zkw.baseZNode) == -1) {
57 LOG.info("No hbase related data available in zookeeper. returning..");
58 return 0;
59 }
60 List<String> children = ZKUtil.listChildrenNoWatch(zkw, zkw.baseZNode);
61 if (children == null) {
62 LOG.info("No child nodes to mirgrate. returning..");
63 return 0;
64 }
65 String childPath = null;
66 for (String child : children) {
67 childPath = ZKUtil.joinZNode(zkw.baseZNode, child);
68 if (child.equals(conf.get("zookeeper.znode.rootserver", "root-region-server"))) {
69
70
71 ZKUtil.deleteNodeRecursively(zkw, childPath);
72
73 } else if (child.equals(conf.get("zookeeper.znode.rs", "rs"))) {
74
75
76 ZKUtil.deleteNodeRecursively(zkw, childPath);
77 } else if (child.equals(conf.get("zookeeper.znode.draining.rs", "draining"))) {
78
79
80
81
82 ZKUtil.deleteNodeRecursively(zkw, childPath);
83 } else if (child.equals(conf.get("zookeeper.znode.master", "master"))) {
84
85
86 ZKUtil.deleteNodeRecursively(zkw, childPath);
87 } else if (child.equals(conf.get("zookeeper.znode.backup.masters", "backup-masters"))) {
88
89
90 ZKUtil.deleteNodeRecursively(zkw, childPath);
91 } else if (child.equals(conf.get("zookeeper.znode.state", "shutdown"))) {
92
93
94 ZKUtil.deleteNodeRecursively(zkw, childPath);
95 } else if (child.equals(conf.get("zookeeper.znode.unassigned", "unassigned"))) {
96
97
98
99
100 ZKUtil.deleteNodeRecursively(zkw, childPath);
101 } else if (child.equals(conf.get("zookeeper.znode.tableEnableDisable", "table"))
102 || child.equals(conf.get("zookeeper.znode.masterTableEnableDisable", "table"))) {
103 checkAndMigrateTableStatesToPB(zkw);
104 } else if (child.equals(conf.get("zookeeper.znode.masterTableEnableDisable92",
105 "table92"))) {
106
107
108 ZKUtil.deleteNodeRecursively(zkw, childPath);
109 } else if (child.equals(conf.get("zookeeper.znode.splitlog", "splitlog"))) {
110
111
112 ZKUtil.deleteNodeRecursively(zkw, childPath);
113 } else if (child.equals(conf.get("zookeeper.znode.replication", "replication"))) {
114 checkAndMigrateReplicationNodesToPB(zkw);
115 } else if (child.equals(conf.get("zookeeper.znode.clusterId", "hbaseid"))) {
116
117 ZKUtil.deleteNodeRecursively(zkw, childPath);
118 } else if (child.equals(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION)) {
119
120 ZKUtil.deleteNodeRecursively(zkw, childPath);
121 }
122 }
123 } catch (Exception e) {
124 LOG.error("Got exception while updating znodes ", e);
125 throw new IOException(e);
126 } finally {
127 if (zkw != null) {
128 zkw.close();
129 }
130 }
131 return 0;
132 }
133
134 private void checkAndMigrateTableStatesToPB(ZooKeeperWatcher zkw) throws KeeperException {
135 List<String> tables = ZKUtil.listChildrenNoWatch(zkw, zkw.tableZNode);
136 if (tables == null) {
137 LOG.info("No table present to migrate table state to PB. returning..");
138 return;
139 }
140 for (String table : tables) {
141 String znode = ZKUtil.joinZNode(zkw.tableZNode, table);
142
143
144 if (table.equals("-ROOT-") || table.equals(".META.")) {
145 ZKUtil.deleteNode(zkw, znode);
146 continue;
147 }
148 byte[] data = ZKUtil.getData(zkw, znode);
149 if (ProtobufUtil.isPBMagicPrefix(data)) continue;
150 ZooKeeperProtos.Table.Builder builder = ZooKeeperProtos.Table.newBuilder();
151 builder.setState(ZooKeeperProtos.Table.State.valueOf(Bytes.toString(data)));
152 data = ProtobufUtil.prependPBMagic(builder.build().toByteArray());
153 ZKUtil.setData(zkw, znode, data);
154 }
155 }
156
157 private void checkAndMigrateReplicationNodesToPB(ZooKeeperWatcher zkw) throws KeeperException {
158 String replicationZnodeName = getConf().get("zookeeper.znode.replication", "replication");
159 String replicationPath = ZKUtil.joinZNode(zkw.baseZNode, replicationZnodeName);
160 List<String> replicationZnodes = ZKUtil.listChildrenNoWatch(zkw, replicationPath);
161 if (replicationZnodes == null) {
162 LOG.info("No replication related znodes present to migrate. returning..");
163 return;
164 }
165 for (String child : replicationZnodes) {
166 String znode = ZKUtil.joinZNode(replicationPath, child);
167 if (child.equals(getConf().get("zookeeper.znode.replication.peers", "peers"))) {
168 List<String> peers = ZKUtil.listChildrenNoWatch(zkw, znode);
169 if (peers == null || peers.isEmpty()) {
170 LOG.info("No peers present to migrate. returning..");
171 continue;
172 }
173 checkAndMigratePeerZnodesToPB(zkw, znode, peers);
174 } else if (child.equals(getConf().get("zookeeper.znode.replication.state", "state"))) {
175
176 ZKUtil.deleteNodeRecursively(zkw, znode);
177 } else if (child.equals(getConf().get("zookeeper.znode.replication.rs", "rs"))) {
178 List<String> rsList = ZKUtil.listChildrenNoWatch(zkw, znode);
179 if (rsList == null || rsList.isEmpty()) continue;
180 for (String rs : rsList) {
181 checkAndMigrateQueuesToPB(zkw, znode, rs);
182 }
183 }
184 }
185 }
186
187 private void checkAndMigrateQueuesToPB(ZooKeeperWatcher zkw, String znode, String rs)
188 throws KeeperException, NoNodeException {
189 String rsPath = ZKUtil.joinZNode(znode, rs);
190 List<String> peers = ZKUtil.listChildrenNoWatch(zkw, rsPath);
191 if (peers == null || peers.isEmpty()) return;
192 String peerPath = null;
193 for (String peer : peers) {
194 peerPath = ZKUtil.joinZNode(rsPath, peer);
195 List<String> files = ZKUtil.listChildrenNoWatch(zkw, peerPath);
196 if (files == null || files.isEmpty()) continue;
197 String filePath = null;
198 for (String file : files) {
199 filePath = ZKUtil.joinZNode(peerPath, file);
200 byte[] data = ZKUtil.getData(zkw, filePath);
201 if (data == null || Bytes.equals(data, HConstants.EMPTY_BYTE_ARRAY)) continue;
202 if (ProtobufUtil.isPBMagicPrefix(data)) continue;
203 ZKUtil.setData(zkw, filePath,
204 ZKUtil.positionToByteArray(Long.parseLong(Bytes.toString(data))));
205 }
206 }
207 }
208
209 private void checkAndMigratePeerZnodesToPB(ZooKeeperWatcher zkw, String znode,
210 List<String> peers) throws KeeperException, NoNodeException {
211 for (String peer : peers) {
212 String peerZnode = ZKUtil.joinZNode(znode, peer);
213 byte[] data = ZKUtil.getData(zkw, peerZnode);
214 if (!ProtobufUtil.isPBMagicPrefix(data)) {
215 migrateClusterKeyToPB(zkw, peerZnode, data);
216 }
217 String peerStatePath = ZKUtil.joinZNode(peerZnode,
218 getConf().get("zookeeper.znode.replication.peers.state", "peer-state"));
219 if (ZKUtil.checkExists(zkw, peerStatePath) != -1) {
220 data = ZKUtil.getData(zkw, peerStatePath);
221 if (ProtobufUtil.isPBMagicPrefix(data)) continue;
222 migratePeerStateToPB(zkw, data, peerStatePath);
223 }
224 }
225 }
226
227 private void migrateClusterKeyToPB(ZooKeeperWatcher zkw, String peerZnode, byte[] data)
228 throws KeeperException, NoNodeException {
229 ReplicationPeer peer = ZooKeeperProtos.ReplicationPeer.newBuilder()
230 .setClusterkey(Bytes.toString(data)).build();
231 ZKUtil.setData(zkw, peerZnode, ProtobufUtil.prependPBMagic(peer.toByteArray()));
232 }
233
234 private void migratePeerStateToPB(ZooKeeperWatcher zkw, byte[] data,
235 String peerStatePath)
236 throws KeeperException, NoNodeException {
237 String state = Bytes.toString(data);
238 if (ZooKeeperProtos.ReplicationState.State.ENABLED.name().equals(state)) {
239 ZKUtil.setData(zkw, peerStatePath, ReplicationStateZKBase.ENABLED_ZNODE_BYTES);
240 } else if (ZooKeeperProtos.ReplicationState.State.DISABLED.name().equals(state)) {
241 ZKUtil.setData(zkw, peerStatePath, ReplicationStateZKBase.DISABLED_ZNODE_BYTES);
242 }
243 }
244
245 public static void main(String args[]) throws Exception {
246 System.exit(ToolRunner.run(HBaseConfiguration.create(), new ZKDataMigrator(), args));
247 }
248
249 static class ZKDataMigratorAbortable implements Abortable {
250 private boolean aborted = false;
251
252 @Override
253 public void abort(String why, Throwable e) {
254 LOG.error("Got aborted with reason: " + why + ", and error: " + e);
255 this.aborted = true;
256 }
257
258 @Override
259 public boolean isAborted() {
260 return this.aborted;
261 }
262 }
263 }