1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client.replication;
20
21 import java.io.Closeable;
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Map.Entry;
30 import java.util.Set;
31
32 import org.apache.commons.lang.StringUtils;
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.hbase.classification.InterfaceAudience;
36 import org.apache.hadoop.hbase.classification.InterfaceStability;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.hbase.Abortable;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.classification.InterfaceAudience;
44 import org.apache.hadoop.hbase.classification.InterfaceStability;
45 import org.apache.hadoop.hbase.client.Admin;
46 import org.apache.hadoop.hbase.client.HBaseAdmin;
47 import org.apache.hadoop.hbase.client.Connection;
48 import org.apache.hadoop.hbase.client.ConnectionFactory;
49 import org.apache.hadoop.hbase.replication.ReplicationException;
50 import org.apache.hadoop.hbase.replication.ReplicationFactory;
51 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
52 import org.apache.hadoop.hbase.replication.ReplicationPeers;
53 import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
54 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
55
56 import com.google.common.annotations.VisibleForTesting;
57 import com.google.common.collect.Lists;
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82 @InterfaceAudience.Public
83 @InterfaceStability.Evolving
84 public class ReplicationAdmin implements Closeable {
85 private static final Log LOG = LogFactory.getLog(ReplicationAdmin.class);
86
87 public static final String TNAME = "tableName";
88 public static final String CFNAME = "columnFamlyName";
89
90
91
92 public static final String REPLICATIONTYPE = "replicationType";
93 public static final String REPLICATIONGLOBAL = Integer
94 .toString(HConstants.REPLICATION_SCOPE_GLOBAL);
95
96 private final Connection connection;
97
98
99 private final ReplicationQueuesClient replicationQueuesClient;
100 private final ReplicationPeers replicationPeers;
101
102
103
104
105 private final ZooKeeperWatcher zkw;
106
107
108
109
110
111
112
113 public ReplicationAdmin(Configuration conf) throws IOException {
114 if (!conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY,
115 HConstants.REPLICATION_ENABLE_DEFAULT)) {
116 throw new RuntimeException("hbase.replication isn't true, please " +
117 "enable it in order to use replication");
118 }
119 this.connection = ConnectionFactory.createConnection(conf);
120 zkw = createZooKeeperWatcher();
121 try {
122 this.replicationPeers = ReplicationFactory.getReplicationPeers(zkw, conf, this.connection);
123 this.replicationPeers.init();
124 this.replicationQueuesClient =
125 ReplicationFactory.getReplicationQueuesClient(zkw, conf, this.connection);
126 this.replicationQueuesClient.init();
127
128 } catch (ReplicationException e) {
129 throw new IOException("Error initializing the replication admin client.", e);
130 }
131 }
132
133 private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
134
135 return new ZooKeeperWatcher(connection.getConfiguration(), "ReplicationAdmin", new Abortable() {
136 @Override
137 public void abort(String why, Throwable e) {
138 LOG.error(why, e);
139
140
141 }
142
143 @Override
144 public boolean isAborted() {
145 return false;
146 }
147 });
148 }
149
150
151
152
153
154
155
156
157
158
159 @Deprecated
160 public void addPeer(String id, String clusterKey) throws ReplicationException {
161 this.addPeer(id, new ReplicationPeerConfig().setClusterKey(clusterKey), null);
162 }
163
164 @Deprecated
165 public void addPeer(String id, String clusterKey, String tableCFs)
166 throws ReplicationException {
167 this.replicationPeers.addPeer(id,
168 new ReplicationPeerConfig().setClusterKey(clusterKey), tableCFs);
169 }
170
171
172
173
174
175
176
177
178
179
180 public void addPeer(String id, ReplicationPeerConfig peerConfig,
181 Map<TableName, ? extends Collection<String>> tableCfs) throws ReplicationException {
182 this.replicationPeers.addPeer(id, peerConfig, getTableCfsStr(tableCfs));
183 }
184
185 public static Map<TableName, List<String>> parseTableCFsFromConfig(String tableCFsConfig) {
186 if (tableCFsConfig == null || tableCFsConfig.trim().length() == 0) {
187 return null;
188 }
189
190 Map<TableName, List<String>> tableCFsMap = null;
191
192
193
194 String[] tables = tableCFsConfig.split(";");
195 for (String tab : tables) {
196
197 tab = tab.trim();
198 if (tab.length() == 0) {
199 continue;
200 }
201
202
203 String[] pair = tab.split(":");
204 String tabName = pair[0].trim();
205 if (pair.length > 2 || tabName.length() == 0) {
206 LOG.error("ignore invalid tableCFs setting: " + tab);
207 continue;
208 }
209
210
211 List<String> cfs = null;
212 if (pair.length == 2) {
213 String[] cfsList = pair[1].split(",");
214 for (String cf : cfsList) {
215 String cfName = cf.trim();
216 if (cfName.length() > 0) {
217 if (cfs == null) {
218 cfs = new ArrayList<String>();
219 }
220 cfs.add(cfName);
221 }
222 }
223 }
224
225
226 if (tableCFsMap == null) {
227 tableCFsMap = new HashMap<TableName, List<String>>();
228 }
229 tableCFsMap.put(TableName.valueOf(tabName), cfs);
230 }
231 return tableCFsMap;
232 }
233
234 @VisibleForTesting
235 static String getTableCfsStr(Map<TableName, ? extends Collection<String>> tableCfs) {
236 String tableCfsStr = null;
237 if (tableCfs != null) {
238
239 StringBuilder builder = new StringBuilder();
240 for (Entry<TableName, ? extends Collection<String>> entry : tableCfs.entrySet()) {
241 if (builder.length() > 0) {
242 builder.append(";");
243 }
244 builder.append(entry.getKey());
245 if (entry.getValue() != null && !entry.getValue().isEmpty()) {
246 builder.append(":");
247 builder.append(StringUtils.join(entry.getValue(), ","));
248 }
249 }
250 tableCfsStr = builder.toString();
251 }
252 return tableCfsStr;
253 }
254
255
256
257
258
259 public void removePeer(String id) throws ReplicationException {
260 this.replicationPeers.removePeer(id);
261 }
262
263
264
265
266
267 public void enablePeer(String id) throws ReplicationException {
268 this.replicationPeers.enablePeer(id);
269 }
270
271
272
273
274
275 public void disablePeer(String id) throws ReplicationException {
276 this.replicationPeers.disablePeer(id);
277 }
278
279
280
281
282
283 public int getPeersCount() {
284 return this.replicationPeers.getAllPeerIds().size();
285 }
286
287
288
289
290
291
292 @Deprecated
293 public Map<String, String> listPeers() {
294 Map<String, ReplicationPeerConfig> peers = this.listPeerConfigs();
295 Map<String, String> ret = new HashMap<String, String>(peers.size());
296
297 for (Map.Entry<String, ReplicationPeerConfig> entry : peers.entrySet()) {
298 ret.put(entry.getKey(), entry.getValue().getClusterKey());
299 }
300 return ret;
301 }
302
303 public Map<String, ReplicationPeerConfig> listPeerConfigs() {
304 return this.replicationPeers.getAllPeerConfigs();
305 }
306
307 public ReplicationPeerConfig getPeerConfig(String id) throws ReplicationException {
308 return this.replicationPeers.getReplicationPeerConfig(id);
309 }
310
311
312
313
314
315 public String getPeerTableCFs(String id) throws ReplicationException {
316 return this.replicationPeers.getPeerTableCFsConfig(id);
317 }
318
319
320
321
322
323
324 @Deprecated
325 public void setPeerTableCFs(String id, String tableCFs) throws ReplicationException {
326 this.replicationPeers.setPeerTableCFsConfig(id, tableCFs);
327 }
328
329
330
331
332
333
334
335 public void appendPeerTableCFs(String id, String tableCfs) throws ReplicationException {
336 appendPeerTableCFs(id, parseTableCFsFromConfig(tableCfs));
337 }
338
339
340
341
342
343
344
345 public void appendPeerTableCFs(String id, Map<TableName, ? extends Collection<String>> tableCfs)
346 throws ReplicationException {
347 if (tableCfs == null) {
348 throw new ReplicationException("tableCfs is null");
349 }
350 Map<TableName, List<String>> preTableCfs = parseTableCFsFromConfig(getPeerTableCFs(id));
351 if (preTableCfs == null) {
352 setPeerTableCFs(id, tableCfs);
353 return;
354 }
355
356 for (Map.Entry<TableName, ? extends Collection<String>> entry : tableCfs.entrySet()) {
357 TableName table = entry.getKey();
358 Collection<String> appendCfs = entry.getValue();
359 if (preTableCfs.containsKey(table)) {
360 List<String> cfs = preTableCfs.get(table);
361 if (cfs == null || appendCfs == null) {
362 preTableCfs.put(table, null);
363 } else {
364 Set<String> cfSet = new HashSet<String>(cfs);
365 cfSet.addAll(appendCfs);
366 preTableCfs.put(table, Lists.newArrayList(cfSet));
367 }
368 } else {
369 if (appendCfs == null || appendCfs.isEmpty()) {
370 preTableCfs.put(table, null);
371 } else {
372 preTableCfs.put(table, Lists.newArrayList(appendCfs));
373 }
374 }
375 }
376 setPeerTableCFs(id, preTableCfs);
377 }
378
379
380
381
382
383
384
385 public void removePeerTableCFs(String id, String tableCf) throws ReplicationException {
386 removePeerTableCFs(id, parseTableCFsFromConfig(tableCf));
387 }
388
389
390
391
392
393
394
395 public void removePeerTableCFs(String id, Map<TableName, ? extends Collection<String>> tableCfs)
396 throws ReplicationException {
397 if (tableCfs == null) {
398 throw new ReplicationException("tableCfs is null");
399 }
400
401 Map<TableName, List<String>> preTableCfs = parseTableCFsFromConfig(getPeerTableCFs(id));
402 if (preTableCfs == null) {
403 throw new ReplicationException("Table-Cfs for peer" + id + " is null");
404 }
405 for (Map.Entry<TableName, ? extends Collection<String>> entry: tableCfs.entrySet()) {
406 TableName table = entry.getKey();
407 Collection<String> removeCfs = entry.getValue();
408 if (preTableCfs.containsKey(table)) {
409 List<String> cfs = preTableCfs.get(table);
410 if (cfs == null && removeCfs == null) {
411 preTableCfs.remove(table);
412 } else if (cfs != null && removeCfs != null) {
413 Set<String> cfSet = new HashSet<String>(cfs);
414 cfSet.removeAll(removeCfs);
415 if (cfSet.isEmpty()) {
416 preTableCfs.remove(table);
417 } else {
418 preTableCfs.put(table, Lists.newArrayList(cfSet));
419 }
420 } else if (cfs == null && removeCfs != null) {
421 throw new ReplicationException("Cannot remove cf of table: " + table
422 + " which doesn't specify cfs from table-cfs config in peer: " + id);
423 } else if (cfs != null && removeCfs == null) {
424 throw new ReplicationException("Cannot remove table: " + table
425 + " which has specified cfs from table-cfs config in peer: " + id);
426 }
427 } else {
428 throw new ReplicationException("No table: " + table + " in table-cfs config of peer: " + id);
429 }
430 }
431 setPeerTableCFs(id, preTableCfs);
432 }
433
434
435
436
437
438
439
440
441
442 public void setPeerTableCFs(String id, Map<TableName, ? extends Collection<String>> tableCfs)
443 throws ReplicationException {
444 this.replicationPeers.setPeerTableCFsConfig(id, getTableCfsStr(tableCfs));
445 }
446
447
448
449
450
451
452
453 public boolean getPeerState(String id) throws ReplicationException {
454 return this.replicationPeers.getStatusOfPeerFromBackingStore(id);
455 }
456
457 @Override
458 public void close() throws IOException {
459 if (this.zkw != null) {
460 this.zkw.close();
461 }
462 if (this.connection != null) {
463 this.connection.close();
464 }
465 }
466
467
468
469
470
471
472
473
474
475
476
477
478
479 public List<HashMap<String, String>> listReplicated() throws IOException {
480 List<HashMap<String, String>> replicationColFams = new ArrayList<HashMap<String, String>>();
481
482 Admin admin = connection.getAdmin();
483 HTableDescriptor[] tables;
484 try {
485 tables = admin.listTables();
486 } finally {
487 if (admin!= null) admin.close();
488 }
489
490 for (HTableDescriptor table : tables) {
491 HColumnDescriptor[] columns = table.getColumnFamilies();
492 String tableName = table.getNameAsString();
493 for (HColumnDescriptor column : columns) {
494 if (column.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) {
495
496 HashMap<String, String> replicationEntry = new HashMap<String, String>();
497 replicationEntry.put(TNAME, tableName);
498 replicationEntry.put(CFNAME, column.getNameAsString());
499 replicationEntry.put(REPLICATIONTYPE, REPLICATIONGLOBAL);
500 replicationColFams.add(replicationEntry);
501 }
502 }
503 }
504
505 return replicationColFams;
506 }
507 }