1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client.replication;
20
21 import java.io.Closeable;
22 import java.io.IOException;
23 import java.util.ArrayList;
24 import java.util.Collection;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Map.Entry;
30 import java.util.Set;
31
32 import org.apache.commons.lang.StringUtils;
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.hbase.classification.InterfaceAudience;
36 import org.apache.hadoop.hbase.classification.InterfaceStability;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.hbase.Abortable;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.classification.InterfaceAudience;
44 import org.apache.hadoop.hbase.classification.InterfaceStability;
45 import org.apache.hadoop.hbase.client.Admin;
46 import org.apache.hadoop.hbase.client.HBaseAdmin;
47 import org.apache.hadoop.hbase.client.Connection;
48 import org.apache.hadoop.hbase.client.ConnectionFactory;
49 import org.apache.hadoop.hbase.replication.ReplicationException;
50 import org.apache.hadoop.hbase.replication.ReplicationFactory;
51 import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
52 import org.apache.hadoop.hbase.replication.ReplicationPeers;
53 import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
54 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
55
56 import com.google.common.annotations.VisibleForTesting;
57 import com.google.common.collect.Lists;
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82 @InterfaceAudience.Public
83 @InterfaceStability.Evolving
84 public class ReplicationAdmin implements Closeable {
85 private static final Log LOG = LogFactory.getLog(ReplicationAdmin.class);
86
87 public static final String TNAME = "tableName";
88 public static final String CFNAME = "columnFamlyName";
89
90
91
92 public static final String REPLICATIONTYPE = "replicationType";
93 public static final String REPLICATIONGLOBAL = Integer
94 .toString(HConstants.REPLICATION_SCOPE_GLOBAL);
95
96 private final Connection connection;
97
98
99 private final ReplicationQueuesClient replicationQueuesClient;
100 private final ReplicationPeers replicationPeers;
101
102
103
104
105 private final ZooKeeperWatcher zkw;
106
107
108
109
110
111
112
113 public ReplicationAdmin(Configuration conf) throws IOException {
114 if (!conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY,
115 HConstants.REPLICATION_ENABLE_DEFAULT)) {
116 throw new RuntimeException("hbase.replication isn't true, please " +
117 "enable it in order to use replication");
118 }
119 this.connection = ConnectionFactory.createConnection(conf);
120 try {
121 zkw = createZooKeeperWatcher();
122 try {
123 this.replicationPeers = ReplicationFactory.getReplicationPeers(zkw, conf, this.connection);
124 this.replicationPeers.init();
125 this.replicationQueuesClient =
126 ReplicationFactory.getReplicationQueuesClient(zkw, conf, this.connection);
127 this.replicationQueuesClient.init();
128 } catch (Exception exception) {
129 if (zkw != null) {
130 zkw.close();
131 }
132 throw exception;
133 }
134 } catch (Exception exception) {
135 if (connection != null) {
136 connection.close();
137 }
138 if (exception instanceof IOException) {
139 throw (IOException) exception;
140 } else if (exception instanceof RuntimeException) {
141 throw (RuntimeException) exception;
142 } else {
143 throw new IOException("Error initializing the replication admin client.", exception);
144 }
145 }
146 }
147
148 private ZooKeeperWatcher createZooKeeperWatcher() throws IOException {
149
150 return new ZooKeeperWatcher(connection.getConfiguration(), "ReplicationAdmin", new Abortable() {
151 @Override
152 public void abort(String why, Throwable e) {
153 LOG.error(why, e);
154
155
156 }
157
158 @Override
159 public boolean isAborted() {
160 return false;
161 }
162 });
163 }
164
165
166
167
168
169
170
171
172
173
174 @Deprecated
175 public void addPeer(String id, String clusterKey) throws ReplicationException {
176 this.addPeer(id, new ReplicationPeerConfig().setClusterKey(clusterKey), null);
177 }
178
179 @Deprecated
180 public void addPeer(String id, String clusterKey, String tableCFs)
181 throws ReplicationException {
182 this.replicationPeers.addPeer(id,
183 new ReplicationPeerConfig().setClusterKey(clusterKey), tableCFs);
184 }
185
186
187
188
189
190
191
192
193
194
195 public void addPeer(String id, ReplicationPeerConfig peerConfig,
196 Map<TableName, ? extends Collection<String>> tableCfs) throws ReplicationException {
197 this.replicationPeers.addPeer(id, peerConfig, getTableCfsStr(tableCfs));
198 }
199
200 public static Map<TableName, List<String>> parseTableCFsFromConfig(String tableCFsConfig) {
201 if (tableCFsConfig == null || tableCFsConfig.trim().length() == 0) {
202 return null;
203 }
204
205 Map<TableName, List<String>> tableCFsMap = null;
206
207
208
209 String[] tables = tableCFsConfig.split(";");
210 for (String tab : tables) {
211
212 tab = tab.trim();
213 if (tab.length() == 0) {
214 continue;
215 }
216
217
218 String[] pair = tab.split(":");
219 String tabName = pair[0].trim();
220 if (pair.length > 2 || tabName.length() == 0) {
221 LOG.error("ignore invalid tableCFs setting: " + tab);
222 continue;
223 }
224
225
226 List<String> cfs = null;
227 if (pair.length == 2) {
228 String[] cfsList = pair[1].split(",");
229 for (String cf : cfsList) {
230 String cfName = cf.trim();
231 if (cfName.length() > 0) {
232 if (cfs == null) {
233 cfs = new ArrayList<String>();
234 }
235 cfs.add(cfName);
236 }
237 }
238 }
239
240
241 if (tableCFsMap == null) {
242 tableCFsMap = new HashMap<TableName, List<String>>();
243 }
244 tableCFsMap.put(TableName.valueOf(tabName), cfs);
245 }
246 return tableCFsMap;
247 }
248
249 @VisibleForTesting
250 static String getTableCfsStr(Map<TableName, ? extends Collection<String>> tableCfs) {
251 String tableCfsStr = null;
252 if (tableCfs != null) {
253
254 StringBuilder builder = new StringBuilder();
255 for (Entry<TableName, ? extends Collection<String>> entry : tableCfs.entrySet()) {
256 if (builder.length() > 0) {
257 builder.append(";");
258 }
259 builder.append(entry.getKey());
260 if (entry.getValue() != null && !entry.getValue().isEmpty()) {
261 builder.append(":");
262 builder.append(StringUtils.join(entry.getValue(), ","));
263 }
264 }
265 tableCfsStr = builder.toString();
266 }
267 return tableCfsStr;
268 }
269
270
271
272
273
274 public void removePeer(String id) throws ReplicationException {
275 this.replicationPeers.removePeer(id);
276 }
277
278
279
280
281
282 public void enablePeer(String id) throws ReplicationException {
283 this.replicationPeers.enablePeer(id);
284 }
285
286
287
288
289
290 public void disablePeer(String id) throws ReplicationException {
291 this.replicationPeers.disablePeer(id);
292 }
293
294
295
296
297
298 public int getPeersCount() {
299 return this.replicationPeers.getAllPeerIds().size();
300 }
301
302
303
304
305
306
307 @Deprecated
308 public Map<String, String> listPeers() {
309 Map<String, ReplicationPeerConfig> peers = this.listPeerConfigs();
310 Map<String, String> ret = new HashMap<String, String>(peers.size());
311
312 for (Map.Entry<String, ReplicationPeerConfig> entry : peers.entrySet()) {
313 ret.put(entry.getKey(), entry.getValue().getClusterKey());
314 }
315 return ret;
316 }
317
318 public Map<String, ReplicationPeerConfig> listPeerConfigs() {
319 return this.replicationPeers.getAllPeerConfigs();
320 }
321
322 public ReplicationPeerConfig getPeerConfig(String id) throws ReplicationException {
323 return this.replicationPeers.getReplicationPeerConfig(id);
324 }
325
326
327
328
329
330 public String getPeerTableCFs(String id) throws ReplicationException {
331 return this.replicationPeers.getPeerTableCFsConfig(id);
332 }
333
334
335
336
337
338
339 @Deprecated
340 public void setPeerTableCFs(String id, String tableCFs) throws ReplicationException {
341 this.replicationPeers.setPeerTableCFsConfig(id, tableCFs);
342 }
343
344
345
346
347
348
349
350 public void appendPeerTableCFs(String id, String tableCfs) throws ReplicationException {
351 appendPeerTableCFs(id, parseTableCFsFromConfig(tableCfs));
352 }
353
354
355
356
357
358
359
360 public void appendPeerTableCFs(String id, Map<TableName, ? extends Collection<String>> tableCfs)
361 throws ReplicationException {
362 if (tableCfs == null) {
363 throw new ReplicationException("tableCfs is null");
364 }
365 Map<TableName, List<String>> preTableCfs = parseTableCFsFromConfig(getPeerTableCFs(id));
366 if (preTableCfs == null) {
367 setPeerTableCFs(id, tableCfs);
368 return;
369 }
370
371 for (Map.Entry<TableName, ? extends Collection<String>> entry : tableCfs.entrySet()) {
372 TableName table = entry.getKey();
373 Collection<String> appendCfs = entry.getValue();
374 if (preTableCfs.containsKey(table)) {
375 List<String> cfs = preTableCfs.get(table);
376 if (cfs == null || appendCfs == null) {
377 preTableCfs.put(table, null);
378 } else {
379 Set<String> cfSet = new HashSet<String>(cfs);
380 cfSet.addAll(appendCfs);
381 preTableCfs.put(table, Lists.newArrayList(cfSet));
382 }
383 } else {
384 if (appendCfs == null || appendCfs.isEmpty()) {
385 preTableCfs.put(table, null);
386 } else {
387 preTableCfs.put(table, Lists.newArrayList(appendCfs));
388 }
389 }
390 }
391 setPeerTableCFs(id, preTableCfs);
392 }
393
394
395
396
397
398
399
400 public void removePeerTableCFs(String id, String tableCf) throws ReplicationException {
401 removePeerTableCFs(id, parseTableCFsFromConfig(tableCf));
402 }
403
404
405
406
407
408
409
410 public void removePeerTableCFs(String id, Map<TableName, ? extends Collection<String>> tableCfs)
411 throws ReplicationException {
412 if (tableCfs == null) {
413 throw new ReplicationException("tableCfs is null");
414 }
415
416 Map<TableName, List<String>> preTableCfs = parseTableCFsFromConfig(getPeerTableCFs(id));
417 if (preTableCfs == null) {
418 throw new ReplicationException("Table-Cfs for peer" + id + " is null");
419 }
420 for (Map.Entry<TableName, ? extends Collection<String>> entry: tableCfs.entrySet()) {
421 TableName table = entry.getKey();
422 Collection<String> removeCfs = entry.getValue();
423 if (preTableCfs.containsKey(table)) {
424 List<String> cfs = preTableCfs.get(table);
425 if (cfs == null && removeCfs == null) {
426 preTableCfs.remove(table);
427 } else if (cfs != null && removeCfs != null) {
428 Set<String> cfSet = new HashSet<String>(cfs);
429 cfSet.removeAll(removeCfs);
430 if (cfSet.isEmpty()) {
431 preTableCfs.remove(table);
432 } else {
433 preTableCfs.put(table, Lists.newArrayList(cfSet));
434 }
435 } else if (cfs == null && removeCfs != null) {
436 throw new ReplicationException("Cannot remove cf of table: " + table
437 + " which doesn't specify cfs from table-cfs config in peer: " + id);
438 } else if (cfs != null && removeCfs == null) {
439 throw new ReplicationException("Cannot remove table: " + table
440 + " which has specified cfs from table-cfs config in peer: " + id);
441 }
442 } else {
443 throw new ReplicationException("No table: " + table + " in table-cfs config of peer: " + id);
444 }
445 }
446 setPeerTableCFs(id, preTableCfs);
447 }
448
449
450
451
452
453
454
455
456
457 public void setPeerTableCFs(String id, Map<TableName, ? extends Collection<String>> tableCfs)
458 throws ReplicationException {
459 this.replicationPeers.setPeerTableCFsConfig(id, getTableCfsStr(tableCfs));
460 }
461
462
463
464
465
466
467
468 public boolean getPeerState(String id) throws ReplicationException {
469 return this.replicationPeers.getStatusOfPeerFromBackingStore(id);
470 }
471
472 @Override
473 public void close() throws IOException {
474 if (this.zkw != null) {
475 this.zkw.close();
476 }
477 if (this.connection != null) {
478 this.connection.close();
479 }
480 }
481
482
483
484
485
486
487
488
489
490
491
492
493
494 public List<HashMap<String, String>> listReplicated() throws IOException {
495 List<HashMap<String, String>> replicationColFams = new ArrayList<HashMap<String, String>>();
496
497 Admin admin = connection.getAdmin();
498 HTableDescriptor[] tables;
499 try {
500 tables = admin.listTables();
501 } finally {
502 if (admin!= null) admin.close();
503 }
504
505 for (HTableDescriptor table : tables) {
506 HColumnDescriptor[] columns = table.getColumnFamilies();
507 String tableName = table.getNameAsString();
508 for (HColumnDescriptor column : columns) {
509 if (column.getScope() != HConstants.REPLICATION_SCOPE_LOCAL) {
510
511 HashMap<String, String> replicationEntry = new HashMap<String, String>();
512 replicationEntry.put(TNAME, tableName);
513 replicationEntry.put(CFNAME, column.getNameAsString());
514 replicationEntry.put(REPLICATIONTYPE, REPLICATIONGLOBAL);
515 replicationColFams.add(replicationEntry);
516 }
517 }
518 }
519
520 return replicationColFams;
521 }
522 }