1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.replication.master;
20
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.classification.InterfaceAudience;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.FileStatus;
26 import org.apache.hadoop.fs.Path;
27 import org.apache.hadoop.hbase.Abortable;
28 import org.apache.hadoop.hbase.HConstants;
29 import org.apache.hadoop.hbase.client.HConnectionManager;
30 import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
31 import org.apache.hadoop.hbase.replication.ReplicationException;
32 import org.apache.hadoop.hbase.replication.ReplicationFactory;
33 import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
34 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
35 import org.apache.zookeeper.KeeperException;
36
37 import java.io.IOException;
38 import java.util.List;
39 import java.util.Set;
40
41 import com.google.common.base.Predicate;
42 import com.google.common.collect.ImmutableSet;
43 import com.google.common.collect.Iterables;
44 import com.google.common.collect.Sets;
45
46
47
48
49
50 @InterfaceAudience.Private
51 public class ReplicationLogCleaner extends BaseLogCleanerDelegate implements Abortable {
52 private static final Log LOG = LogFactory.getLog(ReplicationLogCleaner.class);
53 private ZooKeeperWatcher zkw;
54 private ReplicationQueuesClient replicationQueues;
55 private boolean stopped = false;
56 private boolean aborted;
57
58
59 @Override
60 public Iterable<FileStatus> getDeletableFiles(Iterable<FileStatus> files) {
61
62
63 if (this.getConf() == null) {
64 return files;
65 }
66
67 final Set<String> hlogs = loadHLogsFromQueues();
68 return Iterables.filter(files, new Predicate<FileStatus>() {
69 @Override
70 public boolean apply(FileStatus file) {
71 String hlog = file.getPath().getName();
72 boolean logInReplicationQueue = hlogs.contains(hlog);
73 if (LOG.isDebugEnabled()) {
74 if (logInReplicationQueue) {
75 LOG.debug("Found log in ZK, keeping: " + hlog);
76 } else {
77 LOG.debug("Didn't find this log in ZK, deleting: " + hlog);
78 }
79 }
80 return !logInReplicationQueue;
81 }});
82 }
83
84
85
86
87 private Set<String> loadHLogsFromQueues() {
88 List<String> rss = replicationQueues.getListOfReplicators();
89 if (rss == null) {
90 LOG.debug("Didn't find any region server that replicates, won't prevent any deletions.");
91 return ImmutableSet.of();
92 }
93 Set<String> hlogs = Sets.newHashSet();
94 for (String rs: rss) {
95 List<String> listOfPeers = replicationQueues.getAllQueues(rs);
96
97 if (listOfPeers == null) {
98 continue;
99 }
100 for (String id : listOfPeers) {
101 List<String> peersHlogs = replicationQueues.getLogsInQueue(rs, id);
102 if (peersHlogs != null) {
103 hlogs.addAll(peersHlogs);
104 }
105 }
106 }
107 return hlogs;
108 }
109
110 @Override
111 public void setConf(Configuration config) {
112
113 if (!config.getBoolean(HConstants.REPLICATION_ENABLE_KEY,
114 HConstants.REPLICATION_ENABLE_DEFAULT)) {
115 LOG.warn("Not configured - allowing all hlogs to be deleted");
116 return;
117 }
118
119
120 Configuration conf = new Configuration(config);
121 super.setConf(conf);
122 try {
123 this.zkw = new ZooKeeperWatcher(conf, "replicationLogCleaner", null);
124 this.replicationQueues = ReplicationFactory.getReplicationQueuesClient(zkw, conf, this);
125 this.replicationQueues.init();
126 } catch (ReplicationException e) {
127 LOG.error("Error while configuring " + this.getClass().getName(), e);
128 } catch (IOException e) {
129 LOG.error("Error while configuring " + this.getClass().getName(), e);
130 }
131 }
132
133 @Override
134 public void stop(String why) {
135 if (this.stopped) return;
136 this.stopped = true;
137 if (this.zkw != null) {
138 LOG.info("Stopping " + this.zkw);
139 this.zkw.close();
140 }
141
142 HConnectionManager.deleteConnection(this.getConf());
143 }
144
145 @Override
146 public boolean isStopped() {
147 return this.stopped;
148 }
149
150 @Override
151 public void abort(String why, Throwable e) {
152 LOG.warn("Aborting ReplicationLogCleaner because " + why, e);
153 this.aborted = true;
154 stop(why);
155 }
156
157 @Override
158 public boolean isAborted() {
159 return this.aborted;
160 }
161 }