View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.replication.master;
20  
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.classification.InterfaceAudience;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FileStatus;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.Abortable;
28  import org.apache.hadoop.hbase.HConstants;
29  import org.apache.hadoop.hbase.client.HConnectionManager;
30  import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
31  import org.apache.hadoop.hbase.replication.ReplicationFactory;
32  import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
33  import org.apache.hadoop.hbase.replication.ReplicationQueuesClientZKImpl;
34  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
35  import org.apache.zookeeper.KeeperException;
36  
37  import java.io.IOException;
38  import java.util.HashSet;
39  import java.util.List;
40  import java.util.Set;
41  
42  /**
43   * Implementation of a log cleaner that checks if a log is still scheduled for
44   * replication before deleting it when its TTL is over.
45   */
46  @InterfaceAudience.Private
47  public class ReplicationLogCleaner extends BaseLogCleanerDelegate implements Abortable {
48    private static final Log LOG = LogFactory.getLog(ReplicationLogCleaner.class);
49    private ZooKeeperWatcher zkw;
50    private ReplicationQueuesClient replicationQueues;
51    private final Set<String> hlogs = new HashSet<String>();
52    private boolean stopped = false;
53    private boolean aborted;
54  
55  
56    @Override
57    public boolean isLogDeletable(FileStatus fStat) {
58  
59      // all members of this class are null if replication is disabled, and we
60      // return true since false would render the LogsCleaner useless
61      if (this.getConf() == null) {
62        return true;
63      }
64      String log = fStat.getPath().getName();
65      // If we saw the hlog previously, let's consider it's still used
66      // At some point in the future we will refresh the list and it will be gone
67      if (this.hlogs.contains(log)) {
68        return false;
69      }
70  
71      // Let's see it's still there
72      // This solution makes every miss very expensive to process since we
73      // almost completely refresh the cache each time
74      return !refreshHLogsAndSearch(log);
75    }
76  
77    /**
78     * Search through all the hlogs we have in ZK to refresh the cache
79     * If a log is specified and found, then we early out and return true
80     * @param searchedLog log we are searching for, pass null to cache everything
81     *                    that's in zookeeper.
82     * @return false until a specified log is found.
83     */
84    private boolean refreshHLogsAndSearch(String searchedLog) {
85      this.hlogs.clear();
86      final boolean lookForLog = searchedLog != null;
87      List<String> rss = replicationQueues.getListOfReplicators();
88      if (rss == null) {
89        LOG.debug("Didn't find any region server that replicates, deleting: " +
90            searchedLog);
91        return false;
92      }
93      for (String rs: rss) {
94        List<String> listOfPeers = replicationQueues.getAllQueues(rs);
95        // if rs just died, this will be null
96        if (listOfPeers == null) {
97          continue;
98        }
99        for (String id : listOfPeers) {
100         List<String> peersHlogs = replicationQueues.getLogsInQueue(rs, id);
101         if (peersHlogs != null) {
102           this.hlogs.addAll(peersHlogs);
103         }
104         // early exit if we found the log
105         if(lookForLog && this.hlogs.contains(searchedLog)) {
106           LOG.debug("Found log in ZK, keeping: " + searchedLog);
107           return true;
108         }
109       }
110     }
111     LOG.debug("Didn't find this log in ZK, deleting: " + searchedLog);
112     return false;
113   }
114 
115   @Override
116   public void setConf(Configuration config) {
117     // If replication is disabled, keep all members null
118     if (!config.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false)) {
119       return;
120     }
121     // Make my own Configuration.  Then I'll have my own connection to zk that
122     // I can close myself when comes time.
123     Configuration conf = new Configuration(config);
124     super.setConf(conf);
125     try {
126       this.zkw = new ZooKeeperWatcher(conf, "replicationLogCleaner", null);
127       this.replicationQueues = ReplicationFactory.getReplicationQueuesClient(zkw, conf, this);
128       this.replicationQueues.init();
129     } catch (KeeperException e) {
130       LOG.error("Error while configuring " + this.getClass().getName(), e);
131     } catch (IOException e) {
132       LOG.error("Error while configuring " + this.getClass().getName(), e);
133     }
134     refreshHLogsAndSearch(null);
135   }
136 
137 
138   @Override
139   public void stop(String why) {
140     if (this.stopped) return;
141     this.stopped = true;
142     if (this.zkw != null) {
143       LOG.info("Stopping " + this.zkw);
144       this.zkw.close();
145     }
146     // Not sure why we're deleting a connection that we never acquired or used
147     HConnectionManager.deleteConnection(this.getConf());
148   }
149 
150   @Override
151   public boolean isStopped() {
152     return this.stopped;
153   }
154 
155   @Override
156   public void abort(String why, Throwable e) {
157     LOG.warn("Aborting ReplicationLogCleaner because " + why, e);
158     this.aborted = true;
159     stop(why);
160   }
161 
162   @Override
163   public boolean isAborted() {
164     return this.aborted;
165   }
166 }