View Javadoc

1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.replication.master;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FileStatus;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.Abortable;
28  import org.apache.hadoop.hbase.HConstants;
29  import org.apache.hadoop.hbase.client.HConnectionManager;
30  import org.apache.hadoop.hbase.master.cleaner.BaseLogCleanerDelegate;
31  import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
32  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
33  import org.apache.zookeeper.KeeperException;
34  
35  import java.io.IOException;
36  import java.util.HashSet;
37  import java.util.List;
38  import java.util.Set;
39  
40  /**
41   * Implementation of a log cleaner that checks if a log is still scheduled for
42   * replication before deleting it when its TTL is over.
43   */
44  public class ReplicationLogCleaner extends BaseLogCleanerDelegate implements Abortable {
45    private static final Log LOG = LogFactory.getLog(ReplicationLogCleaner.class);
46    private ReplicationZookeeper zkHelper;
47    private Set<String> hlogs = new HashSet<String>();
48    private boolean stopped = false;
49    private boolean aborted;
50  
51    /**
52     * Instantiates the cleaner, does nothing more.
53     */
54    public ReplicationLogCleaner() {}
55  
56    @Override
57    public boolean isLogDeletable(FileStatus fStat) {
58  
59      try {
60        if (!zkHelper.getReplication()) {
61          return false;
62        }
63      } catch (KeeperException e) {
64        abort("Cannot get the state of replication", e);
65        return false;
66      }
67  
68      // all members of this class are null if replication is disabled, and we
69      // return true since false would render the LogsCleaner useless
70      if (this.getConf() == null) {
71        return true;
72      }
73      String log = fStat.getPath().getName();
74      // If we saw the hlog previously, let's consider it's still used
75      // At some point in the future we will refresh the list and it will be gone
76      if (this.hlogs.contains(log)) {
77        return false;
78      }
79  
80      // Let's see it's still there
81      // This solution makes every miss very expensive to process since we
82      // almost completely refresh the cache each time
83      return !refreshHLogsAndSearch(log);
84    }
85  
86    /**
87     * Search through all the hlogs we have in ZK to refresh the cache
88     * If a log is specified and found, then we early out and return true
89     * @param searchedLog log we are searching for, pass null to cache everything
90     *                    that's in zookeeper.
91     * @return false until a specified log is found.
92     */
93    private boolean refreshHLogsAndSearch(String searchedLog) {
94      this.hlogs.clear();
95      final boolean lookForLog = searchedLog != null;
96      List<String> rss = zkHelper.getListOfReplicators();
97      if (rss == null) {
98        LOG.debug("Didn't find any region server that replicates, deleting: " +
99            searchedLog);
100       return false;
101     }
102     for (String rs: rss) {
103       List<String> listOfPeers = zkHelper.getListPeersForRS(rs);
104       // if rs just died, this will be null
105       if (listOfPeers == null) {
106         continue;
107       }
108       for (String id : listOfPeers) {
109         List<String> peersHlogs = zkHelper.getListHLogsForPeerForRS(rs, id);
110         if (peersHlogs != null) {
111           this.hlogs.addAll(peersHlogs);
112         }
113         // early exit if we found the log
114         if(lookForLog && this.hlogs.contains(searchedLog)) {
115           LOG.debug("Found log in ZK, keeping: " + searchedLog);
116           return true;
117         }
118       }
119     }
120     LOG.debug("Didn't find this log in ZK, deleting: " + searchedLog);
121     return false;
122   }
123 
124   @Override
125   public void setConf(Configuration config) {
126     // If replication is disabled, keep all members null
127     if (!config.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false)) {
128       return;
129     }
130     // Make my own Configuration.  Then I'll have my own connection to zk that
131     // I can close myself when comes time.
132     Configuration conf = new Configuration(config);
133     super.setConf(conf);
134     try {
135       ZooKeeperWatcher zkw = new ZooKeeperWatcher(conf, "replicationLogCleaner", null);
136       this.zkHelper = new ReplicationZookeeper(this, conf, zkw);
137     } catch (KeeperException e) {
138       LOG.error("Error while configuring " + this.getClass().getName(), e);
139     } catch (IOException e) {
140       LOG.error("Error while configuring " + this.getClass().getName(), e);
141     }
142     refreshHLogsAndSearch(null);
143   }
144 
145 
146   @Override
147   public void stop(String why) {
148     if (this.stopped) return;
149     this.stopped = true;
150     if (this.zkHelper != null) {
151       LOG.info("Stopping " + this.zkHelper.getZookeeperWatcher());
152       this.zkHelper.getZookeeperWatcher().close();
153     }
154     // Not sure why we're deleting a connection that we never acquired or used
155     HConnectionManager.deleteConnection(this.getConf());
156   }
157 
158   @Override
159   public boolean isStopped() {
160     return this.stopped;
161   }
162 
163   @Override
164   public void abort(String why, Throwable e) {
165     LOG.warn("Aborting ReplicationLogCleaner because " + why, e);
166     this.aborted = true;
167     stop(why);
168   }
169 
170   @Override
171   public boolean isAborted() {
172     return this.aborted;
173   }
174 }