View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import java.io.IOException;
23  import java.util.LinkedList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.fs.FileStatus;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.Chore;
33  import org.apache.hadoop.hbase.RemoteExceptionHandler;
34  import org.apache.hadoop.hbase.Stoppable;
35  import org.apache.hadoop.hbase.regionserver.wal.HLog;
36  
37  import static org.apache.hadoop.hbase.HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS;
38  
39  /**
40   * This Chore, everytime it runs, will clear the HLogs in the old logs folder
41   * that are deletable for each log cleaner in the chain.
42   */
43  public class LogCleaner extends Chore {
44    static final Log LOG = LogFactory.getLog(LogCleaner.class.getName());
45  
46    private final FileSystem fs;
47    private final Path oldLogDir;
48    private List<LogCleanerDelegate> logCleanersChain;
49    private final Configuration conf;
50  
51    /**
52     *
53     * @param p the period of time to sleep between each run
54     * @param s the stopper
55     * @param conf configuration to use
56     * @param fs handle to the FS
57     * @param oldLogDir the path to the archived logs
58     */
59    public LogCleaner(final int p, final Stoppable s,
60                          Configuration conf, FileSystem fs,
61                          Path oldLogDir) {
62      super("LogsCleaner", p, s);
63      this.fs = fs;
64      this.oldLogDir = oldLogDir;
65      this.conf = conf;
66      this.logCleanersChain = new LinkedList<LogCleanerDelegate>();
67  
68      initLogCleanersChain();
69    }
70  
71    /*
72     * Initialize the chain of log cleaners from the configuration. The default
73     * three LogCleanerDelegates in this chain are: TimeToLiveLogCleaner,
74     * ReplicationLogCleaner and SnapshotLogCleaner.
75     */
76    private void initLogCleanersChain() {
77      String[] logCleaners = conf.getStrings(HBASE_MASTER_LOGCLEANER_PLUGINS);
78      if (logCleaners != null) {
79        for (String className : logCleaners) {
80          LogCleanerDelegate logCleaner = newLogCleaner(className, conf);
81          addLogCleaner(logCleaner);
82        }
83      }
84    }
85  
86    /**
87     * A utility method to create new instances of LogCleanerDelegate based
88     * on the class name of the LogCleanerDelegate.
89     * @param className fully qualified class name of the LogCleanerDelegate
90     * @param conf
91     * @return the new instance
92     */
93    public static LogCleanerDelegate newLogCleaner(String className, Configuration conf) {
94      try {
95        Class c = Class.forName(className);
96        LogCleanerDelegate cleaner = (LogCleanerDelegate) c.newInstance();
97        cleaner.setConf(conf);
98        return cleaner;
99      } catch(Exception e) {
100       LOG.warn("Can NOT create LogCleanerDelegate: " + className, e);
101       // skipping if can't instantiate
102       return null;
103     }
104   }
105 
106   /**
107    * Add a LogCleanerDelegate to the log cleaner chain. A log file is deletable
108    * if it is deletable for each LogCleanerDelegate in the chain.
109    * @param logCleaner
110    */
111   public void addLogCleaner(LogCleanerDelegate logCleaner) {
112     if (logCleaner != null && !logCleanersChain.contains(logCleaner)) {
113       logCleanersChain.add(logCleaner);
114       LOG.debug("Add log cleaner in chain: " + logCleaner.getClass().getName());
115     }
116   }
117 
118   @Override
119   protected void chore() {
120     try {
121       FileStatus [] files = this.fs.listStatus(this.oldLogDir);
122       if (files == null) return;
123       FILE: for (FileStatus file : files) {
124         Path filePath = file.getPath();
125         if (HLog.validateHLogFilename(filePath.getName())) {
126           for (LogCleanerDelegate logCleaner : logCleanersChain) {
127             if (logCleaner.isStopped()) {
128               LOG.warn("A log cleaner is stopped, won't delete any log.");
129               return;
130             }
131 
132             if (!logCleaner.isLogDeletable(filePath) ) {
133               // this log is not deletable, continue to process next log file
134               continue FILE;
135             }
136           }
137           // delete this log file if it passes all the log cleaners
138           this.fs.delete(filePath, true);
139         } else {
140           LOG.warn("Found a wrongly formated file: "
141               + file.getPath().getName());
142           this.fs.delete(filePath, true);
143         }
144       }
145     } catch (IOException e) {
146       e = RemoteExceptionHandler.checkIOException(e);
147       LOG.warn("Error while cleaning the logs", e);
148     }
149   }
150 
151   @Override
152   public void run() {
153     try {
154       super.run();
155     } finally {
156       for (LogCleanerDelegate lc: this.logCleanersChain) {
157         try {
158           lc.stop("Exiting");
159         } catch (Throwable t) {
160           LOG.warn("Stopping", t);
161         }
162       }
163     }
164   }
165 }