View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FileStatus;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.hbase.Chore;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.RemoteExceptionHandler;
31  import org.apache.hadoop.hbase.regionserver.wal.HLog;
32  
33  import java.io.IOException;
34  import java.util.LinkedList;
35  import java.util.List;
36  import java.util.concurrent.atomic.AtomicBoolean;
37  
38  /**
39   * This Chore, everytime it runs, will clear the logs in the old logs folder
40   * that are deletable for each log cleaner in the chain, in order to limit the
41   * number of deletes it sends, will only delete maximum 20 in a single run.
42   */
43  public class LogsCleaner extends Chore {
44  
45    static final Log LOG = LogFactory.getLog(LogsCleaner.class.getName());
46  
47    // Max number we can delete on every chore, this is to make sure we don't
48    // issue thousands of delete commands around the same time
49    private final int maxDeletedLogs;
50    private final FileSystem fs;
51    private final Path oldLogDir;
52    private List<LogCleanerDelegate> logCleanersChain;
53    private final Configuration conf;
54  
55    /**
56     *
57     * @param p the period of time to sleep between each run
58     * @param s the stopper boolean
59     * @param conf configuration to use
60     * @param fs handle to the FS
61     * @param oldLogDir the path to the archived logs
62     */
63    public LogsCleaner(final int p, final AtomicBoolean s,
64                          Configuration conf, FileSystem fs,
65                          Path oldLogDir) {
66      super("LogsCleaner", p, s);
67  
68      this.maxDeletedLogs =
69          conf.getInt("hbase.master.logcleaner.maxdeletedlogs", 20);
70      this.fs = fs;
71      this.oldLogDir = oldLogDir;
72      this.conf = conf;
73      this.logCleanersChain = new LinkedList<LogCleanerDelegate>();
74  
75      initLogCleanersChain();
76    }
77  
78    /*
79     * Initialize the chain of log cleaners from the configuration. The default
80     * three LogCleanerDelegates in this chain are: TimeToLiveLogCleaner,
81     * ReplicationLogCleaner and SnapshotLogCleaner.
82     */
83    private void initLogCleanersChain() {
84      String[] logCleaners = conf.getStrings("hbase.master.logcleaner.plugins");
85      if (logCleaners != null) {
86        for (String className : logCleaners) {
87          LogCleanerDelegate logCleaner = newLogCleaner(className, conf);
88          addLogCleaner(logCleaner);
89        }
90      }
91    }
92  
93    /**
94     * A utility method to create new instances of LogCleanerDelegate based
95     * on the class name of the LogCleanerDelegate.
96     * @param className fully qualified class name of the LogCleanerDelegate
97     * @param conf
98     * @return the new instance
99     */
100   public static LogCleanerDelegate newLogCleaner(String className, Configuration conf) {
101     try {
102       Class c = Class.forName(className);
103       LogCleanerDelegate cleaner = (LogCleanerDelegate) c.newInstance();
104       cleaner.setConf(conf);
105       return cleaner;
106     } catch(Exception e) {
107       LOG.warn("Can NOT create LogCleanerDelegate: " + className, e);
108       // skipping if can't instantiate
109       return null;
110     }
111   }
112 
113   /**
114    * Add a LogCleanerDelegate to the log cleaner chain. A log file is deletable
115    * if it is deletable for each LogCleanerDelegate in the chain.
116    * @param logCleaner
117    */
118   public void addLogCleaner(LogCleanerDelegate logCleaner) {
119     if (logCleaner != null && !logCleanersChain.contains(logCleaner)) {
120       logCleanersChain.add(logCleaner);
121       LOG.debug("Add log cleaner in chain: " + logCleaner.getClass().getName());
122     }
123   }
124 
125   @Override
126   protected void chore() {
127     try {
128       FileStatus[] files = this.fs.listStatus(this.oldLogDir);
129       int nbDeletedLog = 0;
130       FILE: for (FileStatus file : files) {
131         Path filePath = file.getPath();
132         if (HLog.validateHLogFilename(filePath.getName())) {
133           for (LogCleanerDelegate logCleaner : logCleanersChain) {
134             if (!logCleaner.isLogDeletable(filePath) ) {
135               // this log is not deletable, continue to process next log file
136               continue FILE;
137             }
138           }
139           // delete this log file if it passes all the log cleaners
140           this.fs.delete(filePath, true);
141           nbDeletedLog++;
142         } else {
143           LOG.warn("Found a wrongly formated file: "
144               + file.getPath().getName());
145           this.fs.delete(filePath, true);
146           nbDeletedLog++;
147         }
148         if (nbDeletedLog >= maxDeletedLogs) {
149           break;
150         }
151       }
152     } catch (IOException e) {
153       e = RemoteExceptionHandler.checkIOException(e);
154       LOG.warn("Error while cleaning the logs", e);
155     }
156   }
157 }