View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FileStatus;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.hbase.Chore;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.RemoteExceptionHandler;
31  import org.apache.hadoop.hbase.regionserver.wal.HLog;
32  
33  import java.io.IOException;
34  import java.util.concurrent.atomic.AtomicBoolean;
35  import java.util.regex.Pattern;
36  
37  /**
38   * This Chore, everytime it runs, will clear the logs in the old logs folder
39   * that are older than hbase.master.logcleaner.ttl and, in order to limit the
40   * number of deletes it sends, will only delete maximum 20 in a single run.
41   */
42  public class OldLogsCleaner extends Chore {
43  
44    static final Log LOG = LogFactory.getLog(OldLogsCleaner.class.getName());
45  
46    // Max number we can delete on every chore, this is to make sure we don't
47    // issue thousands of delete commands around the same time
48    private final int maxDeletedLogs;
49    private final FileSystem fs;
50    private final Path oldLogDir;
51    private final LogCleanerDelegate logCleaner;
52    private final Configuration conf;
53  
54    /**
55     *
56     * @param p the period of time to sleep between each run
57     * @param s the stopper boolean
58     * @param conf configuration to use
59     * @param fs handle to the FS
60     * @param oldLogDir the path to the archived logs
61     */
62    public OldLogsCleaner(final int p, final AtomicBoolean s,
63                          Configuration conf, FileSystem fs,
64                          Path oldLogDir) {
65      super("OldLogsCleaner", p, s);
66      // Use the log cleaner provided by replication if enabled, unless something
67      // was already provided
68      if (conf.getBoolean(HConstants.REPLICATION_ENABLE_KEY, false) &&
69          conf.get("hbase.master.logcleanerplugin.impl") == null) {
70        conf.set("hbase.master.logcleanerplugin.impl",
71            "org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner");
72      }
73      this.maxDeletedLogs =
74          conf.getInt("hbase.master.logcleaner.maxdeletedlogs", 20);
75      this.fs = fs;
76      this.oldLogDir = oldLogDir;
77      this.conf = conf;
78      this.logCleaner = getLogCleaner();
79    }
80  
81    private LogCleanerDelegate getLogCleaner() {
82      try {
83        Class c = Class.forName(conf.get("hbase.master.logcleanerplugin.impl",
84          TimeToLiveLogCleaner.class.getCanonicalName()));
85        LogCleanerDelegate cleaner = (LogCleanerDelegate) c.newInstance();
86        cleaner.setConf(conf);
87        return cleaner;
88      } catch (Exception e) {
89        LOG.warn("Passed log cleaner implementation throws errors, " +
90            "defaulting to TimeToLiveLogCleaner", e);
91        return new TimeToLiveLogCleaner();
92      }
93    }
94  
95    @Override
96    protected void chore() {
97      try {
98        FileStatus[] files = this.fs.listStatus(this.oldLogDir);
99        int nbDeletedLog = 0;
100       for (FileStatus file : files) {
101         Path filePath = file.getPath();
102         if (HLog.validateHLogFilename(filePath.getName())) {
103           if (logCleaner.isLogDeletable(filePath) ) {
104             this.fs.delete(filePath, true);
105             nbDeletedLog++;
106           }
107         } else {
108           LOG.warn("Found a wrongly formated file: "
109               + file.getPath().getName());
110           this.fs.delete(filePath, true);
111           nbDeletedLog++;
112         }
113         if (nbDeletedLog >= maxDeletedLogs) {
114           break;
115         }
116       }
117     } catch (IOException e) {
118       e = RemoteExceptionHandler.checkIOException(e);
119       LOG.warn("Error while cleaning the logs", e);
120     }
121   }
122 }