View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.cleaner;
19  
20  import java.io.IOException;
21  import java.util.LinkedList;
22  import java.util.List;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FileStatus;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.Chore;
31  import org.apache.hadoop.hbase.HBaseFileSystem;
32  import org.apache.hadoop.hbase.RemoteExceptionHandler;
33  import org.apache.hadoop.hbase.Stoppable;
34  import org.apache.hadoop.hbase.util.FSUtils;
35  
36  /**
37   * Abstract Cleaner that uses a chain of delegates to clean a directory of files
38   * @param <T> Cleaner delegate class that is dynamically loaded from configuration
39   */
40  public abstract class CleanerChore<T extends FileCleanerDelegate> extends Chore {
41  
42    private static final Log LOG = LogFactory.getLog(CleanerChore.class.getName());
43  
44    private final FileSystem fs;
45    private final Path oldFileDir;
46    private final Configuration conf;
47    List<T> cleanersChain;
48  
49    /**
50     * @param name name of the chore being run
51     * @param sleepPeriod the period of time to sleep between each run
52     * @param s the stopper
53     * @param conf configuration to use
54     * @param fs handle to the FS
55     * @param oldFileDir the path to the archived files
56     * @param confKey configuration key for the classes to instantiate
57     */
58    public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf,
59        FileSystem fs, Path oldFileDir, String confKey) {
60      super(name, sleepPeriod, s);
61      this.fs = fs;
62      this.oldFileDir = oldFileDir;
63      this.conf = conf;
64  
65      initCleanerChain(confKey);
66    }
67  
68    /**
69     * Validate the file to see if it even belongs in the directory. If it is valid, then the file
70     * will go through the cleaner delegates, but otherwise the file is just deleted.
71     * @param file full {@link Path} of the file to be checked
72     * @return <tt>true</tt> if the file is valid, <tt>false</tt> otherwise
73     */
74    protected abstract boolean validate(Path file);
75  
76    /**
77     * Instanitate and initialize all the file cleaners set in the configuration
78     * @param confKey key to get the file cleaner classes from the configuration
79     */
80    private void initCleanerChain(String confKey) {
81      this.cleanersChain = new LinkedList<T>();
82      String[] logCleaners = conf.getStrings(confKey);
83      if (logCleaners != null) {
84        for (String className : logCleaners) {
85          T logCleaner = newFileCleaner(className, conf);
86          if (logCleaner != null) {
87            LOG.debug("initialize cleaner=" + className);
88            this.cleanersChain.add(logCleaner);
89          }
90        }
91      }
92    }
93  
94    /**
95     * A utility method to create new instances of LogCleanerDelegate based on the class name of the
96     * LogCleanerDelegate.
97     * @param className fully qualified class name of the LogCleanerDelegate
98     * @param conf
99     * @return the new instance
100    */
101   public T newFileCleaner(String className, Configuration conf) {
102     try {
103       Class<? extends FileCleanerDelegate> c = Class.forName(className).asSubclass(
104         FileCleanerDelegate.class);
105       @SuppressWarnings("unchecked")
106       T cleaner = (T) c.newInstance();
107       cleaner.setConf(conf);
108       return cleaner;
109     } catch (Exception e) {
110       LOG.warn("Can NOT create CleanerDelegate: " + className, e);
111       // skipping if can't instantiate
112       return null;
113     }
114   }
115 
116   @Override
117   protected void chore() {
118     try {
119       FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir, null);
120       // if the path (file or directory) doesn't exist, then we can just return
121       if (files == null) return;
122       // loop over the found files and see if they should be deleted
123       for (FileStatus file : files) {
124         try {
125           if (file.isDir()) checkAndDeleteDirectory(file.getPath());
126           else checkAndDelete(file);
127         } catch (IOException e) {
128           e = RemoteExceptionHandler.checkIOException(e);
129           LOG.warn("Error while cleaning the logs", e);
130         }
131       }
132     } catch (IOException e) {
133       LOG.warn("Failed to get status of:" + oldFileDir);
134     }
135 
136   }
137 
138   /**
139    * Attempt to delete a directory and all files under that directory. Each child file is passed
140    * through the delegates to see if it can be deleted. If the directory has not children when the
141    * cleaners have finished it is deleted.
142    * <p>
143    * If new children files are added between checks of the directory, the directory will <b>not</b>
144    * be deleted.
145    * @param toCheck directory to check
146    * @return <tt>true</tt> if the directory was deleted, <tt>false</tt> otherwise.
147    * @throws IOException if there is an unexpected filesystem error
148    */
149   public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
150     if (LOG.isTraceEnabled()) {
151       LOG.trace("Checking directory: " + toCheck);
152     }
153     FileStatus[] children = FSUtils.listStatus(fs, toCheck, null);
154     // if the directory doesn't exist, then we are done
155     if (children == null) {
156       try {
157         return HBaseFileSystem.deleteFileFromFileSystem(fs, toCheck);
158       } catch (IOException e) {
159         if (LOG.isTraceEnabled()) {
160           LOG.trace("Couldn't delete directory: " + toCheck, e);
161         }
162       }
163       // couldn't delete w/o exception, so we can't return success.
164       return false;
165     }     
166 
167     boolean canDeleteThis = true;
168     for (FileStatus child : children) {
169       Path path = child.getPath();
170       // attempt to delete all the files under the directory
171       if (child.isDir()) {
172         if (!checkAndDeleteDirectory(path)) {
173           canDeleteThis = false;
174         }
175       }
176       // otherwise we can just check the file
177       else if (!checkAndDelete(child)) {
178         canDeleteThis = false;
179       }
180     }
181 
182     // if the directory has children, we can't delete it, so we are done
183     if (!canDeleteThis) return false;
184 
185     // otherwise, all the children (that we know about) have been deleted, so we should try to
186     // delete this directory. However, don't do so recursively so we don't delete files that have
187     // been added since we last checked.
188     try {
189       return HBaseFileSystem.deleteFileFromFileSystem(fs, toCheck);
190     } catch (IOException e) {
191       if (LOG.isTraceEnabled()) {
192         LOG.trace("Couldn't delete directory: " + toCheck, e);
193       }
194     }
195 
196     // couldn't delete w/o exception, so we can't return success.
197     return false;
198   }
199 
200   /**
201    * Run the given file through each of the cleaners to see if it should be deleted, deleting it if
202    * necessary.
203    * @param fStat path of the file to check (and possibly delete)
204    * @throws IOException if cann't delete a file because of a filesystem issue
205    * @throws IllegalArgumentException if the file is a directory and has children
206    */
207   private boolean checkAndDelete(FileStatus fStat) throws IOException, IllegalArgumentException {
208     Path filePath = fStat.getPath();
209     // first check to see if the path is valid
210     if (!validate(filePath)) {
211       LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
212       boolean success = HBaseFileSystem.deleteDirFromFileSystem(fs, filePath);
213       if (!success) LOG.warn("Attempted to delete:" + filePath
214           + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
215 
216       return success;
217     }
218     // check each of the cleaners for the file
219     for (T cleaner : cleanersChain) {
220       if (cleaner.isStopped() || this.stopper.isStopped()) {
221         LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any file in:"
222             + this.oldFileDir);
223         return false;
224       }
225 
226       if (!cleaner.isFileDeletable(fStat)) {
227         // this file is not deletable, then we are done
228         if (LOG.isTraceEnabled()) {
229           LOG.trace(filePath + " is not deletable according to:" + cleaner);
230         }
231         return false;
232       }
233     }
234     // delete this file if it passes all the cleaners
235     if (LOG.isTraceEnabled()) {
236       LOG.trace("Removing:" + filePath + " from archive");
237     }
238     boolean success = HBaseFileSystem.deleteFileFromFileSystem(fs, filePath);
239     if (!success) {
240       LOG.warn("Attempted to delete:" + filePath
241           + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
242     }
243     return success;
244   }
245 
246   @Override
247   public void cleanup() {
248     for (T lc : this.cleanersChain) {
249       try {
250         lc.stop("Exiting");
251       } catch (Throwable t) {
252         LOG.warn("Stopping", t);
253       }
254     }
255   }
256 }