1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.cleaner;
19
20 import java.io.IOException;
21 import java.util.LinkedList;
22 import java.util.List;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.FileStatus;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.Chore;
31 import org.apache.hadoop.hbase.RemoteExceptionHandler;
32 import org.apache.hadoop.hbase.Stoppable;
33 import org.apache.hadoop.hbase.util.FSUtils;
34
35
36
37
38
39 public abstract class CleanerChore<T extends FileCleanerDelegate> extends Chore {
40
41 private static final Log LOG = LogFactory.getLog(CleanerChore.class.getName());
42
43 private final FileSystem fs;
44 private final Path oldFileDir;
45 private final Configuration conf;
46 protected List<T> cleanersChain;
47
48
49
50
51
52
53
54
55
56
57 public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf,
58 FileSystem fs, Path oldFileDir, String confKey) {
59 super(name, sleepPeriod, s);
60 this.fs = fs;
61 this.oldFileDir = oldFileDir;
62 this.conf = conf;
63
64 initCleanerChain(confKey);
65 }
66
67
68
69
70
71
72
73 protected abstract boolean validate(Path file);
74
75
76
77
78
79 private void initCleanerChain(String confKey) {
80 this.cleanersChain = new LinkedList<T>();
81 String[] logCleaners = conf.getStrings(confKey);
82 if (logCleaners != null) {
83 for (String className : logCleaners) {
84 T logCleaner = newFileCleaner(className, conf);
85 if (logCleaner != null) {
86 LOG.debug("initialize cleaner=" + className);
87 this.cleanersChain.add(logCleaner);
88 }
89 }
90 }
91 }
92
93
94
95
96
97
98
99
100 public T newFileCleaner(String className, Configuration conf) {
101 try {
102 Class<? extends FileCleanerDelegate> c = Class.forName(className).asSubclass(
103 FileCleanerDelegate.class);
104 @SuppressWarnings("unchecked")
105 T cleaner = (T) c.newInstance();
106 cleaner.setConf(conf);
107 return cleaner;
108 } catch (Exception e) {
109 LOG.warn("Can NOT create CleanerDelegate: " + className, e);
110
111 return null;
112 }
113 }
114
115 @Override
116 protected void chore() {
117 try {
118 FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir, null);
119
120 if (files == null) return;
121
122 for (FileStatus file : files) {
123 try {
124 if (file.isDir()) checkAndDeleteDirectory(file.getPath());
125 else checkAndDelete(file);
126 } catch (IOException e) {
127 e = RemoteExceptionHandler.checkIOException(e);
128 LOG.warn("Error while cleaning the logs", e);
129 }
130 }
131 } catch (IOException e) {
132 LOG.warn("Failed to get status of: " + oldFileDir);
133 }
134
135 }
136
137
138
139
140
141
142
143
144
145
146
147
148 public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
149 if (LOG.isTraceEnabled()) {
150 LOG.trace("Checking directory: " + toCheck);
151 }
152 FileStatus[] children = FSUtils.listStatus(fs, toCheck);
153
154 if (children == null) {
155 try {
156 return fs.delete(toCheck, false);
157 } catch (IOException e) {
158 if (LOG.isTraceEnabled()) {
159 LOG.trace("Couldn't delete directory: " + toCheck, e);
160 }
161 }
162
163 return false;
164 }
165
166 boolean canDeleteThis = true;
167 for (FileStatus child : children) {
168 Path path = child.getPath();
169
170 if (child.isDir()) {
171 if (!checkAndDeleteDirectory(path)) {
172 canDeleteThis = false;
173 }
174 }
175
176 else if (!checkAndDelete(child)) {
177 canDeleteThis = false;
178 }
179 }
180
181
182 if (!canDeleteThis) return false;
183
184
185
186
187 try {
188 return fs.delete(toCheck, false);
189 } catch (IOException e) {
190 if (LOG.isTraceEnabled()) {
191 LOG.trace("Couldn't delete directory: " + toCheck, e);
192 }
193 }
194
195
196 return false;
197 }
198
199
200
201
202
203
204
205
206 private boolean checkAndDelete(FileStatus fStat) throws IOException, IllegalArgumentException {
207 Path filePath = fStat.getPath();
208
209 if (!validate(filePath)) {
210 LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
211 boolean success = this.fs.delete(filePath, true);
212 if(!success)
213 LOG.warn("Attempted to delete: " + filePath
214 + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
215
216 return success;
217 }
218
219
220 for (T cleaner : cleanersChain) {
221 if (cleaner.isStopped() || this.stopper.isStopped()) {
222 LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any file in:"
223 + this.oldFileDir);
224 return false;
225 }
226
227 if (!cleaner.isFileDeletable(fStat)) {
228
229 if (LOG.isTraceEnabled()) {
230 LOG.trace(filePath + " is not deletable according to:" + cleaner);
231 }
232 return false;
233 }
234 }
235
236 if (LOG.isTraceEnabled()) {
237 LOG.trace("Removing: " + filePath + " from archive");
238 }
239 boolean success = this.fs.delete(filePath, false);
240 if (!success) {
241 LOG.warn("Attempted to delete:" + filePath
242 + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
243 }
244 return success;
245 }
246
247
248 @Override
249 public void cleanup() {
250 for (T lc : this.cleanersChain) {
251 try {
252 lc.stop("Exiting");
253 } catch (Throwable t) {
254 LOG.warn("Stopping", t);
255 }
256 }
257 }
258 }