1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.cleaner;
19
20 import java.io.IOException;
21 import java.util.LinkedList;
22 import java.util.List;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.FileStatus;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.Chore;
31 import org.apache.hadoop.hbase.RemoteExceptionHandler;
32 import org.apache.hadoop.hbase.Stoppable;
33 import org.apache.hadoop.hbase.util.FSUtils;
34
35
36
37
38
39 public abstract class CleanerChore<T extends FileCleanerDelegate> extends Chore {
40
41 private static final Log LOG = LogFactory.getLog(CleanerChore.class.getName());
42
43 private final FileSystem fs;
44 private final Path oldFileDir;
45 private final Configuration conf;
46 protected List<T> cleanersChain;
47
48
49
50
51
52
53
54
55
56
57 public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf,
58 FileSystem fs, Path oldFileDir, String confKey) {
59 super(name, sleepPeriod, s);
60 this.fs = fs;
61 this.oldFileDir = oldFileDir;
62 this.conf = conf;
63
64 initCleanerChain(confKey);
65 }
66
67
68
69
70
71
72
73 protected abstract boolean validate(Path file);
74
75
76
77
78
79 private void initCleanerChain(String confKey) {
80 this.cleanersChain = new LinkedList<T>();
81 String[] logCleaners = conf.getStrings(confKey);
82 if (logCleaners != null) {
83 for (String className : logCleaners) {
84 T logCleaner = newFileCleaner(className, conf);
85 if (logCleaner != null) {
86 LOG.debug("initialize cleaner=" + className);
87 this.cleanersChain.add(logCleaner);
88 }
89 }
90 }
91 }
92
93
94
95
96
97
98
99
100 public T newFileCleaner(String className, Configuration conf) {
101 try {
102 Class<? extends FileCleanerDelegate> c = Class.forName(className).asSubclass(
103 FileCleanerDelegate.class);
104 @SuppressWarnings("unchecked")
105 T cleaner = (T) c.newInstance();
106 cleaner.setConf(conf);
107 return cleaner;
108 } catch (Exception e) {
109 LOG.warn("Can NOT create CleanerDelegate: " + className, e);
110
111 return null;
112 }
113 }
114
115 @Override
116 protected void chore() {
117 try {
118 FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir, null);
119
120 if (files == null) return;
121
122 for (FileStatus file : files) {
123 try {
124 if (file.isDir()) checkAndDeleteDirectory(file.getPath());
125 else checkAndDelete(file.getPath());
126 } catch (IOException e) {
127 e = RemoteExceptionHandler.checkIOException(e);
128 LOG.warn("Error while cleaning the logs", e);
129 }
130 }
131 } catch (IOException e) {
132 LOG.warn("Failed to get status of: " + oldFileDir);
133 }
134
135 }
136
137
138
139
140
141
142
143
144
145
146
147
148 public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
149 if (LOG.isTraceEnabled()) {
150 LOG.trace("Checking directory: " + toCheck);
151 }
152 FileStatus[] children = FSUtils.listStatus(fs, toCheck);
153
154 if (children == null) {
155 try {
156 return fs.delete(toCheck, false);
157 } catch (IOException e) {
158 if (LOG.isTraceEnabled()) {
159 LOG.trace("Couldn't delete directory: " + toCheck, e);
160 }
161 }
162
163 return false;
164 }
165
166 boolean canDeleteThis = true;
167 for (FileStatus child : children) {
168 Path path = child.getPath();
169
170 if (child.isDir()) {
171 if (!checkAndDeleteDirectory(path)) {
172 canDeleteThis = false;
173 }
174 }
175
176 else if (!checkAndDelete(path)) {
177 canDeleteThis = false;
178 }
179 }
180
181
182 if (!canDeleteThis) return false;
183
184
185
186
187 try {
188 return fs.delete(toCheck, false);
189 } catch (IOException e) {
190 if (LOG.isTraceEnabled()) {
191 LOG.trace("Couldn't delete directory: " + toCheck, e);
192 }
193 }
194
195
196 return false;
197 }
198
199
200
201
202
203
204
205
206 private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumentException {
207
208 if (!validate(filePath)) {
209 LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
210 boolean success = this.fs.delete(filePath, true);
211 if(!success)
212 LOG.warn("Attempted to delete: " + filePath
213 + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
214
215 return success;
216 }
217
218
219 for (T cleaner : cleanersChain) {
220 if (cleaner.isStopped() || this.stopper.isStopped()) {
221 LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any file in:"
222 + this.oldFileDir);
223 return false;
224 }
225
226 if (!cleaner.isFileDeletable(filePath)) {
227
228 if (LOG.isTraceEnabled()) {
229 LOG.trace(filePath + " is not deletable according to:" + cleaner);
230 }
231 return false;
232 }
233 }
234
235 if (LOG.isTraceEnabled()) {
236 LOG.trace("Removing: " + filePath + " from archive");
237 }
238 boolean success = this.fs.delete(filePath, false);
239 if (!success) {
240 LOG.warn("Attempted to delete:" + filePath
241 + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
242 }
243 return success;
244 }
245
246
247 @Override
248 public void cleanup() {
249 for (T lc : this.cleanersChain) {
250 try {
251 lc.stop("Exiting");
252 } catch (Throwable t) {
253 LOG.warn("Stopping", t);
254 }
255 }
256 }
257 }