1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.cleaner;
19
20 import java.io.IOException;
21 import java.util.LinkedList;
22 import java.util.List;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.FileStatus;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.Chore;
31 import org.apache.hadoop.hbase.HBaseFileSystem;
32 import org.apache.hadoop.hbase.RemoteExceptionHandler;
33 import org.apache.hadoop.hbase.Stoppable;
34 import org.apache.hadoop.hbase.util.FSUtils;
35
36
37
38
39
40 public abstract class CleanerChore<T extends FileCleanerDelegate> extends Chore {
41
42 private static final Log LOG = LogFactory.getLog(CleanerChore.class.getName());
43
44 private final FileSystem fs;
45 private final Path oldFileDir;
46 private final Configuration conf;
47 List<T> cleanersChain;
48
49
50
51
52
53
54
55
56
57
58 public CleanerChore(String name, final int sleepPeriod, final Stoppable s, Configuration conf,
59 FileSystem fs, Path oldFileDir, String confKey) {
60 super(name, sleepPeriod, s);
61 this.fs = fs;
62 this.oldFileDir = oldFileDir;
63 this.conf = conf;
64
65 initCleanerChain(confKey);
66 }
67
68
69
70
71
72
73
74 protected abstract boolean validate(Path file);
75
76
77
78
79
80 private void initCleanerChain(String confKey) {
81 this.cleanersChain = new LinkedList<T>();
82 String[] logCleaners = conf.getStrings(confKey);
83 if (logCleaners != null) {
84 for (String className : logCleaners) {
85 T logCleaner = newFileCleaner(className, conf);
86 if (logCleaner != null) {
87 LOG.debug("initialize cleaner=" + className);
88 this.cleanersChain.add(logCleaner);
89 }
90 }
91 }
92 }
93
94
95
96
97
98
99
100
101 public T newFileCleaner(String className, Configuration conf) {
102 try {
103 Class<? extends FileCleanerDelegate> c = Class.forName(className).asSubclass(
104 FileCleanerDelegate.class);
105 @SuppressWarnings("unchecked")
106 T cleaner = (T) c.newInstance();
107 cleaner.setConf(conf);
108 return cleaner;
109 } catch (Exception e) {
110 LOG.warn("Can NOT create CleanerDelegate: " + className, e);
111
112 return null;
113 }
114 }
115
116 @Override
117 protected void chore() {
118 try {
119 FileStatus[] files = FSUtils.listStatus(this.fs, this.oldFileDir, null);
120
121 if (files == null) return;
122
123 for (FileStatus file : files) {
124 try {
125 if (file.isDir()) checkAndDeleteDirectory(file.getPath());
126 else checkAndDelete(file.getPath());
127 } catch (IOException e) {
128 e = RemoteExceptionHandler.checkIOException(e);
129 LOG.warn("Error while cleaning the logs", e);
130 }
131 }
132 } catch (IOException e) {
133 LOG.warn("Failed to get status of:" + oldFileDir);
134 }
135
136 }
137
138
139
140
141
142
143
144
145
146
147
148
149 public boolean checkAndDeleteDirectory(Path toCheck) throws IOException {
150 if (LOG.isTraceEnabled()) {
151 LOG.trace("Checking directory: " + toCheck);
152 }
153 FileStatus[] children = FSUtils.listStatus(fs, toCheck, null);
154
155 if (children == null) {
156 try {
157 return HBaseFileSystem.deleteFileFromFileSystem(fs, toCheck);
158 } catch (IOException e) {
159 if (LOG.isTraceEnabled()) {
160 LOG.trace("Couldn't delete directory: " + toCheck, e);
161 }
162 }
163
164 return false;
165 }
166
167 boolean canDeleteThis = true;
168 for (FileStatus child : children) {
169 Path path = child.getPath();
170
171 if (child.isDir()) {
172 if (!checkAndDeleteDirectory(path)) {
173 canDeleteThis = false;
174 }
175 }
176
177 else if (!checkAndDelete(path)) {
178 canDeleteThis = false;
179 }
180 }
181
182
183 if (!canDeleteThis) return false;
184
185
186
187
188 try {
189 return HBaseFileSystem.deleteFileFromFileSystem(fs, toCheck);
190 } catch (IOException e) {
191 if (LOG.isTraceEnabled()) {
192 LOG.trace("Couldn't delete directory: " + toCheck, e);
193 }
194 }
195
196
197 return false;
198 }
199
200
201
202
203
204
205
206
207 private boolean checkAndDelete(Path filePath) throws IOException, IllegalArgumentException {
208
209 if (!validate(filePath)) {
210 LOG.warn("Found a wrongly formatted file: " + filePath.getName() + " deleting it.");
211 boolean success = HBaseFileSystem.deleteDirFromFileSystem(fs, filePath);
212 if (!success) LOG.warn("Attempted to delete:" + filePath
213 + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
214
215 return success;
216 }
217
218 for (T cleaner : cleanersChain) {
219 if (cleaner.isStopped() || this.stopper.isStopped()) {
220 LOG.warn("A file cleaner" + this.getName() + " is stopped, won't delete any file in:"
221 + this.oldFileDir);
222 return false;
223 }
224
225 if (!cleaner.isFileDeletable(filePath)) {
226
227 if (LOG.isTraceEnabled()) {
228 LOG.trace(filePath + " is not deletable according to:" + cleaner);
229 }
230 return false;
231 }
232 }
233
234 if (LOG.isTraceEnabled()) {
235 LOG.trace("Removing:" + filePath + " from archive");
236 }
237 boolean success = HBaseFileSystem.deleteFileFromFileSystem(fs, filePath);
238 if (!success) {
239 LOG.warn("Attempted to delete:" + filePath
240 + ", but couldn't. Run cleaner chain and attempt to delete on next pass.");
241 }
242 return success;
243 }
244
245 @Override
246 public void cleanup() {
247 for (T lc : this.cleanersChain) {
248 try {
249 lc.stop("Exiting");
250 } catch (Throwable t) {
251 LOG.warn("Stopping", t);
252 }
253 }
254 }
255 }