1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.cleaner;
19
20 import static org.junit.Assert.assertFalse;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.FileSystem;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.HBaseTestingUtility;
31 import org.apache.hadoop.hbase.SmallTests;
32 import org.apache.hadoop.hbase.Stoppable;
33 import org.apache.hadoop.hbase.util.FSUtils;
34 import org.junit.After;
35 import org.junit.Test;
36 import org.junit.experimental.categories.Category;
37 import org.mockito.Mockito;
38 import org.mockito.invocation.InvocationOnMock;
39 import org.mockito.stubbing.Answer;
40
41 @Category(SmallTests.class)
42 public class TestCleanerChore {
43
44 private static final Log LOG = LogFactory.getLog(TestCleanerChore.class);
45 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
46
47 @After
48 public void cleanup() throws Exception {
49
50 UTIL.cleanupTestDir();
51 }
52
53 @Test
54 public void testSavesFilesOnRequest() throws Exception {
55 Stoppable stop = new StoppableImplementation();
56 Configuration conf = UTIL.getConfiguration();
57 Path testDir = UTIL.getDataTestDir();
58 FileSystem fs = UTIL.getTestFileSystem();
59 String confKey = "hbase.test.cleaner.delegates";
60 conf.set(confKey, NeverDelete.class.getName());
61
62 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
63
64
65 Path parent = new Path(testDir, "parent");
66 Path file = new Path(parent, "someFile");
67 fs.mkdirs(parent);
68
69 fs.create(file).close();
70 assertTrue("Test file didn't get created.", fs.exists(file));
71
72
73 chore.chore();
74
75
76 assertTrue("File didn't get deleted", fs.exists(file));
77 assertTrue("Empty directory didn't get deleted", fs.exists(parent));
78 }
79
80 @Test
81 public void testDeletesEmptyDirectories() throws Exception {
82 Stoppable stop = new StoppableImplementation();
83 Configuration conf = UTIL.getConfiguration();
84 Path testDir = UTIL.getDataTestDir();
85 FileSystem fs = UTIL.getTestFileSystem();
86 String confKey = "hbase.test.cleaner.delegates";
87 conf.set(confKey, AlwaysDelete.class.getName());
88
89 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
90
91
92 Path parent = new Path(testDir, "parent");
93 Path child = new Path(parent, "child");
94 Path emptyChild = new Path(parent, "emptyChild");
95 Path file = new Path(child, "someFile");
96 fs.mkdirs(child);
97 fs.mkdirs(emptyChild);
98
99 fs.create(file).close();
100
101 Path topFile = new Path(testDir, "topFile");
102 fs.create(topFile).close();
103 assertTrue("Test file didn't get created.", fs.exists(file));
104 assertTrue("Test file didn't get created.", fs.exists(topFile));
105
106
107 chore.chore();
108
109
110 assertFalse("File didn't get deleted", fs.exists(topFile));
111 assertFalse("File didn't get deleted", fs.exists(file));
112 assertFalse("Empty directory didn't get deleted", fs.exists(child));
113 assertFalse("Empty directory didn't get deleted", fs.exists(parent));
114 }
115
116
117
118
119
120
121 @Test
122 public void testDoesNotCheckDirectories() throws Exception {
123 Stoppable stop = new StoppableImplementation();
124 Configuration conf = UTIL.getConfiguration();
125 Path testDir = UTIL.getDataTestDir();
126 FileSystem fs = UTIL.getTestFileSystem();
127 String confKey = "hbase.test.cleaner.delegates";
128 conf.set(confKey, AlwaysDelete.class.getName());
129
130 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
131
132 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
133 AlwaysDelete spy = Mockito.spy(delegate);
134 chore.cleanersChain.set(0, spy);
135
136
137 Path parent = new Path(testDir, "parent");
138 Path file = new Path(parent, "someFile");
139 fs.mkdirs(parent);
140
141 fs.create(file).close();
142 assertTrue("Test file didn't get created.", fs.exists(file));
143
144 chore.chore();
145
146 Mockito.verify(spy, Mockito.never()).isFileDeletable(parent);
147 Mockito.reset(spy);
148 }
149
150 @Test
151 public void testStoppedCleanerDoesNotDeleteFiles() throws Exception {
152 Stoppable stop = new StoppableImplementation();
153 Configuration conf = UTIL.getConfiguration();
154 Path testDir = UTIL.getDataTestDir();
155 FileSystem fs = UTIL.getTestFileSystem();
156 String confKey = "hbase.test.cleaner.delegates";
157 conf.set(confKey, AlwaysDelete.class.getName());
158
159 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
160
161
162 Path topFile = new Path(testDir, "topFile");
163 fs.create(topFile).close();
164 assertTrue("Test file didn't get created.", fs.exists(topFile));
165
166
167 stop.stop("testing stop");
168
169
170 chore.chore();
171
172
173 assertTrue("File got deleted while chore was stopped", fs.exists(topFile));
174 }
175
176
177
178
179
180
181 @Test
182 public void testCleanerDoesNotDeleteDirectoryWithLateAddedFiles() throws IOException {
183 Stoppable stop = new StoppableImplementation();
184 Configuration conf = UTIL.getConfiguration();
185 final Path testDir = UTIL.getDataTestDir();
186 final FileSystem fs = UTIL.getTestFileSystem();
187 String confKey = "hbase.test.cleaner.delegates";
188 conf.set(confKey, AlwaysDelete.class.getName());
189
190 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
191
192 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
193 AlwaysDelete spy = Mockito.spy(delegate);
194 chore.cleanersChain.set(0, spy);
195
196
197 final Path parent = new Path(testDir, "parent");
198 Path file = new Path(parent, "someFile");
199 fs.mkdirs(parent);
200
201 fs.create(file).close();
202 assertTrue("Test file didn't get created.", fs.exists(file));
203 final Path addedFile = new Path(parent, "addedFile");
204
205
206 Mockito.doAnswer(new Answer<Boolean>() {
207 @Override
208 public Boolean answer(InvocationOnMock invocation) throws Throwable {
209 fs.create(addedFile).close();
210 FSUtils.logFileSystemState(fs, testDir, LOG);
211 return (Boolean) invocation.callRealMethod();
212 }
213 }).when(spy).isFileDeletable(Mockito.any(Path.class));
214
215
216 chore.chore();
217
218
219 assertTrue("Added file unexpectedly deleted", fs.exists(addedFile));
220 assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
221 assertFalse("Original file unexpectedly retained", fs.exists(file));
222 Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(Path.class));
223 Mockito.reset(spy);
224 }
225
226
227
228
229
230
231
232
233
234
235 @Test
236 public void testNoExceptionFromDirectoryWithRacyChildren() throws Exception {
237 Stoppable stop = new StoppableImplementation();
238
239
240 HBaseTestingUtility localUtil = new HBaseTestingUtility();
241 Configuration conf = localUtil.getConfiguration();
242 final Path testDir = UTIL.getDataTestDir();
243 final FileSystem fs = UTIL.getTestFileSystem();
244 LOG.debug("Writing test data to: " + testDir);
245 String confKey = "hbase.test.cleaner.delegates";
246 conf.set(confKey, AlwaysDelete.class.getName());
247
248 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
249
250 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
251 AlwaysDelete spy = Mockito.spy(delegate);
252 chore.cleanersChain.set(0, spy);
253
254
255 final Path parent = new Path(testDir, "parent");
256 Path file = new Path(parent, "someFile");
257 fs.mkdirs(parent);
258
259 fs.create(file).close();
260 assertTrue("Test file didn't get created.", fs.exists(file));
261 final Path racyFile = new Path(parent, "addedFile");
262
263
264 Mockito.doAnswer(new Answer<Boolean>() {
265 @Override
266 public Boolean answer(InvocationOnMock invocation) throws Throwable {
267 fs.create(racyFile).close();
268 FSUtils.logFileSystemState(fs, testDir, LOG);
269 return (Boolean) invocation.callRealMethod();
270 }
271 }).when(spy).isFileDeletable(Mockito.any(Path.class));
272
273
274 if (chore.checkAndDeleteDirectory(parent)) {
275 throw new Exception(
276 "Reported success deleting directory, should have failed when adding file mid-iteration");
277 }
278
279
280 assertTrue("Added file unexpectedly deleted", fs.exists(racyFile));
281 assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
282 assertFalse("Original file unexpectedly retained", fs.exists(file));
283 Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(Path.class));
284 }
285
286 private static class AllValidPaths extends CleanerChore<BaseHFileCleanerDelegate> {
287
288 public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs,
289 Path oldFileDir, String confkey) {
290 super(name, Integer.MAX_VALUE, s, conf, fs, oldFileDir, confkey);
291 }
292
293
294 @Override
295 protected boolean validate(Path file) {
296 return true;
297 }
298 };
299
300 public static class AlwaysDelete extends BaseHFileCleanerDelegate {
301 @Override
302 public boolean isFileDeletable(Path file) {
303 return true;
304 }
305 }
306
307 public static class NeverDelete extends BaseHFileCleanerDelegate {
308 @Override
309 public boolean isFileDeletable(Path file) {
310 return false;
311 }
312 }
313
314
315
316
317 private static class StoppableImplementation implements Stoppable {
318 private volatile boolean stop;
319
320 @Override
321 public void stop(String why) {
322 this.stop = true;
323 }
324
325 @Override
326 public boolean isStopped() {
327 return this.stop;
328 }
329
330 }
331 }