1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.cleaner;
19
20 import static org.junit.Assert.assertFalse;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.FileStatus;
29 import org.apache.hadoop.fs.FileSystem;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.SmallTests;
33 import org.apache.hadoop.hbase.Stoppable;
34 import org.apache.hadoop.hbase.util.FSUtils;
35 import org.junit.After;
36 import org.junit.Test;
37 import org.junit.experimental.categories.Category;
38 import org.mockito.Mockito;
39 import org.mockito.invocation.InvocationOnMock;
40 import org.mockito.stubbing.Answer;
41
42 @Category(SmallTests.class)
43 public class TestCleanerChore {
44
45 private static final Log LOG = LogFactory.getLog(TestCleanerChore.class);
46 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
47
48 @After
49 public void cleanup() throws Exception {
50
51 UTIL.cleanupTestDir();
52 }
53
54 @Test
55 public void testSavesFilesOnRequest() throws Exception {
56 Stoppable stop = new StoppableImplementation();
57 Configuration conf = UTIL.getConfiguration();
58 Path testDir = UTIL.getDataTestDir();
59 FileSystem fs = UTIL.getTestFileSystem();
60 String confKey = "hbase.test.cleaner.delegates";
61 conf.set(confKey, NeverDelete.class.getName());
62
63 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
64
65
66 Path parent = new Path(testDir, "parent");
67 Path file = new Path(parent, "someFile");
68 fs.mkdirs(parent);
69
70 fs.create(file).close();
71 assertTrue("Test file didn't get created.", fs.exists(file));
72
73
74 chore.chore();
75
76
77 assertTrue("File didn't get deleted", fs.exists(file));
78 assertTrue("Empty directory didn't get deleted", fs.exists(parent));
79 }
80
81 @Test
82 public void testDeletesEmptyDirectories() throws Exception {
83 Stoppable stop = new StoppableImplementation();
84 Configuration conf = UTIL.getConfiguration();
85 Path testDir = UTIL.getDataTestDir();
86 FileSystem fs = UTIL.getTestFileSystem();
87 String confKey = "hbase.test.cleaner.delegates";
88 conf.set(confKey, AlwaysDelete.class.getName());
89
90 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
91
92
93 Path parent = new Path(testDir, "parent");
94 Path child = new Path(parent, "child");
95 Path emptyChild = new Path(parent, "emptyChild");
96 Path file = new Path(child, "someFile");
97 fs.mkdirs(child);
98 fs.mkdirs(emptyChild);
99
100 fs.create(file).close();
101
102 Path topFile = new Path(testDir, "topFile");
103 fs.create(topFile).close();
104 assertTrue("Test file didn't get created.", fs.exists(file));
105 assertTrue("Test file didn't get created.", fs.exists(topFile));
106
107
108 chore.chore();
109
110
111 assertFalse("File didn't get deleted", fs.exists(topFile));
112 assertFalse("File didn't get deleted", fs.exists(file));
113 assertFalse("Empty directory didn't get deleted", fs.exists(child));
114 assertFalse("Empty directory didn't get deleted", fs.exists(parent));
115 }
116
117
118
119
120
121
122 @Test
123 public void testDoesNotCheckDirectories() throws Exception {
124 Stoppable stop = new StoppableImplementation();
125 Configuration conf = UTIL.getConfiguration();
126 Path testDir = UTIL.getDataTestDir();
127 FileSystem fs = UTIL.getTestFileSystem();
128 String confKey = "hbase.test.cleaner.delegates";
129 conf.set(confKey, AlwaysDelete.class.getName());
130
131 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
132
133 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
134 AlwaysDelete spy = Mockito.spy(delegate);
135 chore.cleanersChain.set(0, spy);
136
137
138 Path parent = new Path(testDir, "parent");
139 Path file = new Path(parent, "someFile");
140 fs.mkdirs(parent);
141 assertTrue("Test parent didn't get created.", fs.exists(parent));
142
143 fs.create(file).close();
144 assertTrue("Test file didn't get created.", fs.exists(file));
145
146 FileStatus fStat = fs.getFileStatus(parent);
147 chore.chore();
148
149 Mockito.verify(spy, Mockito.never()).isFileDeletable(fStat);
150 Mockito.reset(spy);
151 }
152
153 @Test
154 public void testStoppedCleanerDoesNotDeleteFiles() throws Exception {
155 Stoppable stop = new StoppableImplementation();
156 Configuration conf = UTIL.getConfiguration();
157 Path testDir = UTIL.getDataTestDir();
158 FileSystem fs = UTIL.getTestFileSystem();
159 String confKey = "hbase.test.cleaner.delegates";
160 conf.set(confKey, AlwaysDelete.class.getName());
161
162 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
163
164
165 Path topFile = new Path(testDir, "topFile");
166 fs.create(topFile).close();
167 assertTrue("Test file didn't get created.", fs.exists(topFile));
168
169
170 stop.stop("testing stop");
171
172
173 chore.chore();
174
175
176 assertTrue("File got deleted while chore was stopped", fs.exists(topFile));
177 }
178
179
180
181
182
183
184 @Test
185 public void testCleanerDoesNotDeleteDirectoryWithLateAddedFiles() throws IOException {
186 Stoppable stop = new StoppableImplementation();
187 Configuration conf = UTIL.getConfiguration();
188 final Path testDir = UTIL.getDataTestDir();
189 final FileSystem fs = UTIL.getTestFileSystem();
190 String confKey = "hbase.test.cleaner.delegates";
191 conf.set(confKey, AlwaysDelete.class.getName());
192
193 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
194
195 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
196 AlwaysDelete spy = Mockito.spy(delegate);
197 chore.cleanersChain.set(0, spy);
198
199
200 final Path parent = new Path(testDir, "parent");
201 Path file = new Path(parent, "someFile");
202 fs.mkdirs(parent);
203
204 fs.create(file).close();
205 assertTrue("Test file didn't get created.", fs.exists(file));
206 final Path addedFile = new Path(parent, "addedFile");
207
208
209 Mockito.doAnswer(new Answer<Boolean>() {
210 @Override
211 public Boolean answer(InvocationOnMock invocation) throws Throwable {
212 fs.create(addedFile).close();
213 FSUtils.logFileSystemState(fs, testDir, LOG);
214 return (Boolean) invocation.callRealMethod();
215 }
216 }).when(spy).isFileDeletable(Mockito.any(FileStatus.class));
217
218
219 chore.chore();
220
221
222 assertTrue("Added file unexpectedly deleted", fs.exists(addedFile));
223 assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
224 assertFalse("Original file unexpectedly retained", fs.exists(file));
225 Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class));
226 Mockito.reset(spy);
227 }
228
229
230
231
232
233
234
235
236
237
238 @Test
239 public void testNoExceptionFromDirectoryWithRacyChildren() throws Exception {
240 Stoppable stop = new StoppableImplementation();
241
242
243 HBaseTestingUtility localUtil = new HBaseTestingUtility();
244 Configuration conf = localUtil.getConfiguration();
245 final Path testDir = UTIL.getDataTestDir();
246 final FileSystem fs = UTIL.getTestFileSystem();
247 LOG.debug("Writing test data to: " + testDir);
248 String confKey = "hbase.test.cleaner.delegates";
249 conf.set(confKey, AlwaysDelete.class.getName());
250
251 AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey);
252
253 AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
254 AlwaysDelete spy = Mockito.spy(delegate);
255 chore.cleanersChain.set(0, spy);
256
257
258 final Path parent = new Path(testDir, "parent");
259 Path file = new Path(parent, "someFile");
260 fs.mkdirs(parent);
261
262 fs.create(file).close();
263 assertTrue("Test file didn't get created.", fs.exists(file));
264 final Path racyFile = new Path(parent, "addedFile");
265
266
267 Mockito.doAnswer(new Answer<Boolean>() {
268 @Override
269 public Boolean answer(InvocationOnMock invocation) throws Throwable {
270 fs.create(racyFile).close();
271 FSUtils.logFileSystemState(fs, testDir, LOG);
272 return (Boolean) invocation.callRealMethod();
273 }
274 }).when(spy).isFileDeletable(Mockito.any(FileStatus.class));
275
276
277 if (chore.checkAndDeleteDirectory(parent)) {
278 throw new Exception(
279 "Reported success deleting directory, should have failed when adding file mid-iteration");
280 }
281
282
283 assertTrue("Added file unexpectedly deleted", fs.exists(racyFile));
284 assertTrue("Parent directory deleted unexpectedly", fs.exists(parent));
285 assertFalse("Original file unexpectedly retained", fs.exists(file));
286 Mockito.verify(spy, Mockito.times(1)).isFileDeletable(Mockito.any(FileStatus.class));
287 }
288
289 private static class AllValidPaths extends CleanerChore<BaseHFileCleanerDelegate> {
290
291 public AllValidPaths(String name, Stoppable s, Configuration conf, FileSystem fs,
292 Path oldFileDir, String confkey) {
293 super(name, Integer.MAX_VALUE, s, conf, fs, oldFileDir, confkey);
294 }
295
296
297 @Override
298 protected boolean validate(Path file) {
299 return true;
300 }
301 };
302
303 public static class AlwaysDelete extends BaseHFileCleanerDelegate {
304 @Override
305 public boolean isFileDeletable(FileStatus fStat) {
306 return true;
307 }
308 }
309
310 public static class NeverDelete extends BaseHFileCleanerDelegate {
311 @Override
312 public boolean isFileDeletable(FileStatus fStat) {
313 return false;
314 }
315 }
316
317
318
319
320 private static class StoppableImplementation implements Stoppable {
321 private volatile boolean stop;
322
323 @Override
324 public void stop(String why) {
325 this.stop = true;
326 }
327
328 @Override
329 public boolean isStopped() {
330 return this.stop;
331 }
332
333 }
334 }