1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.master.cleaner;
21
22 import static org.junit.Assert.assertEquals;
23
24 import java.io.IOException;
25 import java.net.URLEncoder;
26 import java.util.concurrent.atomic.AtomicBoolean;
27
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.fs.FileStatus;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.*;
33 import org.apache.hadoop.hbase.catalog.CatalogTracker;
34 import org.apache.hadoop.hbase.master.cleaner.LogCleaner;
35 import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
36 import org.apache.hadoop.hbase.replication.regionserver.Replication;
37 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
38 import org.junit.AfterClass;
39 import org.junit.BeforeClass;
40 import org.junit.Test;
41 import org.junit.experimental.categories.Category;
42
43 @Category(MediumTests.class)
44 public class TestLogsCleaner {
45
46 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
47
48
49
50
51 @BeforeClass
52 public static void setUpBeforeClass() throws Exception {
53 TEST_UTIL.startMiniZKCluster();
54 }
55
56
57
58
59 @AfterClass
60 public static void tearDownAfterClass() throws Exception {
61 TEST_UTIL.shutdownMiniZKCluster();
62 }
63
64 @Test
65 public void testLogCleaning() throws Exception{
66 Configuration conf = TEST_UTIL.getConfiguration();
67
68 long ttl = 2000;
69 conf.setLong("hbase.master.logcleaner.ttl", ttl);
70 conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
71 Replication.decorateMasterConfiguration(conf);
72 Server server = new DummyServer();
73 ReplicationZookeeper zkHelper =
74 new ReplicationZookeeper(server, new AtomicBoolean(true));
75
76 Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(),
77 HConstants.HREGION_OLDLOGDIR_NAME);
78 String fakeMachineName =
79 URLEncoder.encode(server.getServerName().toString(), "UTF8");
80
81 FileSystem fs = FileSystem.get(conf);
82 LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, oldLogDir);
83
84
85 long now = System.currentTimeMillis();
86 fs.delete(oldLogDir, true);
87 fs.mkdirs(oldLogDir);
88
89 fs.createNewFile(new Path(oldLogDir, "a"));
90 fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + "a"));
91
92
93 System.out.println("Now is: " + now);
94 for (int i = 1; i < 31; i++) {
95
96
97 Path fileName = new Path(oldLogDir, fakeMachineName + "." + (now - i) );
98 fs.createNewFile(fileName);
99
100
101
102
103 if (i % (30/3) == 1) {
104 zkHelper.addLogToList(fileName.getName(), fakeMachineName);
105 System.out.println("Replication log file: " + fileName);
106 }
107 }
108
109
110 Thread.sleep(ttl);
111 fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + now));
112
113
114
115 fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now + 10000) ));
116
117 for (FileStatus stat : fs.listStatus(oldLogDir)) {
118 System.out.println(stat.getPath().toString());
119 }
120
121 assertEquals(34, fs.listStatus(oldLogDir).length);
122
123 cleaner.chore();
124
125
126
127 assertEquals(5, fs.listStatus(oldLogDir).length);
128
129 for (FileStatus file : fs.listStatus(oldLogDir)) {
130 System.out.println("Kept log files: " + file.getPath().getName());
131 }
132 }
133
134 static class DummyServer implements Server {
135
136 @Override
137 public Configuration getConfiguration() {
138 return TEST_UTIL.getConfiguration();
139 }
140
141 @Override
142 public ZooKeeperWatcher getZooKeeper() {
143 try {
144 return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
145 } catch (IOException e) {
146 e.printStackTrace();
147 }
148 return null;
149 }
150
151 @Override
152 public CatalogTracker getCatalogTracker() {
153 return null;
154 }
155
156 @Override
157 public ServerName getServerName() {
158 return new ServerName("regionserver,60020,000000");
159 }
160
161 @Override
162 public void abort(String why, Throwable e) {}
163
164 @Override
165 public boolean isAborted() {
166 return false;
167 }
168
169 @Override
170 public void stop(String why) {}
171
172 @Override
173 public boolean isStopped() {
174 return false;
175 }
176 }
177
178 @org.junit.Rule
179 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
180 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
181 }
182