1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.cleaner;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.net.URLEncoder;
24
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileStatus;
27 import org.apache.hadoop.fs.FileSystem;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.CoordinatedStateManager;
30 import org.apache.hadoop.hbase.HBaseTestingUtility;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.testclassification.MediumTests;
33 import org.apache.hadoop.hbase.Server;
34 import org.apache.hadoop.hbase.ServerName;
35 import org.apache.hadoop.hbase.Waiter;
36 import org.apache.hadoop.hbase.client.ClusterConnection;
37 import org.apache.hadoop.hbase.replication.ReplicationFactory;
38 import org.apache.hadoop.hbase.replication.ReplicationQueues;
39 import org.apache.hadoop.hbase.replication.regionserver.Replication;
40 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
41 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
42 import org.junit.AfterClass;
43 import org.junit.BeforeClass;
44 import org.junit.Test;
45 import org.junit.experimental.categories.Category;
46
47 @Category(MediumTests.class)
48 public class TestLogsCleaner {
49
50 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
51
52
53
54
55 @BeforeClass
56 public static void setUpBeforeClass() throws Exception {
57 TEST_UTIL.startMiniZKCluster();
58 }
59
60
61
62
63 @AfterClass
64 public static void tearDownAfterClass() throws Exception {
65 TEST_UTIL.shutdownMiniZKCluster();
66 }
67
68 @Test
69 public void testLogCleaning() throws Exception{
70 Configuration conf = TEST_UTIL.getConfiguration();
71
72 long ttl = 10000;
73 conf.setLong("hbase.master.logcleaner.ttl", ttl);
74 conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
75 Replication.decorateMasterConfiguration(conf);
76 Server server = new DummyServer();
77 ReplicationQueues repQueues =
78 ReplicationFactory.getReplicationQueues(server.getZooKeeper(), conf, server);
79 repQueues.init(server.getServerName().toString());
80 final Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(),
81 HConstants.HREGION_OLDLOGDIR_NAME);
82 String fakeMachineName =
83 URLEncoder.encode(server.getServerName().toString(), "UTF8");
84
85 final FileSystem fs = FileSystem.get(conf);
86
87
88 long now = System.currentTimeMillis();
89 fs.delete(oldLogDir, true);
90 fs.mkdirs(oldLogDir);
91
92 fs.createNewFile(new Path(oldLogDir, "a"));
93 fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + "a"));
94
95
96 System.out.println("Now is: " + now);
97 for (int i = 1; i < 31; i++) {
98
99
100 Path fileName = new Path(oldLogDir, fakeMachineName + "." + (now - i) );
101 fs.createNewFile(fileName);
102
103
104
105
106 if (i % (30/3) == 1) {
107 repQueues.addLog(fakeMachineName, fileName.getName());
108 System.out.println("Replication log file: " + fileName);
109 }
110 }
111
112
113 Thread.sleep(ttl);
114 fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + now));
115
116
117
118 fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now + 10000) ));
119
120 for (FileStatus stat : fs.listStatus(oldLogDir)) {
121 System.out.println(stat.getPath().toString());
122 }
123
124 assertEquals(34, fs.listStatus(oldLogDir).length);
125
126 LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, oldLogDir);
127 cleaner.chore();
128
129
130
131 TEST_UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
132 @Override
133 public boolean evaluate() throws Exception {
134 return 5 == fs.listStatus(oldLogDir).length;
135 }
136 });
137
138 for (FileStatus file : fs.listStatus(oldLogDir)) {
139 System.out.println("Kept log files: " + file.getPath().getName());
140 }
141 }
142
143 static class DummyServer implements Server {
144
145 @Override
146 public Configuration getConfiguration() {
147 return TEST_UTIL.getConfiguration();
148 }
149
150 @Override
151 public ZooKeeperWatcher getZooKeeper() {
152 try {
153 return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
154 } catch (IOException e) {
155 e.printStackTrace();
156 }
157 return null;
158 }
159
160 @Override
161 public CoordinatedStateManager getCoordinatedStateManager() {
162 return null;
163 }
164
165 @Override
166 public ClusterConnection getConnection() {
167 return null;
168 }
169
170 @Override
171 public MetaTableLocator getMetaTableLocator() {
172 return null;
173 }
174
175 @Override
176 public ServerName getServerName() {
177 return ServerName.valueOf("regionserver,60020,000000");
178 }
179
180 @Override
181 public void abort(String why, Throwable e) {}
182
183 @Override
184 public boolean isAborted() {
185 return false;
186 }
187
188 @Override
189 public void stop(String why) {}
190
191 @Override
192 public boolean isStopped() {
193 return false;
194 }
195 }
196 }