1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.cleaner;
19
20 import static org.junit.Assert.assertEquals;
21
22 import java.io.IOException;
23 import java.lang.reflect.Field;
24 import java.net.URLEncoder;
25 import java.util.LinkedList;
26
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.fs.FileStatus;
29 import org.apache.hadoop.fs.FileSystem;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.*;
32 import org.apache.hadoop.hbase.catalog.CatalogTracker;
33 import org.apache.hadoop.hbase.replication.ReplicationFactory;
34 import org.apache.hadoop.hbase.replication.ReplicationQueues;
35 import org.apache.hadoop.hbase.replication.ReplicationQueuesClient;
36 import org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner;
37 import org.apache.hadoop.hbase.replication.regionserver.Replication;
38 import org.apache.hadoop.hbase.testclassification.MediumTests;
39 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
40 import org.junit.AfterClass;
41 import org.junit.BeforeClass;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44 import org.mockito.Mockito;
45
46 @Category(MediumTests.class)
47 public class TestLogsCleaner {
48
49 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
50
51
52
53
54 @BeforeClass
55 public static void setUpBeforeClass() throws Exception {
56 TEST_UTIL.startMiniZKCluster();
57 }
58
59
60
61
62 @AfterClass
63 public static void tearDownAfterClass() throws Exception {
64 TEST_UTIL.shutdownMiniZKCluster();
65 }
66
67 @Test
68 public void testLogCleaning() throws Exception{
69 Configuration conf = TEST_UTIL.getConfiguration();
70
71 long ttl = 10000;
72 conf.setLong("hbase.master.logcleaner.ttl", ttl);
73 conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
74 Replication.decorateMasterConfiguration(conf);
75 Server server = new DummyServer();
76 ReplicationQueues repQueues =
77 ReplicationFactory.getReplicationQueues(server.getZooKeeper(), conf, server);
78 repQueues.init(server.getServerName().toString());
79 final Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(),
80 HConstants.HREGION_OLDLOGDIR_NAME);
81 String fakeMachineName =
82 URLEncoder.encode(server.getServerName().toString(), "UTF8");
83
84 final FileSystem fs = FileSystem.get(conf);
85
86
87 long now = System.currentTimeMillis();
88 fs.delete(oldLogDir, true);
89 fs.mkdirs(oldLogDir);
90
91 fs.createNewFile(new Path(oldLogDir, "a"));
92 fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + "a"));
93
94
95 System.out.println("Now is: " + now);
96 for (int i = 1; i < 31; i++) {
97
98
99 Path fileName = new Path(oldLogDir, fakeMachineName + "." + (now - i) );
100 fs.createNewFile(fileName);
101
102
103
104
105 if (i % (30/3) == 1) {
106 repQueues.addLog(fakeMachineName, fileName.getName());
107 System.out.println("Replication log file: " + fileName);
108 }
109 }
110
111
112 Thread.sleep(ttl);
113 fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + now));
114
115
116
117 fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now + 10000) ));
118
119 for (FileStatus stat : fs.listStatus(oldLogDir)) {
120 System.out.println(stat.getPath().toString());
121 }
122
123 assertEquals(34, fs.listStatus(oldLogDir).length);
124
125 LogCleaner cleaner = new LogCleaner(1000, server, conf, fs, oldLogDir);
126
127 cleaner.chore();
128
129
130
131 TEST_UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
132 @Override
133 public boolean evaluate() throws Exception {
134 return 5 == fs.listStatus(oldLogDir).length;
135 }
136 });
137
138 for (FileStatus file : fs.listStatus(oldLogDir)) {
139 System.out.println("Kept log files: " + file.getPath().getName());
140 }
141 }
142
143 @Test(timeout=5000)
144 public void testZnodeCversionChange() throws Exception {
145 Configuration conf = TEST_UTIL.getConfiguration();
146 ReplicationLogCleaner cleaner = new ReplicationLogCleaner();
147 cleaner.setConf(conf);
148
149 ReplicationQueuesClient rqcMock = Mockito.mock(ReplicationQueuesClient.class);
150 Mockito.when(rqcMock.getQueuesZNodeCversion()).thenReturn(1, 2, 3, 4);
151
152 Field rqc = ReplicationLogCleaner.class.getDeclaredField("replicationQueues");
153 rqc.setAccessible(true);
154
155 rqc.set(cleaner, rqcMock);
156
157
158 cleaner.getDeletableFiles(new LinkedList<FileStatus>());
159 }
160
161 static class DummyServer implements Server {
162
163 @Override
164 public Configuration getConfiguration() {
165 return TEST_UTIL.getConfiguration();
166 }
167
168 @Override
169 public ZooKeeperWatcher getZooKeeper() {
170 try {
171 return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
172 } catch (IOException e) {
173 e.printStackTrace();
174 }
175 return null;
176 }
177
178 @Override
179 public CatalogTracker getCatalogTracker() {
180 return null;
181 }
182
183 @Override
184 public ServerName getServerName() {
185 return ServerName.valueOf("regionserver,60020,000000");
186 }
187
188 @Override
189 public void abort(String why, Throwable e) {}
190
191 @Override
192 public boolean isAborted() {
193 return false;
194 }
195
196 @Override
197 public void stop(String why) {}
198
199 @Override
200 public boolean isStopped() {
201 return false;
202 }
203 }
204
205 }
206