View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.cleaner;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  import java.net.URLEncoder;
24  
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FileStatus;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.CoordinatedStateManager;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.testclassification.MediumTests;
33  import org.apache.hadoop.hbase.Server;
34  import org.apache.hadoop.hbase.ServerName;
35  import org.apache.hadoop.hbase.Waiter;
36  import org.apache.hadoop.hbase.client.ClusterConnection;
37  import org.apache.hadoop.hbase.replication.ReplicationFactory;
38  import org.apache.hadoop.hbase.replication.ReplicationQueues;
39  import org.apache.hadoop.hbase.replication.regionserver.Replication;
40  import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
41  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
42  import org.junit.AfterClass;
43  import org.junit.BeforeClass;
44  import org.junit.Test;
45  import org.junit.experimental.categories.Category;
46  
47  @Category(MediumTests.class)
48  public class TestLogsCleaner {
49  
50    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
51  
52    /**
53     * @throws java.lang.Exception
54     */
55    @BeforeClass
56    public static void setUpBeforeClass() throws Exception {
57      TEST_UTIL.startMiniZKCluster();
58    }
59  
60    /**
61     * @throws java.lang.Exception
62     */
63    @AfterClass
64    public static void tearDownAfterClass() throws Exception {
65      TEST_UTIL.shutdownMiniZKCluster();
66    }
67  
68    @Test
69    public void testLogCleaning() throws Exception{
70      Configuration conf = TEST_UTIL.getConfiguration();
71      // set TTL
72      long ttl = 10000;
73      conf.setLong("hbase.master.logcleaner.ttl", ttl);
74      conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
75      Replication.decorateMasterConfiguration(conf);
76      Server server = new DummyServer();
77      ReplicationQueues repQueues =
78          ReplicationFactory.getReplicationQueues(server.getZooKeeper(), conf, server);
79      repQueues.init(server.getServerName().toString());
80      final Path oldLogDir = new Path(TEST_UTIL.getDataTestDir(),
81          HConstants.HREGION_OLDLOGDIR_NAME);
82      String fakeMachineName =
83        URLEncoder.encode(server.getServerName().toString(), "UTF8");
84  
85      final FileSystem fs = FileSystem.get(conf);
86  
87      // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files
88      long now = System.currentTimeMillis();
89      fs.delete(oldLogDir, true);
90      fs.mkdirs(oldLogDir);
91      // Case 1: 2 invalid files, which would be deleted directly
92      fs.createNewFile(new Path(oldLogDir, "a"));
93      fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + "a"));
94      // Case 2: 1 "recent" file, not even deletable for the first log cleaner
95      // (TimeToLiveLogCleaner), so we are not going down the chain
96      System.out.println("Now is: " + now);
97      for (int i = 1; i < 31; i++) {
98        // Case 3: old files which would be deletable for the first log cleaner
99        // (TimeToLiveLogCleaner), and also for the second (ReplicationLogCleaner)
100       Path fileName = new Path(oldLogDir, fakeMachineName + "." + (now - i) );
101       fs.createNewFile(fileName);
102       // Case 4: put 3 old log files in ZK indicating that they are scheduled
103       // for replication so these files would pass the first log cleaner
104       // (TimeToLiveLogCleaner) but would be rejected by the second
105       // (ReplicationLogCleaner)
106       if (i % (30/3) == 1) {
107         repQueues.addLog(fakeMachineName, fileName.getName());
108         System.out.println("Replication log file: " + fileName);
109       }
110     }
111 
112     // sleep for sometime to get newer modifcation time
113     Thread.sleep(ttl);
114     fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + now));
115 
116     // Case 2: 1 newer file, not even deletable for the first log cleaner
117     // (TimeToLiveLogCleaner), so we are not going down the chain
118     fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now + 10000) ));
119 
120     for (FileStatus stat : fs.listStatus(oldLogDir)) {
121       System.out.println(stat.getPath().toString());
122     }
123 
124     assertEquals(34, fs.listStatus(oldLogDir).length);
125 
126     LogCleaner cleaner  = new LogCleaner(1000, server, conf, fs, oldLogDir);
127     cleaner.chore();
128 
129     // We end up with the current log file, a newer one and the 3 old log
130     // files which are scheduled for replication
131     TEST_UTIL.waitFor(1000, new Waiter.Predicate<Exception>() {
132       @Override
133       public boolean evaluate() throws Exception {
134         return 5 == fs.listStatus(oldLogDir).length;
135       }
136     });
137 
138     for (FileStatus file : fs.listStatus(oldLogDir)) {
139       System.out.println("Kept log files: " + file.getPath().getName());
140     }
141   }
142 
143   static class DummyServer implements Server {
144 
145     @Override
146     public Configuration getConfiguration() {
147       return TEST_UTIL.getConfiguration();
148     }
149 
150     @Override
151     public ZooKeeperWatcher getZooKeeper() {
152       try {
153         return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
154       } catch (IOException e) {
155         e.printStackTrace();
156       }
157       return null;
158     }
159 
160     @Override
161     public CoordinatedStateManager getCoordinatedStateManager() {
162       return null;
163     }
164 
165     @Override
166     public ClusterConnection getConnection() {
167       return null;
168     }
169 
170     @Override
171     public MetaTableLocator getMetaTableLocator() {
172       return null;
173     }
174 
175     @Override
176     public ServerName getServerName() {
177       return ServerName.valueOf("regionserver,60020,000000");
178     }
179 
180     @Override
181     public void abort(String why, Throwable e) {}
182 
183     @Override
184     public boolean isAborted() {
185       return false;
186     }
187 
188     @Override
189     public void stop(String why) {}
190 
191     @Override
192     public boolean isStopped() {
193       return false;
194     }
195   }
196 }