1   /**
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.master;
21  
22  import static org.junit.Assert.assertEquals;
23  
24  import org.apache.hadoop.fs.FileStatus;
25  import org.junit.After;
26  import org.junit.AfterClass;
27  import org.junit.Before;
28  import org.junit.BeforeClass;
29  import org.junit.Test;
30  
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.regionserver.HRegionServer;
34  import org.apache.hadoop.hbase.replication.ReplicationZookeeperWrapper;
35  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.conf.Configuration;
39  
40  import java.net.URLEncoder;
41  import java.util.concurrent.atomic.AtomicBoolean;
42  
43  public class TestLogsCleaner {
44  
45    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
46  
47    private ReplicationZookeeperWrapper zkHelper;
48  
49    /**
50     * @throws java.lang.Exception
51     */
52    @BeforeClass
53    public static void setUpBeforeClass() throws Exception {
54      TEST_UTIL.startMiniZKCluster();
55    }
56  
57    /**
58     * @throws java.lang.Exception
59     */
60    @AfterClass
61    public static void tearDownAfterClass() throws Exception {
62      TEST_UTIL.shutdownMiniZKCluster();
63    }
64  
65    /**
66     * @throws java.lang.Exception
67     */
68    @Before
69    public void setUp() throws Exception {
70      Configuration conf = TEST_UTIL.getConfiguration();
71      zkHelper = new ReplicationZookeeperWrapper(
72          ZooKeeperWrapper.createInstance(conf, HRegionServer.class.getName()),
73          conf, new AtomicBoolean(true), "test-cluster");
74    }
75  
76    /**
77     * @throws java.lang.Exception
78     */
79    @After
80    public void tearDown() throws Exception {
81    }
82  
83    @Test
84    public void testLogCleaning() throws Exception{
85      Configuration c = TEST_UTIL.getConfiguration();
86      Path oldLogDir = new Path(TEST_UTIL.getTestDir(),
87          HConstants.HREGION_OLDLOGDIR_NAME);
88      String fakeMachineName = URLEncoder.encode("regionserver:60020", "UTF8");
89  
90      FileSystem fs = FileSystem.get(c);
91      AtomicBoolean stop = new AtomicBoolean(false);
92      LogsCleaner cleaner = new LogsCleaner(1000, stop,c, fs, oldLogDir);
93  
94      // Create 2 invalid files, 1 "recent" file, 1 very new file and 30 old files
95      long now = System.currentTimeMillis();
96      fs.delete(oldLogDir, true);
97      fs.mkdirs(oldLogDir);
98      // Case 1: 2 invalid files, which would be deleted directly
99      fs.createNewFile(new Path(oldLogDir, "a"));
100     fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + "a"));
101     // Case 2: 1 "recent" file, not even deletable for the first log cleaner
102     // (TimeToLiveLogCleaner), so we are not going down the chain
103     fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + now));
104     System.out.println("Now is: " + now);
105     for (int i = 0; i < 30; i++) {
106       // Case 3: old files which would be deletable for the first log cleaner
107       // (TimeToLiveLogCleaner), and also for the second (ReplicationLogCleaner)
108       Path fileName = new Path(oldLogDir, fakeMachineName + "." +
109           (now - 6000000 - i) );
110       fs.createNewFile(fileName);
111       // Case 4: put 3 old log files in ZK indicating that they are scheduled
112       // for replication so these files would pass the first log cleaner
113       // (TimeToLiveLogCleaner) but would be rejected by the second
114       // (ReplicationLogCleaner)
115       if (i % (30/3) == 0) {
116         zkHelper.addLogToList(fileName.getName(), fakeMachineName);
117         System.out.println("Replication log file: " + fileName);
118       }
119     }
120     for (FileStatus stat : fs.listStatus(oldLogDir)) {
121       System.out.println(stat.getPath().toString());
122     }
123 
124     // Case 2: 1 newer file, not even deletable for the first log cleaner
125     // (TimeToLiveLogCleaner), so we are not going down the chain
126     fs.createNewFile(new Path(oldLogDir, fakeMachineName + "." + (now + 10000) ));
127 
128     assertEquals(34, fs.listStatus(oldLogDir).length);
129 
130     // This will take care of 20 old log files (default max we can delete)
131     cleaner.chore();
132 
133     assertEquals(14, fs.listStatus(oldLogDir).length);
134 
135     // We will delete all remaining log files which are not scheduled for
136     // replication and those that are invalid
137     cleaner.chore();
138 
139     // We end up with the current log file, a newer one and the 3 old log
140     // files which are scheduled for replication
141     assertEquals(5, fs.listStatus(oldLogDir).length);
142 
143     for (FileStatus file : fs.listStatus(oldLogDir)) {
144       System.out.println("Keeped log files: " + file.getPath().getName());
145     }
146   }
147 
148 }