View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.cleaner;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FileStatus;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.Server;
33  import org.apache.hadoop.hbase.ServerName;
34  import org.apache.hadoop.hbase.testclassification.SmallTests;
35  import org.apache.hadoop.hbase.HRegionInfo;
36  import org.apache.hadoop.hbase.CoordinatedStateManager;
37  import org.apache.hadoop.hbase.client.ClusterConnection;
38  import org.apache.hadoop.hbase.io.HFileLink;
39  import org.apache.hadoop.hbase.util.FSUtils;
40  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
41  import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
42  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
43  import org.junit.Test;
44  import org.junit.experimental.categories.Category;
45  
46  /**
47   * Test the HFileLink Cleaner.
48   * HFiles with links cannot be deleted until a link is present.
49   */
50  @Category(SmallTests.class)
51  public class TestHFileLinkCleaner {
52  
53    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
54  
55    @Test
56    public void testHFileLinkCleaning() throws Exception {
57      Configuration conf = TEST_UTIL.getConfiguration();
58      FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
59      conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName());
60      Path rootDir = FSUtils.getRootDir(conf);
61      FileSystem fs = FileSystem.get(conf);
62  
63      final TableName tableName = TableName.valueOf("test-table");
64      final TableName tableLinkName = TableName.valueOf("test-link");
65      final String hfileName = "1234567890";
66      final String familyName = "cf";
67  
68      HRegionInfo hri = new HRegionInfo(tableName);
69      HRegionInfo hriLink = new HRegionInfo(tableLinkName);
70  
71      Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
72      Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
73            tableName, hri.getEncodedName(), familyName);
74      Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
75            tableLinkName, hriLink.getEncodedName(), familyName);
76  
77      // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf);
78      Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName);
79      fs.mkdirs(familyPath);
80      Path hfilePath = new Path(familyPath, hfileName);
81      fs.createNewFile(hfilePath);
82  
83      // Create link to hfile
84      Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName,
85                                          hriLink.getEncodedName(), familyName);
86      fs.mkdirs(familyLinkPath);
87      HFileLink.create(conf, fs, familyLinkPath, hri, hfileName);
88      Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName);
89      assertTrue(fs.exists(linkBackRefDir));
90      FileStatus[] backRefs = fs.listStatus(linkBackRefDir);
91      assertEquals(1, backRefs.length);
92      Path linkBackRef = backRefs[0].getPath();
93  
94      // Initialize cleaner
95      final long ttl = 1000;
96      conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
97      Server server = new DummyServer();
98      HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir);
99  
100     // Link backref cannot be removed
101     cleaner.chore();
102     assertTrue(fs.exists(linkBackRef));
103     assertTrue(fs.exists(hfilePath));
104 
105     // Link backref can be removed
106     fs.rename(FSUtils.getTableDir(rootDir, tableLinkName),
107         FSUtils.getTableDir(archiveDir, tableLinkName));
108     cleaner.chore();
109     assertFalse("Link should be deleted", fs.exists(linkBackRef));
110 
111     // HFile can be removed
112     Thread.sleep(ttl * 2);
113     cleaner.chore();
114     assertFalse("HFile should be deleted", fs.exists(hfilePath));
115 
116     // Remove everything
117     for (int i = 0; i < 4; ++i) {
118       Thread.sleep(ttl * 2);
119       cleaner.chore();
120     }
121     assertFalse("HFile should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableName)));
122     assertFalse("Link should be deleted", fs.exists(FSUtils.getTableDir(archiveDir, tableLinkName)));
123 
124     cleaner.interrupt();
125   }
126 
127   private static Path getFamilyDirPath (final Path rootDir, final TableName table,
128     final String region, final String family) {
129     return new Path(new Path(FSUtils.getTableDir(rootDir, table), region), family);
130   }
131 
132   static class DummyServer implements Server {
133 
134     @Override
135     public Configuration getConfiguration() {
136       return TEST_UTIL.getConfiguration();
137     }
138 
139     @Override
140     public ZooKeeperWatcher getZooKeeper() {
141       try {
142         return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
143       } catch (IOException e) {
144         e.printStackTrace();
145       }
146       return null;
147     }
148 
149     @Override
150     public CoordinatedStateManager getCoordinatedStateManager() {
151       return null;
152     }
153 
154     @Override
155     public ClusterConnection getConnection() {
156       return null;
157     }
158 
159     @Override
160     public MetaTableLocator getMetaTableLocator() {
161       return null;
162     }
163 
164     @Override
165     public ServerName getServerName() {
166       return ServerName.valueOf("regionserver,60020,000000");
167     }
168 
169     @Override
170     public void abort(String why, Throwable e) {}
171 
172     @Override
173     public boolean isAborted() {
174       return false;
175     }
176 
177     @Override
178     public void stop(String why) {}
179 
180     @Override
181     public boolean isStopped() {
182       return false;
183     }
184   }
185 }