1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.cleaner;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FileStatus;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.Server;
33  import org.apache.hadoop.hbase.ServerName;
34  import org.apache.hadoop.hbase.SmallTests;
35  import org.apache.hadoop.hbase.backup.HFileArchiver;
36  import org.apache.hadoop.hbase.catalog.CatalogTracker;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.io.HFileLink;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hbase.util.FSUtils;
41  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
42  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
43  import org.junit.Test;
44  import org.junit.experimental.categories.Category;
45  
46  /**
47   * Test the HFileLink Cleaner.
48   * HFiles with links cannot be deleted until a link is present.
49   */
50  @Category(SmallTests.class)
51  public class TestHFileLinkCleaner {
52  
53    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
54  
55    @Test
56    public void testHFileLinkCleaning() throws Exception {
57      Configuration conf = TEST_UTIL.getConfiguration();
58      conf.set(HConstants.HBASE_DIR, TEST_UTIL.getDataTestDir().toString());
59      conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, HFileLinkCleaner.class.getName());
60      Path rootDir = FSUtils.getRootDir(conf);
61      FileSystem fs = FileSystem.get(conf);
62  
63      final String tableName = "test-table";
64      final String tableLinkName = "test-link";
65      final String hfileName = "1234567890";
66      final String familyName = "cf";
67  
68      HRegionInfo hri = new HRegionInfo(Bytes.toBytes(tableName));
69      HRegionInfo hriLink = new HRegionInfo(Bytes.toBytes(tableLinkName));
70  
71      Path archiveDir = HFileArchiveUtil.getArchivePath(conf);
72      Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
73            tableName, hri.getEncodedName(), familyName);
74      Path archiveLinkStoreDir = HFileArchiveUtil.getStoreArchivePath(conf,
75            tableLinkName, hriLink.getEncodedName(), familyName);
76  
77      // Create hfile /hbase/table-link/region/cf/getEncodedName.HFILE(conf);
78      Path familyPath = getFamilyDirPath(archiveDir, tableName, hri.getEncodedName(), familyName);
79      fs.mkdirs(familyPath);
80      Path hfilePath = new Path(familyPath, hfileName);
81      fs.createNewFile(hfilePath);
82  
83      // Create link to hfile
84      Path familyLinkPath = getFamilyDirPath(rootDir, tableLinkName,
85                                          hriLink.getEncodedName(), familyName);
86      fs.mkdirs(familyLinkPath);
87      HFileLink.create(conf, fs, familyLinkPath, hri, hfileName);
88      Path linkBackRefDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName);
89      assertTrue(fs.exists(linkBackRefDir));
90      FileStatus[] backRefs = fs.listStatus(linkBackRefDir);
91      assertEquals(1, backRefs.length);
92      Path linkBackRef = backRefs[0].getPath();
93  
94      // Initialize cleaner
95      final long ttl = 1000;
96      conf.setLong(TimeToLiveHFileCleaner.TTL_CONF_KEY, ttl);
97      Server server = new DummyServer();
98      HFileCleaner cleaner = new HFileCleaner(1000, server, conf, fs, archiveDir);
99  
100     // Link backref cannot be removed
101     cleaner.chore();
102     assertTrue(fs.exists(linkBackRef));
103     assertTrue(fs.exists(hfilePath));
104 
105     // Link backref can be removed
106     fs.rename(new Path(rootDir, tableLinkName), new Path(archiveDir, tableLinkName));
107     cleaner.chore();
108     assertFalse("Link should be deleted", fs.exists(linkBackRef));
109 
110     // HFile can be removed
111     Thread.sleep(ttl * 2);
112     cleaner.chore();
113     assertFalse("HFile should be deleted", fs.exists(hfilePath));
114 
115     // Remove everything
116     for (int i = 0; i < 4; ++i) {
117       Thread.sleep(ttl * 2);
118       cleaner.chore();
119     }
120     assertFalse("HFile should be deleted", fs.exists(new Path(archiveDir, tableName)));
121     assertFalse("Link should be deleted", fs.exists(new Path(archiveDir, tableLinkName)));
122 
123     cleaner.interrupt();
124   }
125 
126   private static Path getFamilyDirPath (final Path rootDir, final String table,
127     final String region, final String family) {
128     return new Path(new Path(new Path(rootDir, table), region), family);
129   }
130 
131   static class DummyServer implements Server {
132 
133     @Override
134     public Configuration getConfiguration() {
135       return TEST_UTIL.getConfiguration();
136     }
137 
138     @Override
139     public ZooKeeperWatcher getZooKeeper() {
140       try {
141         return new ZooKeeperWatcher(getConfiguration(), "dummy server", this);
142       } catch (IOException e) {
143         e.printStackTrace();
144       }
145       return null;
146     }
147 
148     @Override
149     public CatalogTracker getCatalogTracker() {
150       return null;
151     }
152 
153     @Override
154     public ServerName getServerName() {
155       return new ServerName("regionserver,60020,000000");
156     }
157 
158     @Override
159     public void abort(String why, Throwable e) {}
160 
161     @Override
162     public boolean isAborted() {
163       return false;
164     }
165 
166     @Override
167     public void stop(String why) {}
168 
169     @Override
170     public boolean isStopped() {
171       return false;
172     }
173   }
174 }