1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.backup;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.MediumTests;
39  import org.apache.hadoop.hbase.Stoppable;
40  import org.apache.hadoop.hbase.client.HBaseAdmin;
41  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
42  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.hadoop.hbase.regionserver.HRegionServer;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.FSUtils;
47  import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil;
48  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
49  import org.apache.hadoop.hbase.util.StoppableImplementation;
50  import org.junit.After;
51  import org.junit.AfterClass;
52  import org.junit.Assert;
53  import org.junit.BeforeClass;
54  import org.junit.Test;
55  import org.junit.experimental.categories.Category;
56  
57  /**
58   * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up
59   * a region
60   */
61  @Category(MediumTests.class)
62  public class TestHFileArchiving {
63  
64    private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class);
65    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
66    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
67  
68    /**
69     * Setup the config for the cluster
70     */
71    @BeforeClass
72    public static void setupCluster() throws Exception {
73      setupConf(UTIL.getConfiguration());
74      UTIL.startMiniCluster();
75  
76      // We don't want the cleaner to remove files. The tests do that.
77      UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().interrupt();
78    }
79  
80    private static void setupConf(Configuration conf) {
81      // disable the ui
82      conf.setInt("hbase.regionsever.info.port", -1);
83      // drop the memstore size so we get flushes
84      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
85      // disable major compactions
86      conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);
87  
88      // prevent aggressive region split
89      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
90        ConstantSizeRegionSplitPolicy.class.getName());
91    }
92  
93    @After
94    public void tearDown() throws Exception {
95      // cleanup the archive directory
96      try {
97        clearArchiveDirectory();
98      } catch (IOException e) {
99        Assert.fail("Failure to delete archive directory:" + e.getMessage());
100     }
101   }
102 
103   @AfterClass
104   public static void cleanupTest() throws Exception {
105     try {
106       UTIL.shutdownMiniCluster();
107     } catch (Exception e) {
108       // NOOP;
109     }
110   }
111 
112   @Test
113   public void testRemovesRegionDirOnArchive() throws Exception {
114     byte[] TABLE_NAME = Bytes.toBytes("testRemovesRegionDirOnArchive");
115     UTIL.createTable(TABLE_NAME, TEST_FAM);
116 
117     final HBaseAdmin admin = UTIL.getHBaseAdmin();
118 
119     // get the current store files for the region
120     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
121     // make sure we only have 1 region serving this table
122     assertEquals(1, servingRegions.size());
123     HRegion region = servingRegions.get(0);
124 
125     // and load the table
126     UTIL.loadRegion(region, TEST_FAM);
127 
128     // shutdown the table so we can manipulate the files
129     admin.disableTable(TABLE_NAME);
130 
131     FileSystem fs = UTIL.getTestFileSystem();
132 
133     // now attempt to depose the region
134     Path regionDir = HRegion.getRegionDir(region.getTableDir().getParent(), region.getRegionInfo());
135 
136     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
137 
138     // check for the existence of the archive directory and some files in it
139     Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
140     assertTrue(fs.exists(archiveDir));
141 
142     // check to make sure the store directory was copied
143     FileStatus[] stores = fs.listStatus(archiveDir);
144     assertTrue(stores.length == 1);
145 
146     // make sure we archived the store files
147     FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
148     assertTrue(storeFiles.length > 0);
149 
150     // then ensure the region's directory isn't present
151     assertFalse(fs.exists(regionDir));
152 
153     UTIL.deleteTable(TABLE_NAME);
154   }
155 
156   /**
157    * Test that the region directory is removed when we archive a region without store files, but
158    * still has hidden files.
159    * @throws Exception
160    */
161   @Test
162   public void testDeleteRegionWithNoStoreFiles() throws Exception {
163     byte[] TABLE_NAME = Bytes.toBytes("testDeleteRegionWithNoStoreFiles");
164     UTIL.createTable(TABLE_NAME, TEST_FAM);
165 
166     // get the current store files for the region
167     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
168     // make sure we only have 1 region serving this table
169     assertEquals(1, servingRegions.size());
170     HRegion region = servingRegions.get(0);
171 
172     FileSystem fs = region.getFilesystem();
173 
174     // make sure there are some files in the regiondir
175     Path rootDir = FSUtils.getRootDir(fs.getConf());
176     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
177     FileStatus[] regionFiles = FSUtils.listStatus(fs, regionDir, null);
178     Assert.assertNotNull("No files in the region directory", regionFiles);
179     if (LOG.isDebugEnabled()) {
180       List<Path> files = new ArrayList<Path>();
181       for (FileStatus file : regionFiles) {
182         files.add(file.getPath());
183       }
184       LOG.debug("Current files:" + files);
185     }
186     // delete the visible folders so we just have hidden files/folders
187     final PathFilter dirFilter = new FSUtils.DirFilter(fs);
188     PathFilter nonHidden = new PathFilter() {
189       @Override
190       public boolean accept(Path file) {
191         return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
192       }
193     };
194     FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
195     for (FileStatus store : storeDirs) {
196       LOG.debug("Deleting store for test");
197       fs.delete(store.getPath(), true);
198     }
199 
200     // then archive the region
201     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
202 
203     // and check to make sure the region directoy got deleted
204     assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
205 
206     UTIL.deleteTable(TABLE_NAME);
207   }
208 
209   @Test
210   public void testArchiveOnTableDelete() throws Exception {
211     byte[] TABLE_NAME = Bytes.toBytes("testArchiveOnTableDelete");
212     UTIL.createTable(TABLE_NAME, TEST_FAM);
213 
214     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
215     // make sure we only have 1 region serving this table
216     assertEquals(1, servingRegions.size());
217     HRegion region = servingRegions.get(0);
218 
219     // get the parent RS and monitor
220     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
221     FileSystem fs = hrs.getFileSystem();
222 
223     // put some data on the region
224     LOG.debug("-------Loading table");
225     UTIL.loadRegion(region, TEST_FAM);
226 
227     // get the hfiles in the region
228     List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
229     assertEquals("More that 1 region for test table.", 1, regions.size());
230 
231     region = regions.get(0);
232     // wait for all the compactions to complete
233     region.waitForFlushesAndCompactions();
234 
235     // disable table to prevent new updates
236     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
237     LOG.debug("Disabled table");
238 
239     // remove all the files from the archive to get a fair comparison
240     clearArchiveDirectory();
241 
242     // then get the current store files
243     Path regionDir = region.getRegionDir();
244     List<String> storeFiles = getRegionStoreFiles(fs, regionDir);
245 
246     // then delete the table so the hfiles get archived
247     UTIL.deleteTable(TABLE_NAME);
248 
249     assertArchiveFiles(fs, storeFiles, 30000);
250   }
251 
252   private void assertArchiveFiles(FileSystem fs, List<String> storeFiles, long timeout) throws IOException {
253     long end = System.currentTimeMillis() + timeout;
254     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
255     List<String> archivedFiles = new ArrayList<String>();
256 
257     // We have to ensure that the DeleteTableHandler is finished. HBaseAdmin.deleteXXX() can return before all files
258     // are archived. We should fix HBASE-5487 and fix synchronous operations from admin.
259     while (System.currentTimeMillis() < end) {
260       archivedFiles = getAllFileNames(fs, archiveDir);
261       if (archivedFiles.size() >= storeFiles.size()) {
262         break;
263       }
264     }
265 
266     Collections.sort(storeFiles);
267     Collections.sort(archivedFiles);
268 
269     LOG.debug("Store files:");
270     for (int i = 0; i < storeFiles.size(); i++) {
271       LOG.debug(i + " - " + storeFiles.get(i));
272     }
273     LOG.debug("Archive files:");
274     for (int i = 0; i < archivedFiles.size(); i++) {
275       LOG.debug(i + " - " + archivedFiles.get(i));
276     }
277 
278     assertTrue("Archived files are missing some of the store files!",
279       archivedFiles.containsAll(storeFiles));
280   }
281 
282   /**
283    * Test that the store files are archived when a column family is removed.
284    * @throws Exception
285    */
286   @Test
287   public void testArchiveOnTableFamilyDelete() throws Exception {
288     byte[] TABLE_NAME = Bytes.toBytes("testArchiveOnTableFamilyDelete");
289     UTIL.createTable(TABLE_NAME, TEST_FAM);
290 
291     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
292     // make sure we only have 1 region serving this table
293     assertEquals(1, servingRegions.size());
294     HRegion region = servingRegions.get(0);
295 
296     // get the parent RS and monitor
297     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
298     FileSystem fs = hrs.getFileSystem();
299 
300     // put some data on the region
301     LOG.debug("-------Loading table");
302     UTIL.loadRegion(region, TEST_FAM);
303 
304     // get the hfiles in the region
305     List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
306     assertEquals("More that 1 region for test table.", 1, regions.size());
307 
308     region = regions.get(0);
309     // wait for all the compactions to complete
310     region.waitForFlushesAndCompactions();
311 
312     // disable table to prevent new updates
313     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
314     LOG.debug("Disabled table");
315 
316     // remove all the files from the archive to get a fair comparison
317     clearArchiveDirectory();
318 
319     // then get the current store files
320     Path regionDir = region.getRegionDir();
321     List<String> storeFiles = getRegionStoreFiles(fs, regionDir);
322 
323     // then delete the table so the hfiles get archived
324     UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM);
325 
326     assertArchiveFiles(fs, storeFiles, 30000);
327 
328     UTIL.deleteTable(TABLE_NAME);
329   }
330 
331   /**
332    * Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
333    */
334   @Test
335   public void testCleaningRace() throws Exception {
336     final long TEST_TIME = 20 * 1000;
337 
338     Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
339     Path rootDir = UTIL.getDataTestDir("testCleaningRace");
340     FileSystem fs = UTIL.getTestFileSystem();
341 
342     Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
343     Path regionDir = new Path("table", "abcdef");
344     Path familyDir = new Path(regionDir, "cf");
345 
346     Path sourceRegionDir = new Path(rootDir, regionDir);
347     fs.mkdirs(sourceRegionDir);
348 
349     Stoppable stoppable = new StoppableImplementation();
350 
351     // The cleaner should be looping without long pauses to reproduce the race condition.
352     HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir);
353     try {
354       cleaner.start();
355 
356       // Keep creating/archiving new files while the cleaner is running in the other thread
357       long startTime = System.currentTimeMillis();
358       for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) {
359         Path file = new Path(familyDir,  String.valueOf(fid));
360         Path sourceFile = new Path(rootDir, file);
361         Path archiveFile = new Path(archiveDir, file);
362 
363         fs.createNewFile(sourceFile);
364 
365         try {
366           // Try to archive the file
367           HFileArchiver.archiveRegion(fs, rootDir,
368               sourceRegionDir.getParent(), sourceRegionDir);
369 
370           // The archiver succeded, the file is no longer in the original location
371           // but it's in the archive location.
372           LOG.debug("hfile=" + fid + " should be in the archive");
373           assertTrue(fs.exists(archiveFile));
374           assertFalse(fs.exists(sourceFile));
375         } catch (IOException e) {
376           // The archiver is unable to archive the file. Probably HBASE-7643 race condition.
377           // in this case, the file should not be archived, and we should have the file
378           // in the original location.
379           LOG.debug("hfile=" + fid + " should be in the source location");
380           assertFalse(fs.exists(archiveFile));
381           assertTrue(fs.exists(sourceFile));
382 
383           // Avoid to have this file in the next run
384           fs.delete(sourceFile, false);
385         }
386       }
387     } finally {
388       stoppable.stop("test end");
389       cleaner.join();
390       fs.delete(rootDir, true);
391     }
392   }
393 
394   private void clearArchiveDirectory() throws IOException {
395     UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true);
396   }
397 
398   /**
399    * Get the names of all the files below the given directory
400    * @param fs
401    * @param archiveDir
402    * @return
403    * @throws IOException
404    */
405   private List<String> getAllFileNames(final FileSystem fs, Path archiveDir) throws IOException {
406     FileStatus[] files = FSUtils.listStatus(fs, archiveDir, null);
407     return recurseOnFiles(fs, files, new ArrayList<String>());
408   }
409 
410   /** Recursively lookup all the file names under the file[] array **/
411   private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<String> fileNames)
412       throws IOException {
413     if (files == null || files.length == 0) return fileNames;
414 
415     for (FileStatus file : files) {
416       if (file.isDir()) {
417         recurseOnFiles(fs, FSUtils.listStatus(fs, file.getPath(), null), fileNames);
418       } else fileNames.add(file.getPath().getName());
419     }
420     return fileNames;
421   }
422 
423   private List<String> getRegionStoreFiles(final FileSystem fs, final Path regionDir)
424       throws IOException {
425     List<String> storeFiles = getAllFileNames(fs, regionDir);
426     // remove all the non-storefile named files for the region
427     for (int i = 0; i < storeFiles.size(); i++) {
428       String file = storeFiles.get(i);
429       if (file.contains(HRegion.REGIONINFO_FILE) || file.contains("hlog")) {
430         storeFiles.remove(i--);
431       }
432     }
433     storeFiles.remove(HRegion.REGIONINFO_FILE);
434     return storeFiles;
435   }
436 }