View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.backup;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.MediumTests;
40  import org.apache.hadoop.hbase.Stoppable;
41  import org.apache.hadoop.hbase.client.HBaseAdmin;
42  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
43  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
44  import org.apache.hadoop.hbase.regionserver.HRegion;
45  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
46  import org.apache.hadoop.hbase.regionserver.HRegionServer;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.FSUtils;
49  import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil;
50  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
51  import org.apache.hadoop.hbase.util.StoppableImplementation;
52  import org.junit.After;
53  import org.junit.AfterClass;
54  import org.junit.Assert;
55  import org.junit.Before;
56  import org.junit.BeforeClass;
57  import org.junit.Test;
58  import org.junit.experimental.categories.Category;
59  
60  /**
61   * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up
62   * a region
63   */
64  @Category(MediumTests.class)
65  public class TestHFileArchiving {
66  
67    private static final String STRING_TABLE_NAME = "test_table";
68  
69    private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class);
70    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
71    private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
72    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
73  
74    /**
75     * Setup the config for the cluster
76     */
77    @BeforeClass
78    public static void setupCluster() throws Exception {
79      setupConf(UTIL.getConfiguration());
80      UTIL.startMiniCluster();
81  
82      // We don't want the cleaner to remove files. The tests do that.
83      UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().interrupt();
84    }
85  
86    private static void setupConf(Configuration conf) {
87      // disable the ui
88      conf.setInt("hbase.regionsever.info.port", -1);
89      // drop the memstore size so we get flushes
90      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
91      // disable major compactions
92      conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);
93  
94      // prevent aggressive region split
95      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
96        ConstantSizeRegionSplitPolicy.class.getName());
97    }
98  
99    @Before
100   public void setup() throws Exception {
101     UTIL.createTable(TABLE_NAME, TEST_FAM);
102   }
103 
104   @After
105   public void tearDown() throws Exception {
106     // cleanup the cluster if its up still
107     if (UTIL.getHBaseAdmin().tableExists(STRING_TABLE_NAME)) {
108       UTIL.deleteTable(TABLE_NAME);
109     }
110     // and cleanup the archive directory
111     try {
112       clearArchiveDirectory();
113     } catch (IOException e) {
114       Assert.fail("Failure to delete archive directory:" + e.getMessage());
115     }
116   }
117 
118   @AfterClass
119   public static void cleanupTest() throws Exception {
120     try {
121       UTIL.shutdownMiniCluster();
122     } catch (Exception e) {
123       // NOOP;
124     }
125   }
126 
127   @Test
128   public void testRemovesRegionDirOnArchive() throws Exception {
129     final HBaseAdmin admin = UTIL.getHBaseAdmin();
130 
131     // get the current store files for the region
132     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
133     // make sure we only have 1 region serving this table
134     assertEquals(1, servingRegions.size());
135     HRegion region = servingRegions.get(0);
136 
137     // and load the table
138     UTIL.loadRegion(region, TEST_FAM);
139 
140     // shutdown the table so we can manipulate the files
141     admin.disableTable(STRING_TABLE_NAME);
142 
143     FileSystem fs = UTIL.getTestFileSystem();
144 
145     // now attempt to depose the region
146     Path rootDir = region.getRegionFileSystem().getTableDir().getParent();
147     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
148 
149     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
150 
151     // check for the existence of the archive directory and some files in it
152     Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
153     assertTrue(fs.exists(archiveDir));
154 
155     // check to make sure the store directory was copied
156     FileStatus[] stores = fs.listStatus(archiveDir);
157     assertTrue(stores.length == 1);
158 
159     // make sure we archived the store files
160     FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
161     assertTrue(storeFiles.length > 0);
162 
163     // then ensure the region's directory isn't present
164     assertFalse(fs.exists(regionDir));
165   }
166 
167   /**
168    * Test that the region directory is removed when we archive a region without store files, but
169    * still has hidden files.
170    * @throws Exception
171    */
172   @Test
173   public void testDeleteRegionWithNoStoreFiles() throws Exception {
174     // get the current store files for the region
175     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
176     // make sure we only have 1 region serving this table
177     assertEquals(1, servingRegions.size());
178     HRegion region = servingRegions.get(0);
179 
180     FileSystem fs = region.getRegionFileSystem().getFileSystem();
181 
182     // make sure there are some files in the regiondir
183     Path rootDir = FSUtils.getRootDir(fs.getConf());
184     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
185     FileStatus[] regionFiles = FSUtils.listStatus(fs, regionDir, null);
186     Assert.assertNotNull("No files in the region directory", regionFiles);
187     if (LOG.isDebugEnabled()) {
188       List<Path> files = new ArrayList<Path>();
189       for (FileStatus file : regionFiles) {
190         files.add(file.getPath());
191       }
192       LOG.debug("Current files:" + files);
193     }
194     // delete the visible folders so we just have hidden files/folders
195     final PathFilter dirFilter = new FSUtils.DirFilter(fs);
196     PathFilter nonHidden = new PathFilter() {
197       @Override
198       public boolean accept(Path file) {
199         return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
200       }
201     };
202     FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
203     for (FileStatus store : storeDirs) {
204       LOG.debug("Deleting store for test");
205       fs.delete(store.getPath(), true);
206     }
207 
208     // then archive the region
209     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
210 
211     // and check to make sure the region directoy got deleted
212     assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
213   }
214 
215   @Test
216   public void testArchiveOnTableDelete() throws Exception {
217     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TableName.valueOf(TABLE_NAME));
218     // make sure we only have 1 region serving this table
219     assertEquals(1, servingRegions.size());
220     HRegion region = servingRegions.get(0);
221 
222     // get the parent RS and monitor
223     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
224     FileSystem fs = hrs.getFileSystem();
225 
226     // put some data on the region
227     LOG.debug("-------Loading table");
228     UTIL.loadRegion(region, TEST_FAM);
229 
230     // get the hfiles in the region
231     List<HRegion> regions = hrs.getOnlineRegions(TableName.valueOf(TABLE_NAME));
232     assertEquals("More that 1 region for test table.", 1, regions.size());
233 
234     region = regions.get(0);
235     // wait for all the compactions to complete
236     region.waitForFlushesAndCompactions();
237 
238     // disable table to prevent new updates
239     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
240     LOG.debug("Disabled table");
241 
242     // remove all the files from the archive to get a fair comparison
243     clearArchiveDirectory();
244 
245     // then get the current store files
246     List<String> storeFiles = getRegionStoreFiles(region);
247 
248     // then delete the table so the hfiles get archived
249     UTIL.deleteTable(TABLE_NAME);
250     LOG.debug("Deleted table");
251 
252     assertArchiveFiles(fs, storeFiles, 30000);
253   }
254 
255   private void assertArchiveFiles(FileSystem fs, List<String> storeFiles, long timeout) throws IOException {
256     long end = System.currentTimeMillis() + timeout;
257     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
258     List<String> archivedFiles = new ArrayList<String>();
259 
260     // We have to ensure that the DeleteTableHandler is finished. HBaseAdmin.deleteXXX() can return before all files
261     // are archived. We should fix HBASE-5487 and fix synchronous operations from admin.
262     while (System.currentTimeMillis() < end) {
263       archivedFiles = getAllFileNames(fs, archiveDir);
264       if (archivedFiles.size() >= storeFiles.size()) {
265         break;
266       }
267     }
268 
269     Collections.sort(storeFiles);
270     Collections.sort(archivedFiles);
271 
272     LOG.debug("Store files:");
273     for (int i = 0; i < storeFiles.size(); i++) {
274       LOG.debug(i + " - " + storeFiles.get(i));
275     }
276     LOG.debug("Archive files:");
277     for (int i = 0; i < archivedFiles.size(); i++) {
278       LOG.debug(i + " - " + archivedFiles.get(i));
279     }
280 
281     assertTrue("Archived files are missing some of the store files!",
282       archivedFiles.containsAll(storeFiles));
283   }
284 
285 
286   /**
287    * Test that the store files are archived when a column family is removed.
288    * @throws Exception
289    */
290   @Test
291   public void testArchiveOnTableFamilyDelete() throws Exception {
292     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
293     // make sure we only have 1 region serving this table
294     assertEquals(1, servingRegions.size());
295     HRegion region = servingRegions.get(0);
296 
297     // get the parent RS and monitor
298     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
299     FileSystem fs = hrs.getFileSystem();
300 
301     // put some data on the region
302     LOG.debug("-------Loading table");
303     UTIL.loadRegion(region, TEST_FAM);
304 
305     // get the hfiles in the region
306     List<HRegion> regions = hrs.getOnlineRegions(TableName.valueOf(TABLE_NAME));
307     assertEquals("More that 1 region for test table.", 1, regions.size());
308 
309     region = regions.get(0);
310     // wait for all the compactions to complete
311     region.waitForFlushesAndCompactions();
312 
313     // disable table to prevent new updates
314     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
315     LOG.debug("Disabled table");
316 
317     // remove all the files from the archive to get a fair comparison
318     clearArchiveDirectory();
319 
320     // then get the current store files
321     List<String> storeFiles = getRegionStoreFiles(region);
322 
323     // then delete the table so the hfiles get archived
324     UTIL.getHBaseAdmin().deleteColumn(TableName.valueOf(TABLE_NAME), TEST_FAM);
325 
326     assertArchiveFiles(fs, storeFiles, 30000);
327     UTIL.deleteTable(TABLE_NAME);
328   }
329 
330   /**
331    * Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
332    */
333   @Test
334   public void testCleaningRace() throws Exception {
335     final long TEST_TIME = 20 * 1000;
336 
337     Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
338     Path rootDir = UTIL.getDataTestDir("testCleaningRace");
339     FileSystem fs = UTIL.getTestFileSystem();
340 
341     Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
342     Path regionDir = new Path(FSUtils.getTableDir(new Path("./"),
343         TableName.valueOf("table")), "abcdef");
344     Path familyDir = new Path(regionDir, "cf");
345 
346     Path sourceRegionDir = new Path(rootDir, regionDir);
347     fs.mkdirs(sourceRegionDir);
348 
349     Stoppable stoppable = new StoppableImplementation();
350 
351     // The cleaner should be looping without long pauses to reproduce the race condition.
352     HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir);
353     try {
354       cleaner.start();
355 
356       // Keep creating/archiving new files while the cleaner is running in the other thread
357       long startTime = System.currentTimeMillis();
358       for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) {
359         Path file = new Path(familyDir,  String.valueOf(fid));
360         Path sourceFile = new Path(rootDir, file);
361         Path archiveFile = new Path(archiveDir, file);
362 
363         fs.createNewFile(sourceFile);
364 
365         try {
366           // Try to archive the file
367           HFileArchiver.archiveRegion(fs, rootDir,
368               sourceRegionDir.getParent(), sourceRegionDir);
369 
370           // The archiver succeded, the file is no longer in the original location
371           // but it's in the archive location.
372           LOG.debug("hfile=" + fid + " should be in the archive");
373           assertTrue(fs.exists(archiveFile));
374           assertFalse(fs.exists(sourceFile));
375         } catch (IOException e) {
376           // The archiver is unable to archive the file. Probably HBASE-7643 race condition.
377           // in this case, the file should not be archived, and we should have the file
378           // in the original location.
379           LOG.debug("hfile=" + fid + " should be in the source location");
380           assertFalse(fs.exists(archiveFile));
381           assertTrue(fs.exists(sourceFile));
382 
383           // Avoid to have this file in the next run
384           fs.delete(sourceFile, false);
385         }
386       }
387     } finally {
388       stoppable.stop("test end");
389       cleaner.join();
390       fs.delete(rootDir, true);
391     }
392   }
393 
394   private void clearArchiveDirectory() throws IOException {
395     UTIL.getTestFileSystem().delete(
396       new Path(UTIL.getDefaultRootDirPath(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
397   }
398 
399   /**
400    * Get the names of all the files below the given directory
401    * @param fs
402    * @param archiveDir
403    * @return
404    * @throws IOException
405    */
406   private List<String> getAllFileNames(final FileSystem fs, Path archiveDir) throws IOException {
407     FileStatus[] files = FSUtils.listStatus(fs, archiveDir, null);
408     return recurseOnFiles(fs, files, new ArrayList<String>());
409   }
410 
411   /** Recursively lookup all the file names under the file[] array **/
412   private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<String> fileNames)
413       throws IOException {
414     if (files == null || files.length == 0) return fileNames;
415 
416     for (FileStatus file : files) {
417       if (file.isDir()) {
418         recurseOnFiles(fs, FSUtils.listStatus(fs, file.getPath(), null), fileNames);
419       } else fileNames.add(file.getPath().getName());
420     }
421     return fileNames;
422   }
423 
424   private List<String> getRegionStoreFiles(final HRegion region) throws IOException {
425     Path regionDir = region.getRegionFileSystem().getRegionDir();
426     FileSystem fs = region.getRegionFileSystem().getFileSystem();
427     List<String> storeFiles = getAllFileNames(fs, regionDir);
428     // remove all the non-storefile named files for the region
429     for (int i = 0; i < storeFiles.size(); i++) {
430       String file = storeFiles.get(i);
431       if (file.contains(HRegionFileSystem.REGION_INFO_FILE) || file.contains("hlog")) {
432         storeFiles.remove(i--);
433       }
434     }
435     storeFiles.remove(HRegionFileSystem.REGION_INFO_FILE);
436     return storeFiles;
437   }
438 }