View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.backup;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.testclassification.MediumTests;
39  import org.apache.hadoop.hbase.Stoppable;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.client.Admin;
42  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
43  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
44  import org.apache.hadoop.hbase.regionserver.HRegion;
45  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
46  import org.apache.hadoop.hbase.regionserver.HRegionServer;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.FSUtils;
49  import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil;
50  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
51  import org.apache.hadoop.hbase.util.StoppableImplementation;
52  import org.junit.After;
53  import org.junit.AfterClass;
54  import org.junit.Assert;
55  import org.junit.BeforeClass;
56  import org.junit.Test;
57  import org.junit.experimental.categories.Category;
58  
59  /**
60   * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up
61   * a region
62   */
63  @Category(MediumTests.class)
64  public class TestHFileArchiving {
65  
66    private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class);
67    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
68    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
69  
70    /**
71     * Setup the config for the cluster
72     */
73    @BeforeClass
74    public static void setupCluster() throws Exception {
75      setupConf(UTIL.getConfiguration());
76      UTIL.startMiniCluster();
77  
78      // We don't want the cleaner to remove files. The tests do that.
79      UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().interrupt();
80    }
81  
82    private static void setupConf(Configuration conf) {
83      // disable the ui
84      conf.setInt("hbase.regionsever.info.port", -1);
85      // drop the memstore size so we get flushes
86      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
87      // disable major compactions
88      conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);
89  
90      // prevent aggressive region split
91      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
92        ConstantSizeRegionSplitPolicy.class.getName());
93    }
94  
95    @After
96    public void tearDown() throws Exception {
97      // cleanup the archive directory
98      try {
99        clearArchiveDirectory();
100     } catch (IOException e) {
101       Assert.fail("Failure to delete archive directory:" + e.getMessage());
102     }
103   }
104 
105   @AfterClass
106   public static void cleanupTest() throws Exception {
107     try {
108       UTIL.shutdownMiniCluster();
109     } catch (Exception e) {
110       // NOOP;
111     }
112   }
113 
114   @Test
115   public void testRemovesRegionDirOnArchive() throws Exception {
116     TableName TABLE_NAME =
117         TableName.valueOf("testRemovesRegionDirOnArchive");
118     UTIL.createTable(TABLE_NAME, TEST_FAM);
119 
120     final Admin admin = UTIL.getHBaseAdmin();
121 
122     // get the current store files for the region
123     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
124     // make sure we only have 1 region serving this table
125     assertEquals(1, servingRegions.size());
126     HRegion region = servingRegions.get(0);
127 
128     // and load the table
129     UTIL.loadRegion(region, TEST_FAM);
130 
131     // shutdown the table so we can manipulate the files
132     admin.disableTable(TABLE_NAME);
133 
134     FileSystem fs = UTIL.getTestFileSystem();
135 
136     // now attempt to depose the region
137     Path rootDir = region.getRegionFileSystem().getTableDir().getParent();
138     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
139 
140     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
141 
142     // check for the existence of the archive directory and some files in it
143     Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
144     assertTrue(fs.exists(archiveDir));
145 
146     // check to make sure the store directory was copied
147     // check to make sure the store directory was copied
148     FileStatus[] stores = fs.listStatus(archiveDir, new PathFilter() {
149       @Override
150       public boolean accept(Path p) {
151         if (p.getName().contains(HConstants.RECOVERED_EDITS_DIR)) {
152           return false;
153         }
154         return true;
155       }
156     });
157     assertTrue(stores.length == 1);
158 
159     // make sure we archived the store files
160     FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
161     assertTrue(storeFiles.length > 0);
162 
163     // then ensure the region's directory isn't present
164     assertFalse(fs.exists(regionDir));
165 
166     UTIL.deleteTable(TABLE_NAME);
167   }
168 
169   /**
170    * Test that the region directory is removed when we archive a region without store files, but
171    * still has hidden files.
172    * @throws Exception
173    */
174   @Test
175   public void testDeleteRegionWithNoStoreFiles() throws Exception {
176     TableName TABLE_NAME =
177         TableName.valueOf("testDeleteRegionWithNoStoreFiles");
178     UTIL.createTable(TABLE_NAME, TEST_FAM);
179 
180     // get the current store files for the region
181     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
182     // make sure we only have 1 region serving this table
183     assertEquals(1, servingRegions.size());
184     HRegion region = servingRegions.get(0);
185 
186     FileSystem fs = region.getRegionFileSystem().getFileSystem();
187 
188     // make sure there are some files in the regiondir
189     Path rootDir = FSUtils.getRootDir(fs.getConf());
190     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
191     FileStatus[] regionFiles = FSUtils.listStatus(fs, regionDir, null);
192     Assert.assertNotNull("No files in the region directory", regionFiles);
193     if (LOG.isDebugEnabled()) {
194       List<Path> files = new ArrayList<Path>();
195       for (FileStatus file : regionFiles) {
196         files.add(file.getPath());
197       }
198       LOG.debug("Current files:" + files);
199     }
200     // delete the visible folders so we just have hidden files/folders
201     final PathFilter dirFilter = new FSUtils.DirFilter(fs);
202     PathFilter nonHidden = new PathFilter() {
203       @Override
204       public boolean accept(Path file) {
205         return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
206       }
207     };
208     FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
209     for (FileStatus store : storeDirs) {
210       LOG.debug("Deleting store for test");
211       fs.delete(store.getPath(), true);
212     }
213 
214     // then archive the region
215     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
216 
217     // and check to make sure the region directoy got deleted
218     assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
219 
220     UTIL.deleteTable(TABLE_NAME);
221   }
222 
223   @Test
224   public void testArchiveOnTableDelete() throws Exception {
225     TableName TABLE_NAME =
226         TableName.valueOf("testArchiveOnTableDelete");
227     UTIL.createTable(TABLE_NAME, TEST_FAM);
228 
229     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
230     // make sure we only have 1 region serving this table
231     assertEquals(1, servingRegions.size());
232     HRegion region = servingRegions.get(0);
233 
234     // get the parent RS and monitor
235     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
236     FileSystem fs = hrs.getFileSystem();
237 
238     // put some data on the region
239     LOG.debug("-------Loading table");
240     UTIL.loadRegion(region, TEST_FAM);
241 
242     // get the hfiles in the region
243     List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
244     assertEquals("More that 1 region for test table.", 1, regions.size());
245 
246     region = regions.get(0);
247     // wait for all the compactions to complete
248     region.waitForFlushesAndCompactions();
249 
250     // disable table to prevent new updates
251     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
252     LOG.debug("Disabled table");
253 
254     // remove all the files from the archive to get a fair comparison
255     clearArchiveDirectory();
256 
257     // then get the current store files
258     List<String> storeFiles = getRegionStoreFiles(region);
259 
260     // then delete the table so the hfiles get archived
261     UTIL.deleteTable(TABLE_NAME);
262     LOG.debug("Deleted table");
263 
264     assertArchiveFiles(fs, storeFiles, 30000);
265   }
266 
267   private void assertArchiveFiles(FileSystem fs, List<String> storeFiles, long timeout) throws IOException {
268     long end = System.currentTimeMillis() + timeout;
269     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
270     List<String> archivedFiles = new ArrayList<String>();
271 
272     // We have to ensure that the DeleteTableHandler is finished. HBaseAdmin.deleteXXX() can return before all files
273     // are archived. We should fix HBASE-5487 and fix synchronous operations from admin.
274     while (System.currentTimeMillis() < end) {
275       archivedFiles = getAllFileNames(fs, archiveDir);
276       if (archivedFiles.size() >= storeFiles.size()) {
277         break;
278       }
279     }
280 
281     Collections.sort(storeFiles);
282     Collections.sort(archivedFiles);
283 
284     LOG.debug("Store files:");
285     for (int i = 0; i < storeFiles.size(); i++) {
286       LOG.debug(i + " - " + storeFiles.get(i));
287     }
288     LOG.debug("Archive files:");
289     for (int i = 0; i < archivedFiles.size(); i++) {
290       LOG.debug(i + " - " + archivedFiles.get(i));
291     }
292 
293     assertTrue("Archived files are missing some of the store files!",
294       archivedFiles.containsAll(storeFiles));
295   }
296 
297 
298   /**
299    * Test that the store files are archived when a column family is removed.
300    * @throws Exception
301    */
302   @Test
303   public void testArchiveOnTableFamilyDelete() throws Exception {
304     TableName TABLE_NAME =
305         TableName.valueOf("testArchiveOnTableFamilyDelete");
306     UTIL.createTable(TABLE_NAME, TEST_FAM);
307 
308     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
309     // make sure we only have 1 region serving this table
310     assertEquals(1, servingRegions.size());
311     HRegion region = servingRegions.get(0);
312 
313     // get the parent RS and monitor
314     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
315     FileSystem fs = hrs.getFileSystem();
316 
317     // put some data on the region
318     LOG.debug("-------Loading table");
319     UTIL.loadRegion(region, TEST_FAM);
320 
321     // get the hfiles in the region
322     List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
323     assertEquals("More that 1 region for test table.", 1, regions.size());
324 
325     region = regions.get(0);
326     // wait for all the compactions to complete
327     region.waitForFlushesAndCompactions();
328 
329     // disable table to prevent new updates
330     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
331     LOG.debug("Disabled table");
332 
333     // remove all the files from the archive to get a fair comparison
334     clearArchiveDirectory();
335 
336     // then get the current store files
337     List<String> storeFiles = getRegionStoreFiles(region);
338 
339     // then delete the table so the hfiles get archived
340     UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM);
341 
342     assertArchiveFiles(fs, storeFiles, 30000);
343 
344     UTIL.deleteTable(TABLE_NAME);
345   }
346 
347   /**
348    * Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
349    */
350   @Test
351   public void testCleaningRace() throws Exception {
352     final long TEST_TIME = 20 * 1000;
353 
354     Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
355     Path rootDir = UTIL.getDataTestDirOnTestFS("testCleaningRace");
356     FileSystem fs = UTIL.getTestFileSystem();
357 
358     Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
359     Path regionDir = new Path(FSUtils.getTableDir(new Path("./"),
360         TableName.valueOf("table")), "abcdef");
361     Path familyDir = new Path(regionDir, "cf");
362 
363     Path sourceRegionDir = new Path(rootDir, regionDir);
364     fs.mkdirs(sourceRegionDir);
365 
366     Stoppable stoppable = new StoppableImplementation();
367 
368     // The cleaner should be looping without long pauses to reproduce the race condition.
369     HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir);
370     try {
371       cleaner.start();
372 
373       // Keep creating/archiving new files while the cleaner is running in the other thread
374       long startTime = System.currentTimeMillis();
375       for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) {
376         Path file = new Path(familyDir,  String.valueOf(fid));
377         Path sourceFile = new Path(rootDir, file);
378         Path archiveFile = new Path(archiveDir, file);
379 
380         fs.createNewFile(sourceFile);
381 
382         try {
383           // Try to archive the file
384           HFileArchiver.archiveRegion(fs, rootDir,
385               sourceRegionDir.getParent(), sourceRegionDir);
386 
387           // The archiver succeded, the file is no longer in the original location
388           // but it's in the archive location.
389           LOG.debug("hfile=" + fid + " should be in the archive");
390           assertTrue(fs.exists(archiveFile));
391           assertFalse(fs.exists(sourceFile));
392         } catch (IOException e) {
393           // The archiver is unable to archive the file. Probably HBASE-7643 race condition.
394           // in this case, the file should not be archived, and we should have the file
395           // in the original location.
396           LOG.debug("hfile=" + fid + " should be in the source location");
397           assertFalse(fs.exists(archiveFile));
398           assertTrue(fs.exists(sourceFile));
399 
400           // Avoid to have this file in the next run
401           fs.delete(sourceFile, false);
402         }
403       }
404     } finally {
405       stoppable.stop("test end");
406       cleaner.join();
407       fs.delete(rootDir, true);
408     }
409   }
410 
411   private void clearArchiveDirectory() throws IOException {
412     UTIL.getTestFileSystem().delete(
413       new Path(UTIL.getDefaultRootDirPath(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
414   }
415 
416   /**
417    * Get the names of all the files below the given directory
418    * @param fs
419    * @param archiveDir
420    * @return
421    * @throws IOException
422    */
423   private List<String> getAllFileNames(final FileSystem fs, Path archiveDir) throws IOException {
424     FileStatus[] files = FSUtils.listStatus(fs, archiveDir, new PathFilter() {
425       @Override
426       public boolean accept(Path p) {
427         if (p.getName().contains(HConstants.RECOVERED_EDITS_DIR)) {
428           return false;
429         }
430         return true;
431       }
432     });
433     return recurseOnFiles(fs, files, new ArrayList<String>());
434   }
435 
436   /** Recursively lookup all the file names under the file[] array **/
437   private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<String> fileNames)
438       throws IOException {
439     if (files == null || files.length == 0) return fileNames;
440 
441     for (FileStatus file : files) {
442       if (file.isDirectory()) {
443         recurseOnFiles(fs, FSUtils.listStatus(fs, file.getPath(), null), fileNames);
444       } else fileNames.add(file.getPath().getName());
445     }
446     return fileNames;
447   }
448 
449   private List<String> getRegionStoreFiles(final HRegion region) throws IOException {
450     Path regionDir = region.getRegionFileSystem().getRegionDir();
451     FileSystem fs = region.getRegionFileSystem().getFileSystem();
452     List<String> storeFiles = getAllFileNames(fs, regionDir);
453     // remove all the non-storefile named files for the region
454     for (int i = 0; i < storeFiles.size(); i++) {
455       String file = storeFiles.get(i);
456       if (file.contains(HRegionFileSystem.REGION_INFO_FILE) || file.contains("wal")) {
457         storeFiles.remove(i--);
458       }
459     }
460     storeFiles.remove(HRegionFileSystem.REGION_INFO_FILE);
461     return storeFiles;
462   }
463 }