1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.backup;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.MediumTests;
39  import org.apache.hadoop.hbase.Stoppable;
40  import org.apache.hadoop.hbase.backup.HFileArchiver;
41  import org.apache.hadoop.hbase.client.HBaseAdmin;
42  import org.apache.hadoop.hbase.master.MasterFileSystem;
43  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
44  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
45  import org.apache.hadoop.hbase.regionserver.HRegion;
46  import org.apache.hadoop.hbase.regionserver.HRegionServer;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.FSUtils;
49  import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil;
50  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
51  import org.apache.hadoop.hbase.util.StoppableImplementation;
52  import org.junit.After;
53  import org.junit.AfterClass;
54  import org.junit.Assert;
55  import org.junit.Before;
56  import org.junit.BeforeClass;
57  import org.junit.Test;
58  import org.junit.experimental.categories.Category;
59  
60  /**
61   * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up
62   * a region
63   */
64  @Category(MediumTests.class)
65  public class TestHFileArchiving {
66  
67    private static final String STRING_TABLE_NAME = "test_table";
68  
69    private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class);
70    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
71    private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
72    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
73  
74    /**
75     * Setup the config for the cluster
76     */
77    @BeforeClass
78    public static void setupCluster() throws Exception {
79      setupConf(UTIL.getConfiguration());
80      UTIL.startMiniCluster();
81  
82      // We don't want the cleaner to remove files. The tests do that.
83      UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().interrupt();
84    }
85  
86    private static void setupConf(Configuration conf) {
87      // disable the ui
88      conf.setInt("hbase.regionsever.info.port", -1);
89      // drop the memstore size so we get flushes
90      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
91      // disable major compactions
92      conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);
93  
94      // prevent aggressive region split
95      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
96        ConstantSizeRegionSplitPolicy.class.getName());
97    }
98  
99    @Before
100   public void setup() throws Exception {
101     UTIL.createTable(TABLE_NAME, TEST_FAM);
102   }
103 
104   @After
105   public void tearDown() throws Exception {
106     // cleanup the cluster if its up still
107     if (UTIL.getHBaseAdmin().tableExists(STRING_TABLE_NAME)) {
108       UTIL.deleteTable(TABLE_NAME);
109     }
110     // and cleanup the archive directory
111     try {
112       clearArchiveDirectory();
113     } catch (IOException e) {
114       Assert.fail("Failure to delete archive directory:" + e.getMessage());
115     }
116   }
117 
118   @AfterClass
119   public static void cleanupTest() throws Exception {
120     try {
121       UTIL.shutdownMiniCluster();
122     } catch (Exception e) {
123       // NOOP;
124     }
125   }
126 
127   @Test
128   public void testRemovesRegionDirOnArchive() throws Exception {
129     final HBaseAdmin admin = UTIL.getHBaseAdmin();
130 
131     // get the current store files for the region
132     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
133     // make sure we only have 1 region serving this table
134     assertEquals(1, servingRegions.size());
135     HRegion region = servingRegions.get(0);
136 
137     // and load the table
138     UTIL.loadRegion(region, TEST_FAM);
139 
140     // shutdown the table so we can manipulate the files
141     admin.disableTable(STRING_TABLE_NAME);
142 
143     FileSystem fs = UTIL.getTestFileSystem();
144 
145     // now attempt to depose the region
146     Path regionDir = HRegion.getRegionDir(region.getTableDir().getParent(), region.getRegionInfo());
147 
148     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
149 
150     // check for the existence of the archive directory and some files in it
151     Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
152     assertTrue(fs.exists(archiveDir));
153 
154     // check to make sure the store directory was copied
155     FileStatus[] stores = fs.listStatus(archiveDir);
156     assertTrue(stores.length == 1);
157 
158     // make sure we archived the store files
159     FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
160     assertTrue(storeFiles.length > 0);
161 
162     // then ensure the region's directory isn't present
163     assertFalse(fs.exists(regionDir));
164   }
165 
166   /**
167    * Test that the region directory is removed when we archive a region without store files, but
168    * still has hidden files.
169    * @throws Exception
170    */
171   @Test
172   public void testDeleteRegionWithNoStoreFiles() throws Exception {
173     // get the current store files for the region
174     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
175     // make sure we only have 1 region serving this table
176     assertEquals(1, servingRegions.size());
177     HRegion region = servingRegions.get(0);
178 
179     FileSystem fs = region.getFilesystem();
180 
181     // make sure there are some files in the regiondir
182     Path rootDir = FSUtils.getRootDir(fs.getConf());
183     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
184     FileStatus[] regionFiles = FSUtils.listStatus(fs, regionDir, null);
185     Assert.assertNotNull("No files in the region directory", regionFiles);
186     if (LOG.isDebugEnabled()) {
187       List<Path> files = new ArrayList<Path>();
188       for (FileStatus file : regionFiles) {
189         files.add(file.getPath());
190       }
191       LOG.debug("Current files:" + files);
192     }
193     // delete the visible folders so we just have hidden files/folders
194     final PathFilter dirFilter = new FSUtils.DirFilter(fs);
195     PathFilter nonHidden = new PathFilter() {
196       @Override
197       public boolean accept(Path file) {
198         return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
199       }
200     };
201     FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
202     for (FileStatus store : storeDirs) {
203       LOG.debug("Deleting store for test");
204       fs.delete(store.getPath(), true);
205     }
206 
207     // then archive the region
208     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
209 
210     // and check to make sure the region directoy got deleted
211     assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
212   }
213 
214   @Test
215   public void testArchiveOnTableDelete() throws Exception {
216     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
217     // make sure we only have 1 region serving this table
218     assertEquals(1, servingRegions.size());
219     HRegion region = servingRegions.get(0);
220 
221     // get the parent RS and monitor
222     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
223     FileSystem fs = hrs.getFileSystem();
224 
225     // put some data on the region
226     LOG.debug("-------Loading table");
227     UTIL.loadRegion(region, TEST_FAM);
228 
229     // get the hfiles in the region
230     List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
231     assertEquals("More that 1 region for test table.", 1, regions.size());
232 
233     region = regions.get(0);
234     // wait for all the compactions to complete
235     region.waitForFlushesAndCompactions();
236 
237     // disable table to prevent new updates
238     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
239     LOG.debug("Disabled table");
240 
241     // remove all the files from the archive to get a fair comparison
242     clearArchiveDirectory();
243 
244     // then get the current store files
245     Path regionDir = region.getRegionDir();
246     List<String> storeFiles = getRegionStoreFiles(fs, regionDir);
247 
248     // then delete the table so the hfiles get archived
249     UTIL.deleteTable(TABLE_NAME);
250 
251     // then get the files in the archive directory.
252     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
253     List<String> archivedFiles = getAllFileNames(fs, archiveDir);
254     Collections.sort(storeFiles);
255     Collections.sort(archivedFiles);
256 
257     LOG.debug("Store files:");
258     for (int i = 0; i < storeFiles.size(); i++) {
259       LOG.debug(i + " - " + storeFiles.get(i));
260     }
261     LOG.debug("Archive files:");
262     for (int i = 0; i < archivedFiles.size(); i++) {
263       LOG.debug(i + " - " + archivedFiles.get(i));
264     }
265 
266     assertTrue("Archived files are missing some of the store files!",
267       archivedFiles.containsAll(storeFiles));
268   }
269 
270   /**
271    * Test that the store files are archived when a column family is removed.
272    * @throws Exception
273    */
274   @Test
275   public void testArchiveOnTableFamilyDelete() throws Exception {
276     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
277     // make sure we only have 1 region serving this table
278     assertEquals(1, servingRegions.size());
279     HRegion region = servingRegions.get(0);
280 
281     // get the parent RS and monitor
282     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
283     FileSystem fs = hrs.getFileSystem();
284 
285     // put some data on the region
286     LOG.debug("-------Loading table");
287     UTIL.loadRegion(region, TEST_FAM);
288 
289     // get the hfiles in the region
290     List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
291     assertEquals("More that 1 region for test table.", 1, regions.size());
292 
293     region = regions.get(0);
294     // wait for all the compactions to complete
295     region.waitForFlushesAndCompactions();
296 
297     // disable table to prevent new updates
298     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
299     LOG.debug("Disabled table");
300 
301     // remove all the files from the archive to get a fair comparison
302     clearArchiveDirectory();
303 
304     // then get the current store files
305     Path regionDir = region.getRegionDir();
306     List<String> storeFiles = getRegionStoreFiles(fs, regionDir);
307 
308     // then delete the table so the hfiles get archived
309     UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM);
310 
311     // then get the files in the archive directory.
312     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
313     List<String> archivedFiles = getAllFileNames(fs, archiveDir);
314     Collections.sort(storeFiles);
315     Collections.sort(archivedFiles);
316 
317     LOG.debug("Store files:");
318     for (int i = 0; i < storeFiles.size(); i++) {
319       LOG.debug(i + " - " + storeFiles.get(i));
320     }
321     LOG.debug("Archive files:");
322     for (int i = 0; i < archivedFiles.size(); i++) {
323       LOG.debug(i + " - " + archivedFiles.get(i));
324     }
325 
326     assertTrue("Archived files are missing some of the store files!",
327       archivedFiles.containsAll(storeFiles));
328   }
329 
330   /**
331    * Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
332    */
333   @Test
334   public void testCleaningRace() throws Exception {
335     final long TEST_TIME = 20 * 1000;
336 
337     Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
338     Path rootDir = UTIL.getDataTestDir("testCleaningRace");
339     FileSystem fs = UTIL.getTestFileSystem();
340 
341     Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
342     Path regionDir = new Path("table", "abcdef");
343     Path familyDir = new Path(regionDir, "cf");
344 
345     Path sourceRegionDir = new Path(rootDir, regionDir);
346     fs.mkdirs(sourceRegionDir);
347 
348     Stoppable stoppable = new StoppableImplementation();
349 
350     // The cleaner should be looping without long pauses to reproduce the race condition.
351     HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir);
352     try {
353       cleaner.start();
354 
355       // Keep creating/archiving new files while the cleaner is running in the other thread
356       long startTime = System.currentTimeMillis();
357       for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) {
358         Path file = new Path(familyDir,  String.valueOf(fid));
359         Path sourceFile = new Path(rootDir, file);
360         Path archiveFile = new Path(archiveDir, file);
361 
362         fs.createNewFile(sourceFile);
363 
364         try {
365           // Try to archive the file
366           HFileArchiver.archiveRegion(fs, rootDir,
367               sourceRegionDir.getParent(), sourceRegionDir);
368 
369           // The archiver succeded, the file is no longer in the original location
370           // but it's in the archive location.
371           LOG.debug("hfile=" + fid + " should be in the archive");
372           assertTrue(fs.exists(archiveFile));
373           assertFalse(fs.exists(sourceFile));
374         } catch (IOException e) {
375           // The archiver is unable to archive the file. Probably HBASE-7643 race condition.
376           // in this case, the file should not be archived, and we should have the file
377           // in the original location.
378           LOG.debug("hfile=" + fid + " should be in the source location");
379           assertFalse(fs.exists(archiveFile));
380           assertTrue(fs.exists(sourceFile));
381 
382           // Avoid to have this file in the next run
383           fs.delete(sourceFile, false);
384         }
385       }
386     } finally {
387       stoppable.stop("test end");
388       cleaner.join();
389       fs.delete(rootDir, true);
390     }
391   }
392 
393   private void clearArchiveDirectory() throws IOException {
394     UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true);
395   }
396 
397   /**
398    * Get the names of all the files below the given directory
399    * @param fs
400    * @param archiveDir
401    * @return
402    * @throws IOException
403    */
404   private List<String> getAllFileNames(final FileSystem fs, Path archiveDir) throws IOException {
405     FileStatus[] files = FSUtils.listStatus(fs, archiveDir, null);
406     return recurseOnFiles(fs, files, new ArrayList<String>());
407   }
408 
409   /** Recursively lookup all the file names under the file[] array **/
410   private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<String> fileNames)
411       throws IOException {
412     if (files == null || files.length == 0) return fileNames;
413 
414     for (FileStatus file : files) {
415       if (file.isDir()) {
416         recurseOnFiles(fs, FSUtils.listStatus(fs, file.getPath(), null), fileNames);
417       } else fileNames.add(file.getPath().getName());
418     }
419     return fileNames;
420   }
421 
422   private List<String> getRegionStoreFiles(final FileSystem fs, final Path regionDir) 
423       throws IOException {
424     List<String> storeFiles = getAllFileNames(fs, regionDir);
425     // remove all the non-storefile named files for the region
426     for (int i = 0; i < storeFiles.size(); i++) {
427       String file = storeFiles.get(i);
428       if (file.contains(HRegion.REGIONINFO_FILE) || file.contains("hlog")) {
429         storeFiles.remove(i--);
430       }
431     }
432     storeFiles.remove(HRegion.REGIONINFO_FILE);
433     return storeFiles;
434   }
435 }