1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.backup;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.fs.PathFilter;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.MediumTests;
39  import org.apache.hadoop.hbase.Stoppable;
40  import org.apache.hadoop.hbase.backup.HFileArchiver;
41  import org.apache.hadoop.hbase.client.HBaseAdmin;
42  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
43  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
44  import org.apache.hadoop.hbase.regionserver.HRegion;
45  import org.apache.hadoop.hbase.regionserver.HRegionServer;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.FSUtils;
48  import org.apache.hadoop.hbase.util.HFileArchiveTestingUtil;
49  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
50  import org.apache.hadoop.hbase.util.StoppableImplementation;
51  import org.junit.After;
52  import org.junit.AfterClass;
53  import org.junit.Assert;
54  import org.junit.BeforeClass;
55  import org.junit.Test;
56  import org.junit.experimental.categories.Category;
57  
58  /**
59   * Test that the {@link HFileArchiver} correctly removes all the parts of a region when cleaning up
60   * a region
61   */
62  @Category(MediumTests.class)
63  public class TestHFileArchiving {
64  
65    private static final Log LOG = LogFactory.getLog(TestHFileArchiving.class);
66    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
67    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
68  
69    /**
70     * Setup the config for the cluster
71     */
72    @BeforeClass
73    public static void setupCluster() throws Exception {
74      setupConf(UTIL.getConfiguration());
75      UTIL.startMiniCluster();
76  
77      // We don't want the cleaner to remove files. The tests do that.
78      UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().interrupt();
79    }
80  
81    private static void setupConf(Configuration conf) {
82      // disable the ui
83      conf.setInt("hbase.regionsever.info.port", -1);
84      // drop the memstore size so we get flushes
85      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
86      // disable major compactions
87      conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);
88  
89      // prevent aggressive region split
90      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
91        ConstantSizeRegionSplitPolicy.class.getName());
92    }
93  
94    @After
95    public void tearDown() throws Exception {
96      // cleanup the archive directory
97      try {
98        clearArchiveDirectory();
99      } catch (IOException e) {
100       Assert.fail("Failure to delete archive directory:" + e.getMessage());
101     }
102   }
103 
104   @AfterClass
105   public static void cleanupTest() throws Exception {
106     try {
107       UTIL.shutdownMiniCluster();
108     } catch (Exception e) {
109       // NOOP;
110     }
111   }
112 
113   @Test
114   public void testRemovesRegionDirOnArchive() throws Exception {
115     byte[] TABLE_NAME = Bytes.toBytes("testRemovesRegionDirOnArchive");
116     UTIL.createTable(TABLE_NAME, TEST_FAM);
117 
118     final HBaseAdmin admin = UTIL.getHBaseAdmin();
119 
120     // get the current store files for the region
121     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
122     // make sure we only have 1 region serving this table
123     assertEquals(1, servingRegions.size());
124     HRegion region = servingRegions.get(0);
125 
126     // and load the table
127     UTIL.loadRegion(region, TEST_FAM);
128 
129     // shutdown the table so we can manipulate the files
130     admin.disableTable(TABLE_NAME);
131 
132     FileSystem fs = UTIL.getTestFileSystem();
133 
134     // now attempt to depose the region
135     Path regionDir = HRegion.getRegionDir(region.getTableDir().getParent(), region.getRegionInfo());
136 
137     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
138 
139     // check for the existence of the archive directory and some files in it
140     Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
141     assertTrue(fs.exists(archiveDir));
142 
143     // check to make sure the store directory was copied
144     FileStatus[] stores = fs.listStatus(archiveDir);
145     assertTrue(stores.length == 1);
146 
147     // make sure we archived the store files
148     FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
149     assertTrue(storeFiles.length > 0);
150 
151     // then ensure the region's directory isn't present
152     assertFalse(fs.exists(regionDir));
153 
154     UTIL.deleteTable(TABLE_NAME);
155   }
156 
157   /**
158    * Test that the region directory is removed when we archive a region without store files, but
159    * still has hidden files.
160    * @throws Exception
161    */
162   @Test
163   public void testDeleteRegionWithNoStoreFiles() throws Exception {
164     byte[] TABLE_NAME = Bytes.toBytes("testDeleteRegionWithNoStoreFiles");
165     UTIL.createTable(TABLE_NAME, TEST_FAM);
166 
167     // get the current store files for the region
168     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
169     // make sure we only have 1 region serving this table
170     assertEquals(1, servingRegions.size());
171     HRegion region = servingRegions.get(0);
172 
173     FileSystem fs = region.getFilesystem();
174 
175     // make sure there are some files in the regiondir
176     Path rootDir = FSUtils.getRootDir(fs.getConf());
177     Path regionDir = HRegion.getRegionDir(rootDir, region.getRegionInfo());
178     FileStatus[] regionFiles = FSUtils.listStatus(fs, regionDir, null);
179     Assert.assertNotNull("No files in the region directory", regionFiles);
180     if (LOG.isDebugEnabled()) {
181       List<Path> files = new ArrayList<Path>();
182       for (FileStatus file : regionFiles) {
183         files.add(file.getPath());
184       }
185       LOG.debug("Current files:" + files);
186     }
187     // delete the visible folders so we just have hidden files/folders
188     final PathFilter dirFilter = new FSUtils.DirFilter(fs);
189     PathFilter nonHidden = new PathFilter() {
190       @Override
191       public boolean accept(Path file) {
192         return dirFilter.accept(file) && !file.getName().toString().startsWith(".");
193       }
194     };
195     FileStatus[] storeDirs = FSUtils.listStatus(fs, regionDir, nonHidden);
196     for (FileStatus store : storeDirs) {
197       LOG.debug("Deleting store for test");
198       fs.delete(store.getPath(), true);
199     }
200 
201     // then archive the region
202     HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());
203 
204     // and check to make sure the region directoy got deleted
205     assertFalse("Region directory (" + regionDir + "), still exists.", fs.exists(regionDir));
206 
207     UTIL.deleteTable(TABLE_NAME);
208   }
209 
210   @Test
211   public void testArchiveOnTableDelete() throws Exception {
212     byte[] TABLE_NAME = Bytes.toBytes("testArchiveOnTableDelete");
213     UTIL.createTable(TABLE_NAME, TEST_FAM);
214 
215     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
216     // make sure we only have 1 region serving this table
217     assertEquals(1, servingRegions.size());
218     HRegion region = servingRegions.get(0);
219 
220     // get the parent RS and monitor
221     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
222     FileSystem fs = hrs.getFileSystem();
223 
224     // put some data on the region
225     LOG.debug("-------Loading table");
226     UTIL.loadRegion(region, TEST_FAM);
227 
228     // get the hfiles in the region
229     List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
230     assertEquals("More that 1 region for test table.", 1, regions.size());
231 
232     region = regions.get(0);
233     // wait for all the compactions to complete
234     region.waitForFlushesAndCompactions();
235 
236     // disable table to prevent new updates
237     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
238     LOG.debug("Disabled table");
239 
240     // remove all the files from the archive to get a fair comparison
241     clearArchiveDirectory();
242 
243     // then get the current store files
244     Path regionDir = region.getRegionDir();
245     List<String> storeFiles = getRegionStoreFiles(fs, regionDir);
246 
247     // then delete the table so the hfiles get archived
248     UTIL.deleteTable(TABLE_NAME);
249 
250     // then get the files in the archive directory.
251     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
252     List<String> archivedFiles = getAllFileNames(fs, archiveDir);
253     Collections.sort(storeFiles);
254     Collections.sort(archivedFiles);
255 
256     LOG.debug("Store files:");
257     for (int i = 0; i < storeFiles.size(); i++) {
258       LOG.debug(i + " - " + storeFiles.get(i));
259     }
260     LOG.debug("Archive files:");
261     for (int i = 0; i < archivedFiles.size(); i++) {
262       LOG.debug(i + " - " + archivedFiles.get(i));
263     }
264 
265     assertTrue("Archived files are missing some of the store files!",
266       archivedFiles.containsAll(storeFiles));
267   }
268 
269   /**
270    * Test that the store files are archived when a column family is removed.
271    * @throws Exception
272    */
273   @Test
274   public void testArchiveOnTableFamilyDelete() throws Exception {
275     byte[] TABLE_NAME = Bytes.toBytes("testArchiveOnTableFamilyDelete");
276     UTIL.createTable(TABLE_NAME, TEST_FAM);
277 
278     List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
279     // make sure we only have 1 region serving this table
280     assertEquals(1, servingRegions.size());
281     HRegion region = servingRegions.get(0);
282 
283     // get the parent RS and monitor
284     HRegionServer hrs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
285     FileSystem fs = hrs.getFileSystem();
286 
287     // put some data on the region
288     LOG.debug("-------Loading table");
289     UTIL.loadRegion(region, TEST_FAM);
290 
291     // get the hfiles in the region
292     List<HRegion> regions = hrs.getOnlineRegions(TABLE_NAME);
293     assertEquals("More that 1 region for test table.", 1, regions.size());
294 
295     region = regions.get(0);
296     // wait for all the compactions to complete
297     region.waitForFlushesAndCompactions();
298 
299     // disable table to prevent new updates
300     UTIL.getHBaseAdmin().disableTable(TABLE_NAME);
301     LOG.debug("Disabled table");
302 
303     // remove all the files from the archive to get a fair comparison
304     clearArchiveDirectory();
305 
306     // then get the current store files
307     Path regionDir = region.getRegionDir();
308     List<String> storeFiles = getRegionStoreFiles(fs, regionDir);
309 
310     // then delete the table so the hfiles get archived
311     UTIL.getHBaseAdmin().deleteColumn(TABLE_NAME, TEST_FAM);
312 
313     // then get the files in the archive directory.
314     Path archiveDir = HFileArchiveUtil.getArchivePath(UTIL.getConfiguration());
315     List<String> archivedFiles = getAllFileNames(fs, archiveDir);
316     Collections.sort(storeFiles);
317     Collections.sort(archivedFiles);
318 
319     LOG.debug("Store files:");
320     for (int i = 0; i < storeFiles.size(); i++) {
321       LOG.debug(i + " - " + storeFiles.get(i));
322     }
323     LOG.debug("Archive files:");
324     for (int i = 0; i < archivedFiles.size(); i++) {
325       LOG.debug(i + " - " + archivedFiles.get(i));
326     }
327 
328     assertTrue("Archived files are missing some of the store files!",
329       archivedFiles.containsAll(storeFiles));
330 
331     UTIL.deleteTable(TABLE_NAME);
332   }
333 
334   /**
335    * Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
336    */
337   @Test
338   public void testCleaningRace() throws Exception {
339     final long TEST_TIME = 20 * 1000;
340 
341     Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
342     Path rootDir = UTIL.getDataTestDir("testCleaningRace");
343     FileSystem fs = UTIL.getTestFileSystem();
344 
345     Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
346     Path regionDir = new Path("table", "abcdef");
347     Path familyDir = new Path(regionDir, "cf");
348 
349     Path sourceRegionDir = new Path(rootDir, regionDir);
350     fs.mkdirs(sourceRegionDir);
351 
352     Stoppable stoppable = new StoppableImplementation();
353 
354     // The cleaner should be looping without long pauses to reproduce the race condition.
355     HFileCleaner cleaner = new HFileCleaner(1, stoppable, conf, fs, archiveDir);
356     try {
357       cleaner.start();
358 
359       // Keep creating/archiving new files while the cleaner is running in the other thread
360       long startTime = System.currentTimeMillis();
361       for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) {
362         Path file = new Path(familyDir,  String.valueOf(fid));
363         Path sourceFile = new Path(rootDir, file);
364         Path archiveFile = new Path(archiveDir, file);
365 
366         fs.createNewFile(sourceFile);
367 
368         try {
369           // Try to archive the file
370           HFileArchiver.archiveRegion(fs, rootDir,
371               sourceRegionDir.getParent(), sourceRegionDir);
372 
373           // The archiver succeded, the file is no longer in the original location
374           // but it's in the archive location.
375           LOG.debug("hfile=" + fid + " should be in the archive");
376           assertTrue(fs.exists(archiveFile));
377           assertFalse(fs.exists(sourceFile));
378         } catch (IOException e) {
379           // The archiver is unable to archive the file. Probably HBASE-7643 race condition.
380           // in this case, the file should not be archived, and we should have the file
381           // in the original location.
382           LOG.debug("hfile=" + fid + " should be in the source location");
383           assertFalse(fs.exists(archiveFile));
384           assertTrue(fs.exists(sourceFile));
385 
386           // Avoid to have this file in the next run
387           fs.delete(sourceFile, false);
388         }
389       }
390     } finally {
391       stoppable.stop("test end");
392       cleaner.join();
393       fs.delete(rootDir, true);
394     }
395   }
396 
397   private void clearArchiveDirectory() throws IOException {
398     UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true);
399   }
400 
401   /**
402    * Get the names of all the files below the given directory
403    * @param fs
404    * @param archiveDir
405    * @return
406    * @throws IOException
407    */
408   private List<String> getAllFileNames(final FileSystem fs, Path archiveDir) throws IOException {
409     FileStatus[] files = FSUtils.listStatus(fs, archiveDir, null);
410     return recurseOnFiles(fs, files, new ArrayList<String>());
411   }
412 
413   /** Recursively lookup all the file names under the file[] array **/
414   private List<String> recurseOnFiles(FileSystem fs, FileStatus[] files, List<String> fileNames)
415       throws IOException {
416     if (files == null || files.length == 0) return fileNames;
417 
418     for (FileStatus file : files) {
419       if (file.isDir()) {
420         recurseOnFiles(fs, FSUtils.listStatus(fs, file.getPath(), null), fileNames);
421       } else fileNames.add(file.getPath().getName());
422     }
423     return fileNames;
424   }
425 
426   private List<String> getRegionStoreFiles(final FileSystem fs, final Path regionDir) 
427       throws IOException {
428     List<String> storeFiles = getAllFileNames(fs, regionDir);
429     // remove all the non-storefile named files for the region
430     for (int i = 0; i < storeFiles.size(); i++) {
431       String file = storeFiles.get(i);
432       if (file.contains(HRegion.REGIONINFO_FILE) || file.contains("hlog")) {
433         storeFiles.remove(i--);
434       }
435     }
436     storeFiles.remove(HRegion.REGIONINFO_FILE);
437     return storeFiles;
438   }
439 }