1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.cleaner;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collection;
27  import java.util.Collections;
28  import java.util.List;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.fs.FileStatus;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.hbase.HBaseTestingUtility;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.MediumTests;
39  import org.apache.hadoop.hbase.client.HBaseAdmin;
40  import org.apache.hadoop.hbase.client.HTable;
41  import org.apache.hadoop.hbase.master.HMaster;
42  import org.apache.hadoop.hbase.master.snapshot.DisabledTableSnapshotHandler;
43  import org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner;
44  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
45  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
46  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
47  import org.apache.hadoop.hbase.regionserver.HRegion;
48  import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
49  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
50  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
51  import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
52  import org.apache.hadoop.hbase.util.Bytes;
53  import org.apache.hadoop.hbase.util.FSUtils;
54  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
55  import org.junit.After;
56  import org.junit.AfterClass;
57  import org.junit.Before;
58  import org.junit.BeforeClass;
59  import org.junit.Test;
60  import org.junit.experimental.categories.Category;
61  import org.mockito.Mockito;
62  
63  import com.google.common.collect.Lists;
64  
65  /**
66   * Test the master-related aspects of a snapshot
67   */
68  @Category(MediumTests.class)
69  public class TestSnapshotFromMaster {
70  
71    private static final Log LOG = LogFactory.getLog(TestSnapshotFromMaster.class);
72    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
73    private static final int NUM_RS = 2;
74    private static Path rootDir;
75    private static Path snapshots;
76    private static FileSystem fs;
77    private static HMaster master;
78  
79    // for hfile archiving test.
80    private static Path archiveDir;
81    private static final String STRING_TABLE_NAME = "test";
82    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
83    private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
84    // refresh the cache every 1/2 second
85    private static final long cacheRefreshPeriod = 500;
86  
87    /**
88     * Setup the config for the cluster
89     */
90    @BeforeClass
91    public static void setupCluster() throws Exception {
92      setupConf(UTIL.getConfiguration());
93      UTIL.startMiniCluster(NUM_RS);
94      fs = UTIL.getDFSCluster().getFileSystem();
95      master = UTIL.getMiniHBaseCluster().getMaster();
96      rootDir = master.getMasterFileSystem().getRootDir();
97      snapshots = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
98      archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
99    }
100 
101   private static void setupConf(Configuration conf) {
102     // disable the ui
103     conf.setInt("hbase.regionsever.info.port", -1);
104     // change the flush size to a small amount, regulating number of store files
105     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
106     // so make sure we get a compaction when doing a load, but keep around some
107     // files in the store
108     conf.setInt("hbase.hstore.compaction.min", 3);
109     conf.setInt("hbase.hstore.compactionThreshold", 5);
110     // block writes if we get to 12 store files
111     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
112     // drop the number of attempts for the hbase admin
113     conf.setInt("hbase.client.retries.number", 1);
114     // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner)
115     conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
116     conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, "");
117     // Enable snapshot
118     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
119     conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod);
120 
121     // prevent aggressive region split
122     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
123       ConstantSizeRegionSplitPolicy.class.getName());
124   }
125 
126   @Before
127   public void setup() throws Exception {
128     UTIL.createTable(TABLE_NAME, TEST_FAM);
129     master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(null);
130   }
131 
132   @After
133   public void tearDown() throws Exception {
134     UTIL.deleteTable(TABLE_NAME);
135 
136     // delete the archive directory, if its exists
137     if (fs.exists(archiveDir)) {
138       if (!fs.delete(archiveDir, true)) {
139         throw new IOException("Couldn't delete archive directory (" + archiveDir
140             + " for an unknown reason");
141       }
142     }
143 
144     // delete the snapshot directory, if its exists
145     if (fs.exists(snapshots)) {
146       if (!fs.delete(snapshots, true)) {
147         throw new IOException("Couldn't delete snapshots directory (" + snapshots
148             + " for an unknown reason");
149       }
150     }
151   }
152 
153   @AfterClass
154   public static void cleanupTest() throws Exception {
155     try {
156       UTIL.shutdownMiniCluster();
157     } catch (Exception e) {
158       // NOOP;
159     }
160   }
161 
162   /**
163    * Test that the contract from the master for checking on a snapshot are valid.
164    * <p>
165    * <ol>
166    * <li>If a snapshot fails with an error, we expect to get the source error.</li>
167    * <li>If there is no snapshot name supplied, we should get an error.</li>
168    * <li>If asking about a snapshot has hasn't occurred, you should get an error.</li>
169    * </ol>
170    */
171   @Test(timeout = 60000)
172   public void testIsDoneContract() throws Exception {
173 
174     String snapshotName = "asyncExpectedFailureTest";
175 
176     // check that we get an exception when looking up snapshot where one hasn't happened
177     SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(),
178       UnknownSnapshotException.class);
179 
180     // and that we get the same issue, even if we specify a name
181     SnapshotDescription desc = SnapshotDescription.newBuilder()
182       .setName(snapshotName).build();
183     SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(desc),
184       UnknownSnapshotException.class);
185 
186     // set a mock handler to simulate a snapshot
187     DisabledTableSnapshotHandler mockHandler = Mockito.mock(DisabledTableSnapshotHandler.class);
188     Mockito.when(mockHandler.getException()).thenReturn(null);
189     Mockito.when(mockHandler.getSnapshot()).thenReturn(desc);
190     Mockito.when(mockHandler.isFinished()).thenReturn(new Boolean(true));
191 
192     master.getSnapshotManagerForTesting().setSnapshotHandlerForTesting(mockHandler);
193 
194     // if we do a lookup without a snapshot name, we should fail - you should always know your name
195     SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(),
196       UnknownSnapshotException.class);
197 
198     // then do the lookup for the snapshot that it is done
199     boolean isDone = master.isSnapshotDone(new HSnapshotDescription(desc));
200     assertTrue("Snapshot didn't complete when it should have.", isDone);
201 
202     // now try the case where we are looking for a snapshot we didn't take
203     desc = SnapshotDescription.newBuilder().setName("Not A Snapshot").build();
204     SnapshotTestingUtils.expectSnapshotDoneException(master, new HSnapshotDescription(desc),
205       UnknownSnapshotException.class);
206 
207     // then create a snapshot to the fs and make sure that we can find it when checking done
208     snapshotName = "completed";
209     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
210     desc = desc.toBuilder().setName(snapshotName).build();
211     SnapshotDescriptionUtils.writeSnapshotInfo(desc, snapshotDir, fs);
212 
213     isDone = master.isSnapshotDone(new HSnapshotDescription(desc));
214     assertTrue("Completed, on-disk snapshot not found", isDone);
215   }
216 
217   @Test
218   public void testGetCompletedSnapshots() throws Exception {
219     // first check when there are no snapshots
220     List<HSnapshotDescription> snapshots = master.getCompletedSnapshots();
221     assertEquals("Found unexpected number of snapshots", 0, snapshots.size());
222 
223     // write one snapshot to the fs
224     String snapshotName = "completed";
225     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
226     SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();
227     SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);
228 
229     // check that we get one snapshot
230     snapshots = master.getCompletedSnapshots();
231     assertEquals("Found unexpected number of snapshots", 1, snapshots.size());
232     List<HSnapshotDescription> expected = Lists.newArrayList(new HSnapshotDescription(snapshot));
233     assertEquals("Returned snapshots don't match created snapshots", expected, snapshots);
234 
235     // write a second snapshot
236     snapshotName = "completed_two";
237     snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
238     snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();
239     SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);
240     expected.add(new HSnapshotDescription(snapshot));
241 
242     // check that we get one snapshot
243     snapshots = master.getCompletedSnapshots();
244     assertEquals("Found unexpected number of snapshots", 2, snapshots.size());
245     assertEquals("Returned snapshots don't match created snapshots", expected, snapshots);
246   }
247 
248   @Test
249   public void testDeleteSnapshot() throws Exception {
250 
251     String snapshotName = "completed";
252     SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName(snapshotName).build();
253 
254     try {
255       master.deleteSnapshot(new HSnapshotDescription(snapshot));
256       fail("Master didn't throw exception when attempting to delete snapshot that doesn't exist");
257     } catch (IOException e) {
258       LOG.debug("Correctly failed delete of non-existant snapshot:" + e.getMessage());
259     }
260 
261     // write one snapshot to the fs
262     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
263     SnapshotDescriptionUtils.writeSnapshotInfo(snapshot, snapshotDir, fs);
264 
265     // then delete the existing snapshot,which shouldn't cause an exception to be thrown
266     master.deleteSnapshot(new HSnapshotDescription(snapshot));
267   }
268 
269   /**
270    * Test that the snapshot hfile archive cleaner works correctly. HFiles that are in snapshots
271    * should be retained, while those that are not in a snapshot should be deleted.
272    * @throws Exception on failure
273    */
274   @Test
275   public void testSnapshotHFileArchiving() throws Exception {
276     HBaseAdmin admin = UTIL.getHBaseAdmin();
277     // make sure we don't fail on listing snapshots
278     SnapshotTestingUtils.assertNoSnapshots(admin);
279     // load the table
280     UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
281 
282     // disable the table so we can take a snapshot
283     admin.disableTable(TABLE_NAME);
284 
285     // take a snapshot of the table
286     String snapshotName = "snapshot";
287     byte[] snapshotNameBytes = Bytes.toBytes(snapshotName);
288     admin.snapshot(snapshotNameBytes, TABLE_NAME);
289 
290     Configuration conf = master.getConfiguration();
291     LOG.info("After snapshot File-System state");
292     FSUtils.logFileSystemState(fs, rootDir, LOG);
293 
294     // ensure we only have one snapshot
295     SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshotNameBytes, TABLE_NAME);
296 
297     // renable the table so we can compact the regions
298     admin.enableTable(TABLE_NAME);
299 
300     // compact the files so we get some archived files for the table we just snapshotted
301     List<HRegion> regions = UTIL.getHBaseCluster().getRegions(TABLE_NAME);
302     for (HRegion region : regions) {
303       region.waitForFlushesAndCompactions(); // enable can trigger a compaction, wait for it.
304       region.compactStores();
305     }
306     LOG.info("After compaction File-System state");
307     FSUtils.logFileSystemState(fs, rootDir, LOG);
308 
309     // make sure the cleaner has run
310     LOG.debug("Running hfile cleaners");
311     ensureHFileCleanersRun();
312     LOG.info("After cleaners File-System state: " + rootDir);
313     FSUtils.logFileSystemState(fs, rootDir, LOG);
314 
315     // get the snapshot files for the table
316     Path snapshotTable = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
317     FileStatus[] snapshotHFiles = SnapshotTestingUtils.listHFiles(fs, snapshotTable);
318     // check that the files in the archive contain the ones that we need for the snapshot
319     LOG.debug("Have snapshot hfiles:");
320     for (FileStatus file : snapshotHFiles) {
321       LOG.debug(file.getPath());
322     }
323     // get the archived files for the table
324     Collection<String> files = getArchivedHFiles(archiveDir, rootDir, fs, STRING_TABLE_NAME);
325 
326     // and make sure that there is a proper subset
327     for (FileStatus file : snapshotHFiles) {
328       assertTrue("Archived hfiles " + files + " is missing snapshot file:" + file.getPath(),
329         files.contains(file.getPath().getName()));
330     }
331 
332     // delete the existing snapshot
333     admin.deleteSnapshot(snapshotNameBytes);
334     SnapshotTestingUtils.assertNoSnapshots(admin);
335 
336     // make sure that we don't keep around the hfiles that aren't in a snapshot
337     // make sure we wait long enough to refresh the snapshot hfile
338     List<BaseHFileCleanerDelegate> delegates = UTIL.getMiniHBaseCluster().getMaster()
339         .getHFileCleaner().cleanersChain;
340     for (BaseHFileCleanerDelegate delegate: delegates) {
341       if (delegate instanceof SnapshotHFileCleaner) {
342         ((SnapshotHFileCleaner)delegate).getFileCacheForTesting().triggerCacheRefreshForTesting();
343       }
344     }
345     // run the cleaner again
346     LOG.debug("Running hfile cleaners");
347     ensureHFileCleanersRun();
348     LOG.info("After delete snapshot cleaners run File-System state");
349     FSUtils.logFileSystemState(fs, rootDir, LOG);
350 
351     files = getArchivedHFiles(archiveDir, rootDir, fs, STRING_TABLE_NAME);
352     assertEquals("Still have some hfiles in the archive, when their snapshot has been deleted.", 0,
353       files.size());
354   }
355 
356   /**
357    * @return all the HFiles for a given table that have been archived
358    * @throws IOException on expected failure
359    */
360   private final Collection<String> getArchivedHFiles(Path archiveDir, Path rootDir,
361       FileSystem fs, String tableName) throws IOException {
362     Path tableArchive = new Path(archiveDir, tableName);
363     FileStatus[] archivedHFiles = SnapshotTestingUtils.listHFiles(fs, tableArchive);
364     List<String> files = new ArrayList<String>(archivedHFiles.length);
365     LOG.debug("Have archived hfiles: " + tableArchive);
366     for (FileStatus file : archivedHFiles) {
367       LOG.debug(file.getPath());
368       files.add(file.getPath().getName());
369     }
370     // sort the archived files
371 
372     Collections.sort(files);
373     return files;
374   }
375 
376   /**
377    * Make sure the {@link HFileCleaner HFileCleaners} run at least once
378    */
379   private static void ensureHFileCleanersRun() {
380     UTIL.getHBaseCluster().getMaster().getHFileCleaner().chore();
381   }
382 }