View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.backup.example;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.List;
27  import java.util.concurrent.CountDownLatch;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HColumnDescriptor;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.testclassification.MediumTests;
39  import org.apache.hadoop.hbase.Stoppable;
40  import org.apache.hadoop.hbase.client.ClusterConnection;
41  import org.apache.hadoop.hbase.client.ConnectionFactory;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
44  import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
45  import org.apache.hadoop.hbase.regionserver.HRegion;
46  import org.apache.hadoop.hbase.regionserver.Store;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.FSUtils;
49  import org.apache.hadoop.hbase.util.HFileArchiveUtil;
50  import org.apache.hadoop.hbase.util.StoppableImplementation;
51  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
52  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
53  import org.apache.zookeeper.KeeperException;
54  import org.junit.After;
55  import org.junit.AfterClass;
56  import org.junit.BeforeClass;
57  import org.junit.Test;
58  import org.junit.experimental.categories.Category;
59  import org.mockito.Mockito;
60  import org.mockito.invocation.InvocationOnMock;
61  import org.mockito.stubbing.Answer;
62  
63  /**
64   * Spin up a small cluster and check that the hfiles of region are properly long-term archived as
65   * specified via the {@link ZKTableArchiveClient}.
66   */
67  @Category(MediumTests.class)
68  public class TestZooKeeperTableArchiveClient {
69  
70    private static final Log LOG = LogFactory.getLog(TestZooKeeperTableArchiveClient.class);
71    private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU();
72    private static final String STRING_TABLE_NAME = "test";
73    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
74    private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
75    private static ZKTableArchiveClient archivingClient;
76    private final List<Path> toCleanup = new ArrayList<Path>();
77    private static ClusterConnection CONNECTION;
78  
79    /**
80     * Setup the config for the cluster
81     */
82    @BeforeClass
83    public static void setupCluster() throws Exception {
84      setupConf(UTIL.getConfiguration());
85      UTIL.startMiniZKCluster();
86      CONNECTION = (ClusterConnection)ConnectionFactory.createConnection(UTIL.getConfiguration());
87      archivingClient = new ZKTableArchiveClient(UTIL.getConfiguration(), CONNECTION);
88      // make hfile archiving node so we can archive files
89      ZooKeeperWatcher watcher = UTIL.getZooKeeperWatcher();
90      String archivingZNode = ZKTableArchiveClient.getArchiveZNode(UTIL.getConfiguration(), watcher);
91      ZKUtil.createWithParents(watcher, archivingZNode);
92    }
93  
94    private static void setupConf(Configuration conf) {
95      // only compact with 3 files
96      conf.setInt("hbase.hstore.compaction.min", 3);
97    }
98  
99    @After
100   public void tearDown() throws Exception {
101     try {
102       FileSystem fs = UTIL.getTestFileSystem();
103       // cleanup each of the files/directories registered
104       for (Path file : toCleanup) {
105       // remove the table and archive directories
106         FSUtils.delete(fs, file, true);
107       }
108     } catch (IOException e) {
109       LOG.warn("Failure to delete archive directory", e);
110     } finally {
111       toCleanup.clear();
112     }
113     // make sure that backups are off for all tables
114     archivingClient.disableHFileBackup();
115   }
116 
117   @AfterClass
118   public static void cleanupTest() throws Exception {
119     try {
120       CONNECTION.close();
121       UTIL.shutdownMiniZKCluster();
122     } catch (Exception e) {
123       LOG.warn("problem shutting down cluster", e);
124     }
125   }
126 
127   /**
128    * Test turning on/off archiving
129    */
130   @Test (timeout=300000)
131   public void testArchivingEnableDisable() throws Exception {
132     // 1. turn on hfile backups
133     LOG.debug("----Starting archiving");
134     archivingClient.enableHFileBackupAsync(TABLE_NAME);
135     assertTrue("Archving didn't get turned on", archivingClient
136         .getArchivingEnabled(TABLE_NAME));
137 
138     // 2. Turn off archiving and make sure its off
139     archivingClient.disableHFileBackup();
140     assertFalse("Archving didn't get turned off.", archivingClient.getArchivingEnabled(TABLE_NAME));
141 
142     // 3. Check enable/disable on a single table
143     archivingClient.enableHFileBackupAsync(TABLE_NAME);
144     assertTrue("Archving didn't get turned on", archivingClient
145         .getArchivingEnabled(TABLE_NAME));
146 
147     // 4. Turn off archiving and make sure its off
148     archivingClient.disableHFileBackup(TABLE_NAME);
149     assertFalse("Archving didn't get turned off for " + STRING_TABLE_NAME,
150       archivingClient.getArchivingEnabled(TABLE_NAME));
151   }
152 
153   @Test (timeout=300000)
154   public void testArchivingOnSingleTable() throws Exception {
155     createArchiveDirectory();
156     FileSystem fs = UTIL.getTestFileSystem();
157     Path archiveDir = getArchiveDir();
158     Path tableDir = getTableDir(STRING_TABLE_NAME);
159     toCleanup.add(archiveDir);
160     toCleanup.add(tableDir);
161 
162     Configuration conf = UTIL.getConfiguration();
163     // setup the delegate
164     Stoppable stop = new StoppableImplementation();
165     HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop);
166     List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner);
167     final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0);
168 
169     // create the region
170     HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM);
171     HRegion region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd);
172 
173     loadFlushAndCompact(region, TEST_FAM);
174 
175     // get the current hfiles in the archive directory
176     List<Path> files = getAllFiles(fs, archiveDir);
177     if (files == null) {
178       FSUtils.logFileSystemState(fs, UTIL.getDataTestDir(), LOG);
179       throw new RuntimeException("Didn't archive any files!");
180     }
181     CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size());
182 
183     runCleaner(cleaner, finished, stop);
184 
185     // know the cleaner ran, so now check all the files again to make sure they are still there
186     List<Path> archivedFiles = getAllFiles(fs, archiveDir);
187     assertEquals("Archived files changed after running archive cleaner.", files, archivedFiles);
188 
189     // but we still have the archive directory
190     assertTrue(fs.exists(HFileArchiveUtil.getArchivePath(UTIL.getConfiguration())));
191   }
192 
193   /**
194    * Test archiving/cleaning across multiple tables, where some are retained, and others aren't
195    * @throws Exception on failure
196    */
197   @Test (timeout=300000)
198   public void testMultipleTables() throws Exception {
199     createArchiveDirectory();
200     String otherTable = "otherTable";
201 
202     FileSystem fs = UTIL.getTestFileSystem();
203     Path archiveDir = getArchiveDir();
204     Path tableDir = getTableDir(STRING_TABLE_NAME);
205     Path otherTableDir = getTableDir(otherTable);
206 
207     // register cleanup for the created directories
208     toCleanup.add(archiveDir);
209     toCleanup.add(tableDir);
210     toCleanup.add(otherTableDir);
211     Configuration conf = UTIL.getConfiguration();
212     // setup the delegate
213     Stoppable stop = new StoppableImplementation();
214     HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop);
215     List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner);
216     final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0);
217 
218     // create the region
219     HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM);
220     HRegion region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd);
221     loadFlushAndCompact(region, TEST_FAM);
222 
223     // create the another table that we don't archive
224     hcd = new HColumnDescriptor(TEST_FAM);
225     HRegion otherRegion = UTIL.createTestRegion(otherTable, hcd);
226     loadFlushAndCompact(otherRegion, TEST_FAM);
227 
228     // get the current hfiles in the archive directory
229     List<Path> files = getAllFiles(fs, archiveDir);
230     if (files == null) {
231       FSUtils.logFileSystemState(fs, archiveDir, LOG);
232       throw new RuntimeException("Didn't load archive any files!");
233     }
234 
235     // make sure we have files from both tables
236     int initialCountForPrimary = 0;
237     int initialCountForOtherTable = 0;
238     for (Path file : files) {
239       String tableName = file.getParent().getParent().getParent().getName();
240       // check to which table this file belongs
241       if (tableName.equals(otherTable)) initialCountForOtherTable++;
242       else if (tableName.equals(STRING_TABLE_NAME)) initialCountForPrimary++;
243     }
244 
245     assertTrue("Didn't archive files for:" + STRING_TABLE_NAME, initialCountForPrimary > 0);
246     assertTrue("Didn't archive files for:" + otherTable, initialCountForOtherTable > 0);
247 
248     // run the cleaners, checking for each of the directories + files (both should be deleted and
249     // need to be checked) in 'otherTable' and the files (which should be retained) in the 'table'
250     CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size() + 3);
251     // run the cleaner
252     cleaner.start();
253     // wait for the cleaner to check all the files
254     finished.await();
255     // stop the cleaner
256     stop.stop("");
257 
258     // know the cleaner ran, so now check all the files again to make sure they are still there
259     List<Path> archivedFiles = getAllFiles(fs, archiveDir);
260     int archivedForPrimary = 0;
261     for(Path file: archivedFiles) {
262       String tableName = file.getParent().getParent().getParent().getName();
263       // ensure we don't have files from the non-archived table
264       assertFalse("Have a file from the non-archived table: " + file, tableName.equals(otherTable));
265       if (tableName.equals(STRING_TABLE_NAME)) archivedForPrimary++;
266     }
267 
268     assertEquals("Not all archived files for the primary table were retained.", initialCountForPrimary,
269       archivedForPrimary);
270 
271     // but we still have the archive directory
272     assertTrue("Archive directory was deleted via archiver", fs.exists(archiveDir));
273   }
274 
275 
276   private void createArchiveDirectory() throws IOException {
277     //create the archive and test directory
278     FileSystem fs = UTIL.getTestFileSystem();
279     Path archiveDir = getArchiveDir();
280     fs.mkdirs(archiveDir);
281   }
282 
283   private Path getArchiveDir() throws IOException {
284     return new Path(UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
285   }
286 
287   private Path getTableDir(String tableName) throws IOException {
288     Path testDataDir = UTIL.getDataTestDir();
289     FSUtils.setRootDir(UTIL.getConfiguration(), testDataDir);
290     return new Path(testDataDir, tableName);
291   }
292 
293   private HFileCleaner setupAndCreateCleaner(Configuration conf, FileSystem fs, Path archiveDir,
294       Stoppable stop) {
295     conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
296       LongTermArchivingHFileCleaner.class.getCanonicalName());
297     return new HFileCleaner(1000, stop, conf, fs, archiveDir);
298   }
299 
300   /**
301    * Start archiving table for given hfile cleaner
302    * @param tableName table to archive
303    * @param cleaner cleaner to check to make sure change propagated
304    * @return underlying {@link LongTermArchivingHFileCleaner} that is managing archiving
305    * @throws IOException on failure
306    * @throws KeeperException on failure
307    */
308   private List<BaseHFileCleanerDelegate> turnOnArchiving(String tableName, HFileCleaner cleaner)
309       throws IOException, KeeperException {
310     // turn on hfile retention
311     LOG.debug("----Starting archiving for table:" + tableName);
312     archivingClient.enableHFileBackupAsync(Bytes.toBytes(tableName));
313     assertTrue("Archving didn't get turned on", archivingClient.getArchivingEnabled(tableName));
314 
315     // wait for the archiver to get the notification
316     List<BaseHFileCleanerDelegate> cleaners = cleaner.getDelegatesForTesting();
317     LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0);
318     while (!delegate.archiveTracker.keepHFiles(STRING_TABLE_NAME)) {
319       // spin until propagation - should be fast
320     }
321     return cleaners;
322   }
323 
324   /**
325    * Spy on the {@link LongTermArchivingHFileCleaner} to ensure we can catch when the cleaner has
326    * seen all the files
327    * @return a {@link CountDownLatch} to wait on that releases when the cleaner has been called at
328    *         least the expected number of times.
329    */
330   private CountDownLatch setupCleanerWatching(LongTermArchivingHFileCleaner cleaner,
331       List<BaseHFileCleanerDelegate> cleaners, final int expected) {
332     // replace the cleaner with one that we can can check
333     BaseHFileCleanerDelegate delegateSpy = Mockito.spy(cleaner);
334     final int[] counter = new int[] { 0 };
335     final CountDownLatch finished = new CountDownLatch(1);
336     Mockito.doAnswer(new Answer<Iterable<FileStatus>>() {
337 
338       @Override
339       public Iterable<FileStatus> answer(InvocationOnMock invocation) throws Throwable {
340         counter[0]++;
341         LOG.debug(counter[0] + "/ " + expected + ") Wrapping call to getDeletableFiles for files: "
342             + invocation.getArguments()[0]);
343 
344         @SuppressWarnings("unchecked")
345         Iterable<FileStatus> ret = (Iterable<FileStatus>) invocation.callRealMethod();
346         if (counter[0] >= expected) finished.countDown();
347         return ret;
348       }
349     }).when(delegateSpy).getDeletableFiles(Mockito.anyListOf(FileStatus.class));
350     cleaners.set(0, delegateSpy);
351 
352     return finished;
353   }
354 
355   /**
356    * Get all the files (non-directory entries) in the file system under the passed directory
357    * @param dir directory to investigate
358    * @return all files under the directory
359    */
360   private List<Path> getAllFiles(FileSystem fs, Path dir) throws IOException {
361     FileStatus[] files = FSUtils.listStatus(fs, dir, null);
362     if (files == null) {
363       LOG.warn("No files under:" + dir);
364       return null;
365     }
366 
367     List<Path> allFiles = new ArrayList<Path>();
368     for (FileStatus file : files) {
369       if (file.isDirectory()) {
370         List<Path> subFiles = getAllFiles(fs, file.getPath());
371         if (subFiles != null) allFiles.addAll(subFiles);
372         continue;
373       }
374       allFiles.add(file.getPath());
375     }
376     return allFiles;
377   }
378 
379   private void loadFlushAndCompact(HRegion region, byte[] family) throws IOException {
380     // create two hfiles in the region
381     createHFileInRegion(region, family);
382     createHFileInRegion(region, family);
383 
384     Store s = region.getStore(family);
385     int count = s.getStorefilesCount();
386     assertTrue("Don't have the expected store files, wanted >= 2 store files, but was:" + count,
387       count >= 2);
388 
389     // compact the two files into one file to get files in the archive
390     LOG.debug("Compacting stores");
391     region.compactStores(true);
392   }
393 
394   /**
395    * Create a new hfile in the passed region
396    * @param region region to operate on
397    * @param columnFamily family for which to add data
398    * @throws IOException
399    */
400   private void createHFileInRegion(HRegion region, byte[] columnFamily) throws IOException {
401     // put one row in the region
402     Put p = new Put(Bytes.toBytes("row"));
403     p.add(columnFamily, Bytes.toBytes("Qual"), Bytes.toBytes("v1"));
404     region.put(p);
405     // flush the region to make a store file
406     region.flushcache();
407   }
408 
409   /**
410    * @param cleaner
411    */
412   private void runCleaner(HFileCleaner cleaner, CountDownLatch finished, Stoppable stop)
413       throws InterruptedException {
414     // run the cleaner
415     cleaner.start();
416     // wait for the cleaner to check all the files
417     finished.await();
418     // stop the cleaner
419     stop.stop("");
420   }
421 }