1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.backup.example;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.concurrent.CountDownLatch;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileStatus;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.testclassification.MediumTests;
39 import org.apache.hadoop.hbase.Stoppable;
40 import org.apache.hadoop.hbase.client.ClusterConnection;
41 import org.apache.hadoop.hbase.client.ConnectionFactory;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.master.cleaner.BaseHFileCleanerDelegate;
44 import org.apache.hadoop.hbase.master.cleaner.HFileCleaner;
45 import org.apache.hadoop.hbase.regionserver.HRegion;
46 import org.apache.hadoop.hbase.regionserver.Store;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.hadoop.hbase.util.FSUtils;
49 import org.apache.hadoop.hbase.util.HFileArchiveUtil;
50 import org.apache.hadoop.hbase.util.StoppableImplementation;
51 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
52 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
53 import org.apache.zookeeper.KeeperException;
54 import org.junit.After;
55 import org.junit.AfterClass;
56 import org.junit.BeforeClass;
57 import org.junit.Test;
58 import org.junit.experimental.categories.Category;
59 import org.mockito.Mockito;
60 import org.mockito.invocation.InvocationOnMock;
61 import org.mockito.stubbing.Answer;
62
63
64
65
66
67 @Category(MediumTests.class)
68 public class TestZooKeeperTableArchiveClient {
69
70 private static final Log LOG = LogFactory.getLog(TestZooKeeperTableArchiveClient.class);
71 private static final HBaseTestingUtility UTIL = HBaseTestingUtility.createLocalHTU();
72 private static final String STRING_TABLE_NAME = "test";
73 private static final byte[] TEST_FAM = Bytes.toBytes("fam");
74 private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
75 private static ZKTableArchiveClient archivingClient;
76 private final List<Path> toCleanup = new ArrayList<Path>();
77 private static ClusterConnection CONNECTION;
78
79
80
81
82 @BeforeClass
83 public static void setupCluster() throws Exception {
84 setupConf(UTIL.getConfiguration());
85 UTIL.startMiniZKCluster();
86 CONNECTION = (ClusterConnection)ConnectionFactory.createConnection(UTIL.getConfiguration());
87 archivingClient = new ZKTableArchiveClient(UTIL.getConfiguration(), CONNECTION);
88
89 ZooKeeperWatcher watcher = UTIL.getZooKeeperWatcher();
90 String archivingZNode = ZKTableArchiveClient.getArchiveZNode(UTIL.getConfiguration(), watcher);
91 ZKUtil.createWithParents(watcher, archivingZNode);
92 }
93
94 private static void setupConf(Configuration conf) {
95
96 conf.setInt("hbase.hstore.compaction.min", 3);
97 }
98
99 @After
100 public void tearDown() throws Exception {
101 try {
102 FileSystem fs = UTIL.getTestFileSystem();
103
104 for (Path file : toCleanup) {
105
106 FSUtils.delete(fs, file, true);
107 }
108 } catch (IOException e) {
109 LOG.warn("Failure to delete archive directory", e);
110 } finally {
111 toCleanup.clear();
112 }
113
114 archivingClient.disableHFileBackup();
115 }
116
117 @AfterClass
118 public static void cleanupTest() throws Exception {
119 try {
120 CONNECTION.close();
121 UTIL.shutdownMiniZKCluster();
122 } catch (Exception e) {
123 LOG.warn("problem shutting down cluster", e);
124 }
125 }
126
127
128
129
130 @Test (timeout=300000)
131 public void testArchivingEnableDisable() throws Exception {
132
133 LOG.debug("----Starting archiving");
134 archivingClient.enableHFileBackupAsync(TABLE_NAME);
135 assertTrue("Archving didn't get turned on", archivingClient
136 .getArchivingEnabled(TABLE_NAME));
137
138
139 archivingClient.disableHFileBackup();
140 assertFalse("Archving didn't get turned off.", archivingClient.getArchivingEnabled(TABLE_NAME));
141
142
143 archivingClient.enableHFileBackupAsync(TABLE_NAME);
144 assertTrue("Archving didn't get turned on", archivingClient
145 .getArchivingEnabled(TABLE_NAME));
146
147
148 archivingClient.disableHFileBackup(TABLE_NAME);
149 assertFalse("Archving didn't get turned off for " + STRING_TABLE_NAME,
150 archivingClient.getArchivingEnabled(TABLE_NAME));
151 }
152
153 @Test (timeout=300000)
154 public void testArchivingOnSingleTable() throws Exception {
155 createArchiveDirectory();
156 FileSystem fs = UTIL.getTestFileSystem();
157 Path archiveDir = getArchiveDir();
158 Path tableDir = getTableDir(STRING_TABLE_NAME);
159 toCleanup.add(archiveDir);
160 toCleanup.add(tableDir);
161
162 Configuration conf = UTIL.getConfiguration();
163
164 Stoppable stop = new StoppableImplementation();
165 HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop);
166 List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner);
167 final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0);
168
169
170 HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM);
171 HRegion region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd);
172
173 loadFlushAndCompact(region, TEST_FAM);
174
175
176 List<Path> files = getAllFiles(fs, archiveDir);
177 if (files == null) {
178 FSUtils.logFileSystemState(fs, UTIL.getDataTestDir(), LOG);
179 throw new RuntimeException("Didn't archive any files!");
180 }
181 CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size());
182
183 runCleaner(cleaner, finished, stop);
184
185
186 List<Path> archivedFiles = getAllFiles(fs, archiveDir);
187 assertEquals("Archived files changed after running archive cleaner.", files, archivedFiles);
188
189
190 assertTrue(fs.exists(HFileArchiveUtil.getArchivePath(UTIL.getConfiguration())));
191 }
192
193
194
195
196
197 @Test (timeout=300000)
198 public void testMultipleTables() throws Exception {
199 createArchiveDirectory();
200 String otherTable = "otherTable";
201
202 FileSystem fs = UTIL.getTestFileSystem();
203 Path archiveDir = getArchiveDir();
204 Path tableDir = getTableDir(STRING_TABLE_NAME);
205 Path otherTableDir = getTableDir(otherTable);
206
207
208 toCleanup.add(archiveDir);
209 toCleanup.add(tableDir);
210 toCleanup.add(otherTableDir);
211 Configuration conf = UTIL.getConfiguration();
212
213 Stoppable stop = new StoppableImplementation();
214 HFileCleaner cleaner = setupAndCreateCleaner(conf, fs, archiveDir, stop);
215 List<BaseHFileCleanerDelegate> cleaners = turnOnArchiving(STRING_TABLE_NAME, cleaner);
216 final LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0);
217
218
219 HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM);
220 HRegion region = UTIL.createTestRegion(STRING_TABLE_NAME, hcd);
221 loadFlushAndCompact(region, TEST_FAM);
222
223
224 hcd = new HColumnDescriptor(TEST_FAM);
225 HRegion otherRegion = UTIL.createTestRegion(otherTable, hcd);
226 loadFlushAndCompact(otherRegion, TEST_FAM);
227
228
229 List<Path> files = getAllFiles(fs, archiveDir);
230 if (files == null) {
231 FSUtils.logFileSystemState(fs, archiveDir, LOG);
232 throw new RuntimeException("Didn't load archive any files!");
233 }
234
235
236 int initialCountForPrimary = 0;
237 int initialCountForOtherTable = 0;
238 for (Path file : files) {
239 String tableName = file.getParent().getParent().getParent().getName();
240
241 if (tableName.equals(otherTable)) initialCountForOtherTable++;
242 else if (tableName.equals(STRING_TABLE_NAME)) initialCountForPrimary++;
243 }
244
245 assertTrue("Didn't archive files for:" + STRING_TABLE_NAME, initialCountForPrimary > 0);
246 assertTrue("Didn't archive files for:" + otherTable, initialCountForOtherTable > 0);
247
248
249
250 CountDownLatch finished = setupCleanerWatching(delegate, cleaners, files.size() + 3);
251
252 cleaner.start();
253
254 finished.await();
255
256 stop.stop("");
257
258
259 List<Path> archivedFiles = getAllFiles(fs, archiveDir);
260 int archivedForPrimary = 0;
261 for(Path file: archivedFiles) {
262 String tableName = file.getParent().getParent().getParent().getName();
263
264 assertFalse("Have a file from the non-archived table: " + file, tableName.equals(otherTable));
265 if (tableName.equals(STRING_TABLE_NAME)) archivedForPrimary++;
266 }
267
268 assertEquals("Not all archived files for the primary table were retained.", initialCountForPrimary,
269 archivedForPrimary);
270
271
272 assertTrue("Archive directory was deleted via archiver", fs.exists(archiveDir));
273 }
274
275
276 private void createArchiveDirectory() throws IOException {
277
278 FileSystem fs = UTIL.getTestFileSystem();
279 Path archiveDir = getArchiveDir();
280 fs.mkdirs(archiveDir);
281 }
282
283 private Path getArchiveDir() throws IOException {
284 return new Path(UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
285 }
286
287 private Path getTableDir(String tableName) throws IOException {
288 Path testDataDir = UTIL.getDataTestDir();
289 FSUtils.setRootDir(UTIL.getConfiguration(), testDataDir);
290 return new Path(testDataDir, tableName);
291 }
292
293 private HFileCleaner setupAndCreateCleaner(Configuration conf, FileSystem fs, Path archiveDir,
294 Stoppable stop) {
295 conf.setStrings(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS,
296 LongTermArchivingHFileCleaner.class.getCanonicalName());
297 return new HFileCleaner(1000, stop, conf, fs, archiveDir);
298 }
299
300
301
302
303
304
305
306
307
308 private List<BaseHFileCleanerDelegate> turnOnArchiving(String tableName, HFileCleaner cleaner)
309 throws IOException, KeeperException {
310
311 LOG.debug("----Starting archiving for table:" + tableName);
312 archivingClient.enableHFileBackupAsync(Bytes.toBytes(tableName));
313 assertTrue("Archving didn't get turned on", archivingClient.getArchivingEnabled(tableName));
314
315
316 List<BaseHFileCleanerDelegate> cleaners = cleaner.getDelegatesForTesting();
317 LongTermArchivingHFileCleaner delegate = (LongTermArchivingHFileCleaner) cleaners.get(0);
318 while (!delegate.archiveTracker.keepHFiles(STRING_TABLE_NAME)) {
319
320 }
321 return cleaners;
322 }
323
324
325
326
327
328
329
330 private CountDownLatch setupCleanerWatching(LongTermArchivingHFileCleaner cleaner,
331 List<BaseHFileCleanerDelegate> cleaners, final int expected) {
332
333 BaseHFileCleanerDelegate delegateSpy = Mockito.spy(cleaner);
334 final int[] counter = new int[] { 0 };
335 final CountDownLatch finished = new CountDownLatch(1);
336 Mockito.doAnswer(new Answer<Iterable<FileStatus>>() {
337
338 @Override
339 public Iterable<FileStatus> answer(InvocationOnMock invocation) throws Throwable {
340 counter[0]++;
341 LOG.debug(counter[0] + "/ " + expected + ") Wrapping call to getDeletableFiles for files: "
342 + invocation.getArguments()[0]);
343
344 @SuppressWarnings("unchecked")
345 Iterable<FileStatus> ret = (Iterable<FileStatus>) invocation.callRealMethod();
346 if (counter[0] >= expected) finished.countDown();
347 return ret;
348 }
349 }).when(delegateSpy).getDeletableFiles(Mockito.anyListOf(FileStatus.class));
350 cleaners.set(0, delegateSpy);
351
352 return finished;
353 }
354
355
356
357
358
359
360 private List<Path> getAllFiles(FileSystem fs, Path dir) throws IOException {
361 FileStatus[] files = FSUtils.listStatus(fs, dir, null);
362 if (files == null) {
363 LOG.warn("No files under:" + dir);
364 return null;
365 }
366
367 List<Path> allFiles = new ArrayList<Path>();
368 for (FileStatus file : files) {
369 if (file.isDirectory()) {
370 List<Path> subFiles = getAllFiles(fs, file.getPath());
371 if (subFiles != null) allFiles.addAll(subFiles);
372 continue;
373 }
374 allFiles.add(file.getPath());
375 }
376 return allFiles;
377 }
378
379 private void loadFlushAndCompact(HRegion region, byte[] family) throws IOException {
380
381 createHFileInRegion(region, family);
382 createHFileInRegion(region, family);
383
384 Store s = region.getStore(family);
385 int count = s.getStorefilesCount();
386 assertTrue("Don't have the expected store files, wanted >= 2 store files, but was:" + count,
387 count >= 2);
388
389
390 LOG.debug("Compacting stores");
391 region.compactStores(true);
392 }
393
394
395
396
397
398
399
400 private void createHFileInRegion(HRegion region, byte[] columnFamily) throws IOException {
401
402 Put p = new Put(Bytes.toBytes("row"));
403 p.add(columnFamily, Bytes.toBytes("Qual"), Bytes.toBytes("v1"));
404 region.put(p);
405
406 region.flushcache();
407 }
408
409
410
411
412 private void runCleaner(HFileCleaner cleaner, CountDownLatch finished, Stoppable stop)
413 throws InterruptedException {
414
415 cleaner.start();
416
417 finished.await();
418
419 stop.stop("");
420 }
421 }