1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.master.snapshot;
19
20 import static org.junit.Assert.assertFalse;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.Collection;
25 import java.util.HashSet;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.fs.FileSystem;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.HBaseTestingUtility;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.MediumTests;
34 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
35 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
36 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
37 import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
38 import org.apache.hadoop.hbase.util.FSUtils;
39 import org.junit.After;
40 import org.junit.AfterClass;
41 import org.junit.BeforeClass;
42 import org.junit.Test;
43 import org.junit.experimental.categories.Category;
44
45
46
47
48 @Category(MediumTests.class)
49 public class TestSnapshotFileCache {
50
51 private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class);
52 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
53 private static FileSystem fs;
54 private static Path rootDir;
55
56 @BeforeClass
57 public static void startCluster() throws Exception {
58 UTIL.startMiniDFSCluster(1);
59 fs = UTIL.getDFSCluster().getFileSystem();
60 rootDir = UTIL.getDefaultRootDirPath();
61 }
62
63 @AfterClass
64 public static void stopCluster() throws Exception {
65 UTIL.shutdownMiniDFSCluster();
66 }
67
68 @After
69 public void cleanupFiles() throws Exception {
70
71 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
72 fs.delete(snapshotDir, true);
73 }
74
75 @Test(timeout = 10000000)
76 public void testLoadAndDelete() throws Exception {
77
78 long period = Long.MAX_VALUE;
79 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
80 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
81 "test-snapshot-file-cache-refresh", new SnapshotFiles());
82
83 Path snapshot = new Path(snapshotDir, "snapshot");
84 Path region = new Path(snapshot, "7e91021");
85 Path family = new Path(region, "fam");
86 Path file1 = new Path(family, "file1");
87 Path file2 = new Path(family, "file2");
88
89
90 fs.create(file1);
91 fs.create(file2);
92
93 FSUtils.logFileSystemState(fs, rootDir, LOG);
94
95
96 assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
97 assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
98 String not = "file-shouldn't-be-found";
99 assertFalse("Cache found '" + not + "', but it shouldn't have.", cache.contains(not));
100
101
102
103 Thread.sleep(10);
104
105 LOG.debug("Deleting snapshot.");
106
107 if (!fs.delete(snapshot, true)) {
108 throw new IOException("Couldn't delete " + snapshot + " for an unknown reason.");
109 }
110 FSUtils.logFileSystemState(fs, rootDir, LOG);
111
112
113 LOG.debug("Checking to see if file is deleted.");
114 assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
115 assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
116
117
118 cache.triggerCacheRefreshForTesting();
119
120 assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
121 cache.contains(file1.getName()));
122 assertFalse("Cache found '" + file2 + "', but it shouldn't have.",
123 cache.contains(file2.getName()));
124
125 fs.delete(snapshotDir, true);
126 }
127
128 @Test
129 public void testLoadsTmpDir() throws Exception {
130
131 long period = Long.MAX_VALUE;
132 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
133 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
134 "test-snapshot-file-cache-refresh", new SnapshotFiles());
135
136
137 Path snapshot = new Path(snapshotDir, "snapshot");
138 Path region = new Path(snapshot, "7e91021");
139 Path family = new Path(region, "fam");
140 Path file1 = new Path(family, "file1");
141 fs.create(file1);
142
143
144 SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
145 snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
146 region = new Path(snapshot, "7e91021");
147 family = new Path(region, "fam");
148 Path file2 = new Path(family, "file2");
149 fs.create(file2);
150
151 FSUtils.logFileSystemState(fs, rootDir, LOG);
152
153
154 assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
155 assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
156 }
157
158 @Test
159 public void testJustFindLogsDirectory() throws Exception {
160
161 long period = Long.MAX_VALUE;
162 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
163 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
164 "test-snapshot-file-cache-refresh", new SnapshotFileCache.SnapshotFileInspector() {
165 public Collection<String> filesUnderSnapshot(final Path snapshotDir)
166 throws IOException {
167 return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
168 }
169 });
170
171
172 Path snapshot = new Path(snapshotDir, "snapshot");
173 Path region = new Path(snapshot, "7e91021");
174 Path family = new Path(region, "fam");
175 Path file1 = new Path(family, "file1");
176 fs.create(file1);
177
178
179 Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server");
180 Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552");
181 fs.create(log);
182
183 FSUtils.logFileSystemState(fs, rootDir, LOG);
184
185
186 assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
187 cache.contains(file1.getName()));
188 assertTrue("Cache didn't find:" + log, cache.contains(log.getName()));
189 }
190
191 @Test
192 public void testReloadModifiedDirectory() throws IOException {
193
194 long period = Long.MAX_VALUE;
195 Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
196 SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
197 "test-snapshot-file-cache-refresh", new SnapshotFiles());
198
199 Path snapshot = new Path(snapshotDir, "snapshot");
200 Path region = new Path(snapshot, "7e91021");
201 Path family = new Path(region, "fam");
202 Path file1 = new Path(family, "file1");
203 Path file2 = new Path(family, "file2");
204
205
206 fs.create(file1);
207 fs.create(file2);
208
209 FSUtils.logFileSystemState(fs, rootDir, LOG);
210
211 assertTrue("Cache didn't find " + file1, cache.contains(file1.getName()));
212
213
214 fs.delete(snapshot, true);
215 Path file3 = new Path(family, "new_file");
216 fs.create(file3);
217
218 FSUtils.logFileSystemState(fs, rootDir, LOG);
219 assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
220 }
221
222 class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
223 public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
224 Collection<String> files = new HashSet<String>();
225 files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir));
226 files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir));
227 return files;
228 }
229 };
230 }