1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.Collection;
27 import java.util.List;
28 import java.util.Set;
29 import java.util.TreeSet;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.fs.FileStatus;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.fs.PathFilter;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.client.HBaseAdmin;
41 import org.apache.hadoop.hbase.master.HMaster;
42 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
43 import org.apache.hadoop.hbase.regionserver.HRegion;
44 import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
45 import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
46 import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.hadoop.hbase.util.FSTableDescriptors;
49 import org.apache.hadoop.hbase.util.FSUtils;
50 import org.junit.Assert;
51
52
53
54
55 public class SnapshotTestingUtils {
56
57 private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
58
59
60
61
62
63 public static void assertNoSnapshots(HBaseAdmin admin) throws IOException {
64 assertEquals("Have some previous snapshots", 0, admin.listSnapshots().size());
65 }
66
67
68
69
70
71 public static void assertOneSnapshotThatMatches(HBaseAdmin admin, HSnapshotDescription snapshot)
72 throws IOException {
73 assertOneSnapshotThatMatches(admin, snapshot.getName(), snapshot.getTable());
74 }
75
76
77
78
79
80 public static void assertOneSnapshotThatMatches(HBaseAdmin admin, SnapshotDescription snapshot)
81 throws IOException {
82 assertOneSnapshotThatMatches(admin, snapshot.getName(), snapshot.getTable());
83 }
84
85
86
87
88
89 public static List<SnapshotDescription> assertOneSnapshotThatMatches(HBaseAdmin admin,
90 String snapshotName, String tableName) throws IOException {
91
92 List<SnapshotDescription> snapshots = admin.listSnapshots();
93
94 assertEquals("Should only have 1 snapshot", 1, snapshots.size());
95 assertEquals(snapshotName, snapshots.get(0).getName());
96 assertEquals(tableName, snapshots.get(0).getTable());
97
98 return snapshots;
99 }
100
101
102
103
104
105 public static List<SnapshotDescription> assertOneSnapshotThatMatches(HBaseAdmin admin,
106 byte[] snapshot, byte[] tableName) throws IOException {
107 return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot), Bytes.toString(tableName));
108 }
109
110
111
112
113 public static void confirmSnapshotValid(SnapshotDescription snapshotDescriptor,
114 byte[] tableName, byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs,
115 boolean requireLogs, Path logsDir, Set<String> snapshotServers) throws IOException {
116 Path snapshotDir = SnapshotDescriptionUtils
117 .getCompletedSnapshotDir(snapshotDescriptor, rootDir);
118 assertTrue(fs.exists(snapshotDir));
119 Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
120 assertTrue(fs.exists(snapshotinfo));
121
122 if (requireLogs) {
123 TakeSnapshotUtils.verifyAllLogsGotReferenced(fs, logsDir, snapshotServers,
124 snapshotDescriptor, new Path(snapshotDir, HConstants.HREGION_LOGDIR_NAME));
125 }
126
127 HTableDescriptor desc = FSTableDescriptors.getTableDescriptor(fs, rootDir, tableName);
128 HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptor(fs, snapshotDir);
129 assertEquals(desc, snapshotDesc);
130
131
132 List<HRegionInfo> regions = admin.getTableRegions(tableName);
133 for (HRegionInfo info : regions) {
134 String regionName = info.getEncodedName();
135 Path regionDir = new Path(snapshotDir, regionName);
136 HRegionInfo snapshotRegionInfo = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
137 assertEquals(info, snapshotRegionInfo);
138
139 Path familyDir = new Path(regionDir, Bytes.toString(testFamily));
140 assertTrue("Expected to find: " + familyDir + ", but it doesn't exist", fs.exists(familyDir));
141
142 assertTrue(fs.listStatus(familyDir).length > 0);
143 }
144 }
145
146
147
148
149
150
151
152
153
154 public static void waitForSnapshotToComplete(HMaster master, HSnapshotDescription snapshot,
155 long sleep) throws IOException {
156 boolean done = false;
157 while (!done) {
158 done = master.isSnapshotDone(snapshot);
159 try {
160 Thread.sleep(sleep);
161 } catch (InterruptedException e) {
162 throw new IOException(e);
163 }
164 }
165 }
166
167 public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName) throws IOException {
168 SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
169 }
170
171 public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName) throws IOException {
172
173 admin.deleteSnapshot(snapshotName);
174 assertNoSnapshots(admin);
175 }
176
177
178
179
180
181
182
183 public static void expectSnapshotDoneException(HMaster master, HSnapshotDescription snapshot,
184 Class<? extends HBaseSnapshotException> clazz) {
185 try {
186 boolean res = master.isSnapshotDone(snapshot);
187 Assert.fail("didn't fail to lookup a snapshot: res=" + res);
188 } catch (HBaseSnapshotException e) {
189 assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
190 } catch (Throwable t) {
191 Assert.fail("Threw an unexpected exception:" + t);
192 }
193 }
194
195
196
197
198
199
200
201
202 public static FileStatus[] listHFiles(final FileSystem fs, Path tableDir) throws IOException {
203
204 PathFilter regionFilter = new FSUtils.RegionDirFilter(fs);
205 PathFilter familyFilter = new FSUtils.FamilyDirFilter(fs);
206 final PathFilter fileFilter = new PathFilter() {
207 @Override
208 public boolean accept(Path file) {
209 try {
210 return fs.isFile(file);
211 } catch (IOException e) {
212 return false;
213 }
214 }
215 };
216
217 FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, regionFilter);
218
219 if (regionDirs == null || regionDirs.length == 0) return new FileStatus[0];
220
221
222 List<FileStatus> regionFiles = new ArrayList<FileStatus>(regionDirs.length);
223 for (FileStatus regionDir : regionDirs) {
224 FileStatus[] fams = FSUtils.listStatus(fs, regionDir.getPath(), familyFilter);
225
226 if (fams == null || fams.length == 0) continue;
227
228 regionFiles.addAll(SnapshotTestingUtils.getHFilesInRegion(fams, fs, fileFilter));
229 }
230 FileStatus[] files = new FileStatus[regionFiles.size()];
231 regionFiles.toArray(files);
232 return files;
233 }
234
235
236
237
238
239
240
241
242
243 public static Collection<FileStatus> getHFilesInRegion(FileStatus[] families, FileSystem fs,
244 PathFilter fileFilter) throws IOException {
245 Set<FileStatus> files = new TreeSet<FileStatus>();
246 for (FileStatus family : families) {
247
248 FileStatus[] hfiles = FSUtils.listStatus(fs, family.getPath(), fileFilter);
249
250 if (hfiles == null || hfiles.length == 0) continue;
251 files.addAll(Arrays.asList(hfiles));
252 }
253 return files;
254 }
255 }