1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26
27 import java.io.File;
28 import java.io.IOException;
29 import java.util.UUID;
30 import java.util.Set;
31 import java.util.HashSet;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FSDataInputStream;
37 import org.apache.hadoop.fs.FSDataOutputStream;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.fs.permission.FsPermission;
42 import org.apache.hadoop.hbase.HBaseConfiguration;
43 import org.apache.hadoop.hbase.HBaseTestingUtility;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
46 import org.apache.hadoop.hbase.MediumTests;
47 import org.apache.hadoop.hbase.regionserver.wal.HLog;
48 import org.apache.hadoop.hbase.util.MD5Hash;
49 import org.apache.hadoop.hbase.util.FSUtils;
50 import org.junit.*;
51 import org.junit.experimental.categories.Category;
52
53
54
55
56 @Category(MediumTests.class)
57 public class TestFSVisitor {
58 final Log LOG = LogFactory.getLog(getClass());
59
60 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
61
62 private final String TABLE_NAME = "testtb";
63
64 private Set<String> tableFamilies;
65 private Set<String> tableRegions;
66 private Set<String> recoveredEdits;
67 private Set<String> tableHFiles;
68 private Set<String> regionServers;
69 private Set<String> serverLogs;
70
71 private FileSystem fs;
72 private Path tableDir;
73 private Path logsDir;
74 private Path rootDir;
75
76 @Before
77 public void setUp() throws Exception {
78 fs = FileSystem.get(TEST_UTIL.getConfiguration());
79 rootDir = TEST_UTIL.getDataTestDir("hbase");
80 logsDir = new Path(rootDir, HConstants.HREGION_LOGDIR_NAME);
81
82 tableFamilies = new HashSet<String>();
83 tableRegions = new HashSet<String>();
84 recoveredEdits = new HashSet<String>();
85 tableHFiles = new HashSet<String>();
86 regionServers = new HashSet<String>();
87 serverLogs = new HashSet<String>();
88 tableDir = createTableFiles(rootDir, TABLE_NAME, tableRegions, tableFamilies, tableHFiles);
89 createRecoverEdits(tableDir, tableRegions, recoveredEdits);
90 createLogs(logsDir, regionServers, serverLogs);
91 FSUtils.logFileSystemState(fs, rootDir, LOG);
92 }
93
94 @After
95 public void tearDown() throws Exception {
96 fs.delete(rootDir);
97 }
98
99 @Test
100 public void testVisitStoreFiles() throws IOException {
101 final Set<String> regions = new HashSet<String>();
102 final Set<String> families = new HashSet<String>();
103 final Set<String> hfiles = new HashSet<String>();
104 FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
105 public void storeFile(final String region, final String family, final String hfileName)
106 throws IOException {
107 regions.add(region);
108 families.add(family);
109 hfiles.add(hfileName);
110 }
111 });
112 assertEquals(tableRegions, regions);
113 assertEquals(tableFamilies, families);
114 assertEquals(tableHFiles, hfiles);
115 }
116
117 @Test
118 public void testVisitRecoveredEdits() throws IOException {
119 final Set<String> regions = new HashSet<String>();
120 final Set<String> edits = new HashSet<String>();
121 FSVisitor.visitTableRecoveredEdits(fs, tableDir, new FSVisitor.RecoveredEditsVisitor() {
122 public void recoveredEdits (final String region, final String logfile)
123 throws IOException {
124 regions.add(region);
125 edits.add(logfile);
126 }
127 });
128 assertEquals(tableRegions, regions);
129 assertEquals(recoveredEdits, edits);
130 }
131
132 @Test
133 public void testVisitLogFiles() throws IOException {
134 final Set<String> servers = new HashSet<String>();
135 final Set<String> logs = new HashSet<String>();
136 FSVisitor.visitLogFiles(fs, rootDir, new FSVisitor.LogFileVisitor() {
137 public void logFile (final String server, final String logfile) throws IOException {
138 servers.add(server);
139 logs.add(logfile);
140 }
141 });
142 assertEquals(regionServers, servers);
143 assertEquals(serverLogs, logs);
144 }
145
146
147
148
149
150
151
152
153
154
155
156 private Path createTableFiles(final Path rootDir, final String tableName,
157 final Set<String> tableRegions, final Set<String> tableFamilies,
158 final Set<String> tableHFiles) throws IOException {
159 Path tableDir = new Path(rootDir, tableName);
160 for (int r = 0; r < 10; ++r) {
161 String regionName = MD5Hash.getMD5AsHex(Bytes.toBytes(r));
162 tableRegions.add(regionName);
163 Path regionDir = new Path(tableDir, regionName);
164 for (int f = 0; f < 3; ++f) {
165 String familyName = "f" + f;
166 tableFamilies.add(familyName);
167 Path familyDir = new Path(regionDir, familyName);
168 fs.mkdirs(familyDir);
169 for (int h = 0; h < 5; ++h) {
170 String hfileName = UUID.randomUUID().toString().replaceAll("-", "");
171 tableHFiles.add(hfileName);
172 fs.createNewFile(new Path(familyDir, hfileName));
173 }
174 }
175 }
176 return tableDir;
177 }
178
179
180
181
182
183
184
185
186 private void createRecoverEdits(final Path tableDir, final Set<String> tableRegions,
187 final Set<String> recoverEdits) throws IOException {
188 for (String region: tableRegions) {
189 Path regionEditsDir = HLog.getRegionDirRecoveredEditsDir(new Path(tableDir, region));
190 long seqId = System.currentTimeMillis();
191 for (int i = 0; i < 3; ++i) {
192 String editName = String.format("%019d", seqId + i);
193 recoverEdits.add(editName);
194 FSDataOutputStream stream = fs.create(new Path(regionEditsDir, editName));
195 stream.write(Bytes.toBytes("test"));
196 stream.close();
197 }
198 }
199 }
200
201
202
203
204
205
206
207
208
209 private void createLogs(final Path logDir, final Set<String> servers,
210 final Set<String> logs) throws IOException {
211 for (int s = 0; s < 7; ++s) {
212 String server = String.format("server%d,%d,%d", s, s, System.currentTimeMillis());
213 servers.add(server);
214 Path serverLogDir = new Path(logDir, server);
215 fs.mkdirs(serverLogDir);
216 for (int i = 0; i < 5; ++i) {
217 String logfile = server + '.' + i;
218 logs.add(logfile);
219 FSDataOutputStream stream = fs.create(new Path(serverLogDir, logfile));
220 stream.write(Bytes.toBytes("test"));
221 stream.close();
222 }
223 }
224 }
225 }