1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.regex.Matcher;
24 import java.util.regex.Pattern;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileStatus;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
34 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
35 import org.apache.hadoop.hbase.io.HFileLink;
36 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
37 import org.apache.hadoop.hbase.io.Reference;
38 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
40 import org.apache.hadoop.hbase.util.FSUtils;
41
42
43
44
45 @InterfaceAudience.Private
46 public class StoreFileInfo {
47 public static final Log LOG = LogFactory.getLog(StoreFileInfo.class);
48
49
50
51
52
53 public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:_SeqId_[0-9]+_)?";
54
55
56 private static final Pattern HFILE_NAME_PATTERN =
57 Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
58
59
60
61
62
63
64
65
66 private static final Pattern REF_NAME_PATTERN =
67 Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
68 HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
69
70
71 private HDFSBlocksDistribution hdfsBlocksDistribution = null;
72
73
74 private final Reference reference;
75
76
77 private final HFileLink link;
78
79
80 private final FileStatus fileStatus;
81
82
83
84
85
86
87
88 public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path path)
89 throws IOException {
90 this(conf, fs, fs.getFileStatus(path));
91 }
92
93
94
95
96
97
98
99 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
100 throws IOException {
101 this.fileStatus = fileStatus;
102 Path p = fileStatus.getPath();
103 if (HFileLink.isHFileLink(p)) {
104
105 this.reference = null;
106 this.link = new HFileLink(conf, p);
107 if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
108 } else if (isReference(p)) {
109 this.reference = Reference.read(fs, p);
110 Path referencePath = getReferredToFile(p);
111 if (HFileLink.isHFileLink(referencePath)) {
112
113 this.link = new HFileLink(conf, referencePath);
114 } else {
115
116 this.link = null;
117 }
118 if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
119 " reference to " + referencePath);
120 } else if (isHFile(p)) {
121
122 this.reference = null;
123 this.link = null;
124 } else {
125 throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
126 }
127 }
128
129
130 public boolean isReference() {
131 return this.reference != null;
132 }
133
134
135 public boolean isTopReference() {
136 return this.reference != null && Reference.isTopFileRegion(this.reference.getFileRegion());
137 }
138
139
140 public boolean isLink() {
141 return this.link != null && this.reference == null;
142 }
143
144
145 public HDFSBlocksDistribution getHDFSBlockDistribution() {
146 return this.hdfsBlocksDistribution;
147 }
148
149
150
151
152
153
154
155
156 public StoreFile.Reader open(final FileSystem fs, final CacheConfig cacheConf,
157 final DataBlockEncoding dataBlockEncoding) throws IOException {
158 FSDataInputStreamWrapper in;
159 FileStatus status;
160
161 if (this.link != null) {
162
163 in = new FSDataInputStreamWrapper(fs, this.link);
164 status = this.link.getFileStatus(fs);
165 } else if (this.reference != null) {
166
167 Path referencePath = getReferredToFile(this.getPath());
168 in = new FSDataInputStreamWrapper(fs, referencePath);
169 status = fs.getFileStatus(referencePath);
170 } else {
171 in = new FSDataInputStreamWrapper(fs, this.getPath());
172 status = fileStatus;
173 }
174 long length = status.getLen();
175 if (this.reference != null) {
176 hdfsBlocksDistribution = computeRefFileHDFSBlockDistribution(fs, reference, status);
177 return new HalfStoreFileReader(
178 fs, this.getPath(), in, length, cacheConf, reference, dataBlockEncoding);
179 } else {
180 hdfsBlocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, length);
181 return new StoreFile.Reader(fs, this.getPath(), in, length, cacheConf, dataBlockEncoding);
182 }
183 }
184
185
186
187
188 public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
189 throws IOException {
190 FileStatus status;
191 if (this.reference != null) {
192 if (this.link != null) {
193
194 status = link.getFileStatus(fs);
195 } else {
196
197 Path referencePath = getReferredToFile(this.getPath());
198 status = fs.getFileStatus(referencePath);
199 }
200 return computeRefFileHDFSBlockDistribution(fs, reference, status);
201 } else {
202 if (this.link != null) {
203
204 status = link.getFileStatus(fs);
205 } else {
206 status = this.fileStatus;
207 }
208 return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
209 }
210 }
211
212
213 public Path getPath() {
214 return this.fileStatus.getPath();
215 }
216
217
218 public FileStatus getFileStatus() {
219 return this.fileStatus;
220 }
221
222
223 public long getModificationTime() {
224 return this.fileStatus.getModificationTime();
225 }
226
227 @Override
228 public String toString() {
229 return this.getPath() +
230 (isReference() ? "-" + getReferredToFile(this.getPath()) + "-" + reference : "");
231 }
232
233
234
235
236
237 public static boolean isHFile(final Path path) {
238 return isHFile(path.getName());
239 }
240
241 public static boolean isHFile(final String fileName) {
242 Matcher m = HFILE_NAME_PATTERN.matcher(fileName);
243 return m.matches() && m.groupCount() > 0;
244 }
245
246
247
248
249
250 public static boolean isReference(final Path path) {
251 return isReference(path.getName());
252 }
253
254
255
256
257
258 public static boolean isReference(final String name) {
259 Matcher m = REF_NAME_PATTERN.matcher(name);
260 return m.matches() && m.groupCount() > 1;
261 }
262
263
264
265
266
267
268
269
270 public static Path getReferredToFile(final Path p) {
271 Matcher m = REF_NAME_PATTERN.matcher(p.getName());
272 if (m == null || !m.matches()) {
273 LOG.warn("Failed match of store file name " + p.toString());
274 throw new IllegalArgumentException("Failed match of store file name " +
275 p.toString());
276 }
277
278
279 String otherRegion = m.group(2);
280
281 Path tableDir = p.getParent().getParent().getParent();
282 String nameStrippedOfSuffix = m.group(1);
283 LOG.debug("reference '" + p + "' to region=" + otherRegion + " hfile=" + nameStrippedOfSuffix);
284
285
286
287 return new Path(new Path(new Path(tableDir, otherRegion),
288 p.getParent().getName()), nameStrippedOfSuffix);
289 }
290
291
292
293
294
295
296 public static boolean validateStoreFileName(final String fileName) {
297 if (HFileLink.isHFileLink(fileName) || isReference(fileName))
298 return(true);
299 return !fileName.contains("-");
300 }
301
302
303
304
305
306
307 public static boolean isValid(final FileStatus fileStatus)
308 throws IOException {
309 final Path p = fileStatus.getPath();
310
311 if (fileStatus.isDir())
312 return false;
313
314
315
316
317 if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
318 LOG.warn("Skipping " + p + " beccreateStoreDirause its empty. HBASE-646 DATA LOSS?");
319 return false;
320 }
321
322 return validateStoreFileName(p.getName());
323 }
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338 private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
339 final FileSystem fs, final Reference reference, final FileStatus status)
340 throws IOException {
341 if (status == null) {
342 return null;
343 }
344
345 long start = 0;
346 long length = 0;
347
348 if (Reference.isTopFileRegion(reference.getFileRegion())) {
349 start = status.getLen()/2;
350 length = status.getLen() - status.getLen()/2;
351 } else {
352 start = 0;
353 length = status.getLen()/2;
354 }
355 return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
356 }
357 }