1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.FileNotFoundException;
23 import java.io.IOException;
24 import java.util.regex.Matcher;
25 import java.util.regex.Pattern;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileStatus;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
35 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
36 import org.apache.hadoop.hbase.io.HFileLink;
37 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
38 import org.apache.hadoop.hbase.io.Reference;
39 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
40 import org.apache.hadoop.hbase.util.FSUtils;
41
42
43
44
45 @InterfaceAudience.Private
46 public class StoreFileInfo implements Comparable<StoreFileInfo> {
47 public static final Log LOG = LogFactory.getLog(StoreFileInfo.class);
48
49
50
51
52
53 public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:_SeqId_[0-9]+_)?";
54
55
56 private static final Pattern HFILE_NAME_PATTERN =
57 Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
58
59
60
61
62
63
64
65
66 private static final Pattern REF_NAME_PATTERN =
67 Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
68 HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
69
70
71 private Configuration conf;
72
73
74 private HDFSBlocksDistribution hdfsBlocksDistribution = null;
75
76
77 private final Reference reference;
78
79
80 private final HFileLink link;
81
82
83 private final FileStatus fileStatus;
84
85 private RegionCoprocessorHost coprocessorHost;
86
87
88
89
90
91
92
93 public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path path)
94 throws IOException {
95 this(conf, fs, fs.getFileStatus(path));
96 }
97
98
99
100
101
102
103
104 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
105 throws IOException {
106 this.conf = conf;
107 this.fileStatus = fileStatus;
108 Path p = fileStatus.getPath();
109 if (HFileLink.isHFileLink(p)) {
110
111 this.reference = null;
112 this.link = new HFileLink(conf, p);
113 if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
114 } else if (isReference(p)) {
115 this.reference = Reference.read(fs, p);
116 Path referencePath = getReferredToFile(p);
117 if (HFileLink.isHFileLink(referencePath)) {
118
119 this.link = new HFileLink(conf, referencePath);
120 } else {
121
122 this.link = null;
123 }
124 if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
125 " reference to " + referencePath);
126 } else if (isHFile(p)) {
127
128 this.reference = null;
129 this.link = null;
130 } else {
131 throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
132 }
133 }
134
135
136
137
138
139
140
141 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus,
142 final HFileLink link)
143 throws IOException {
144 this.conf = conf;
145 this.fileStatus = fileStatus;
146
147 this.reference = null;
148 this.link = link;
149 }
150
151
152
153
154
155 public void setRegionCoprocessorHost(RegionCoprocessorHost coprocessorHost) {
156 this.coprocessorHost = coprocessorHost;
157 }
158
159
160
161
162
163 public Reference getReference() {
164 return this.reference;
165 }
166
167
168 public boolean isReference() {
169 return this.reference != null;
170 }
171
172
173 public boolean isTopReference() {
174 return this.reference != null && Reference.isTopFileRegion(this.reference.getFileRegion());
175 }
176
177
178 public boolean isLink() {
179 return this.link != null && this.reference == null;
180 }
181
182
183 public HDFSBlocksDistribution getHDFSBlockDistribution() {
184 return this.hdfsBlocksDistribution;
185 }
186
187
188
189
190
191
192
193 public StoreFile.Reader open(final FileSystem fs,
194 final CacheConfig cacheConf) throws IOException {
195 FSDataInputStreamWrapper in;
196 FileStatus status;
197
198 if (this.link != null) {
199
200 in = new FSDataInputStreamWrapper(fs, this.link);
201 status = this.link.getFileStatus(fs);
202 } else if (this.reference != null) {
203
204 Path referencePath = getReferredToFile(this.getPath());
205 in = new FSDataInputStreamWrapper(fs, referencePath);
206 status = fs.getFileStatus(referencePath);
207 } else {
208 in = new FSDataInputStreamWrapper(fs, this.getPath());
209 status = fileStatus;
210 }
211 long length = status.getLen();
212 hdfsBlocksDistribution = computeHDFSBlocksDistribution(fs);
213
214 StoreFile.Reader reader = null;
215 if (this.coprocessorHost != null) {
216 reader = this.coprocessorHost.preStoreFileReaderOpen(fs, this.getPath(), in, length,
217 cacheConf, reference);
218 }
219 if (reader == null) {
220 if (this.reference != null) {
221 reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
222 conf);
223 } else {
224 reader = new StoreFile.Reader(fs, this.getPath(), in, length, cacheConf, conf);
225 }
226 }
227 if (this.coprocessorHost != null) {
228 reader = this.coprocessorHost.postStoreFileReaderOpen(fs, this.getPath(), in, length,
229 cacheConf, reference, reader);
230 }
231 return reader;
232 }
233
234
235
236
237 public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
238 throws IOException {
239
240
241
242 if (this.link != null) {
243 FileNotFoundException exToThrow = null;
244 for (int i = 0; i < this.link.getLocations().length; i++) {
245 try {
246 return computeHDFSBlocksDistributionInternal(fs);
247 } catch (FileNotFoundException ex) {
248
249 exToThrow = ex;
250 }
251 }
252 throw exToThrow;
253 } else {
254 return computeHDFSBlocksDistributionInternal(fs);
255 }
256 }
257
258 private HDFSBlocksDistribution computeHDFSBlocksDistributionInternal(final FileSystem fs)
259 throws IOException {
260 FileStatus status = getReferencedFileStatus(fs);
261 if (this.reference != null) {
262 return computeRefFileHDFSBlockDistribution(fs, reference, status);
263 } else {
264 return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
265 }
266 }
267
268
269
270
271
272
273 public FileStatus getReferencedFileStatus(final FileSystem fs) throws IOException {
274 FileStatus status;
275 if (this.reference != null) {
276 if (this.link != null) {
277 FileNotFoundException exToThrow = null;
278 for (int i = 0; i < this.link.getLocations().length; i++) {
279
280 try {
281 return link.getFileStatus(fs);
282 } catch (FileNotFoundException ex) {
283
284 exToThrow = ex;
285 }
286 }
287 throw exToThrow;
288 } else {
289
290 Path referencePath = getReferredToFile(this.getPath());
291 status = fs.getFileStatus(referencePath);
292 }
293 } else {
294 if (this.link != null) {
295 FileNotFoundException exToThrow = null;
296 for (int i = 0; i < this.link.getLocations().length; i++) {
297
298 try {
299 return link.getFileStatus(fs);
300 } catch (FileNotFoundException ex) {
301
302 exToThrow = ex;
303 }
304 }
305 throw exToThrow;
306 } else {
307 status = this.fileStatus;
308 }
309 }
310 return status;
311 }
312
313
314 public Path getPath() {
315 return this.fileStatus.getPath();
316 }
317
318
319 public FileStatus getFileStatus() {
320 return this.fileStatus;
321 }
322
323
324 public long getModificationTime() {
325 return this.fileStatus.getModificationTime();
326 }
327
328 @Override
329 public String toString() {
330 return this.getPath() +
331 (isReference() ? "-" + getReferredToFile(this.getPath()) + "-" + reference : "");
332 }
333
334
335
336
337
338 public static boolean isHFile(final Path path) {
339 return isHFile(path.getName());
340 }
341
342 public static boolean isHFile(final String fileName) {
343 Matcher m = HFILE_NAME_PATTERN.matcher(fileName);
344 return m.matches() && m.groupCount() > 0;
345 }
346
347
348
349
350
351 public static boolean isReference(final Path path) {
352 return isReference(path.getName());
353 }
354
355
356
357
358
359 public static boolean isReference(final String name) {
360 Matcher m = REF_NAME_PATTERN.matcher(name);
361 return m.matches() && m.groupCount() > 1;
362 }
363
364
365
366
367
368
369
370
371 public static Path getReferredToFile(final Path p) {
372 Matcher m = REF_NAME_PATTERN.matcher(p.getName());
373 if (m == null || !m.matches()) {
374 LOG.warn("Failed match of store file name " + p.toString());
375 throw new IllegalArgumentException("Failed match of store file name " +
376 p.toString());
377 }
378
379
380 String otherRegion = m.group(2);
381
382 Path tableDir = p.getParent().getParent().getParent();
383 String nameStrippedOfSuffix = m.group(1);
384 LOG.debug("reference '" + p + "' to region=" + otherRegion + " hfile=" + nameStrippedOfSuffix);
385
386
387
388 return new Path(new Path(new Path(tableDir, otherRegion),
389 p.getParent().getName()), nameStrippedOfSuffix);
390 }
391
392
393
394
395
396
397 public static boolean validateStoreFileName(final String fileName) {
398 if (HFileLink.isHFileLink(fileName) || isReference(fileName))
399 return(true);
400 return !fileName.contains("-");
401 }
402
403
404
405
406
407
408 public static boolean isValid(final FileStatus fileStatus)
409 throws IOException {
410 final Path p = fileStatus.getPath();
411
412 if (fileStatus.isDirectory())
413 return false;
414
415
416
417
418 if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
419 LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?");
420 return false;
421 }
422
423 return validateStoreFileName(p.getName());
424 }
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439 private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
440 final FileSystem fs, final Reference reference, final FileStatus status)
441 throws IOException {
442 if (status == null) {
443 return null;
444 }
445
446 long start = 0;
447 long length = 0;
448
449 if (Reference.isTopFileRegion(reference.getFileRegion())) {
450 start = status.getLen()/2;
451 length = status.getLen() - status.getLen()/2;
452 } else {
453 start = 0;
454 length = status.getLen()/2;
455 }
456 return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
457 }
458
459 @Override
460 public boolean equals(Object that) {
461 if (that == null) {
462 return false;
463 }
464
465 if (that instanceof StoreFileInfo) {
466 return this.compareTo((StoreFileInfo)that) == 0;
467 }
468
469 return false;
470 };
471
472 @Override
473 public int compareTo(StoreFileInfo o) {
474 return this.fileStatus.compareTo(o.fileStatus);
475 }
476
477 @Override
478 public int hashCode() {
479 return this.fileStatus.hashCode();
480 }
481 }