1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver;
21
22 import java.io.IOException;
23 import java.util.regex.Matcher;
24 import java.util.regex.Pattern;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.hbase.classification.InterfaceAudience;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileStatus;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
34 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
35 import org.apache.hadoop.hbase.io.HFileLink;
36 import org.apache.hadoop.hbase.io.HalfStoreFileReader;
37 import org.apache.hadoop.hbase.io.Reference;
38 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
39 import org.apache.hadoop.hbase.util.FSUtils;
40
41
42
43
44 @InterfaceAudience.Private
45 public class StoreFileInfo {
46 public static final Log LOG = LogFactory.getLog(StoreFileInfo.class);
47
48
49
50
51
52 public static final String HFILE_NAME_REGEX = "[0-9a-f]+(?:_SeqId_[0-9]+_)?";
53
54
55 private static final Pattern HFILE_NAME_PATTERN =
56 Pattern.compile("^(" + HFILE_NAME_REGEX + ")");
57
58
59
60
61
62
63
64
65 private static final Pattern REF_NAME_PATTERN =
66 Pattern.compile(String.format("^(%s|%s)\\.(.+)$",
67 HFILE_NAME_REGEX, HFileLink.LINK_NAME_REGEX));
68
69
70 private Configuration conf;
71
72
73 private HDFSBlocksDistribution hdfsBlocksDistribution = null;
74
75
76 private final Reference reference;
77
78
79 private final HFileLink link;
80
81
82 private final FileStatus fileStatus;
83
84 private RegionCoprocessorHost coprocessorHost;
85
86
87
88
89
90
91
92 public StoreFileInfo(final Configuration conf, final FileSystem fs, final Path path)
93 throws IOException {
94 this(conf, fs, fs.getFileStatus(path));
95 }
96
97
98
99
100
101
102
103 public StoreFileInfo(final Configuration conf, final FileSystem fs, final FileStatus fileStatus)
104 throws IOException {
105 this.conf = conf;
106 this.fileStatus = fileStatus;
107 Path p = fileStatus.getPath();
108 if (HFileLink.isHFileLink(p)) {
109
110 this.reference = null;
111 this.link = new HFileLink(conf, p);
112 if (LOG.isTraceEnabled()) LOG.trace(p + " is a link");
113 } else if (isReference(p)) {
114 this.reference = Reference.read(fs, p);
115 Path referencePath = getReferredToFile(p);
116 if (HFileLink.isHFileLink(referencePath)) {
117
118 this.link = new HFileLink(conf, referencePath);
119 } else {
120
121 this.link = null;
122 }
123 if (LOG.isTraceEnabled()) LOG.trace(p + " is a " + reference.getFileRegion() +
124 " reference to " + referencePath);
125 } else if (isHFile(p)) {
126
127 this.reference = null;
128 this.link = null;
129 } else {
130 throw new IOException("path=" + p + " doesn't look like a valid StoreFile");
131 }
132 }
133
134
135
136
137
138 public void setRegionCoprocessorHost(RegionCoprocessorHost coprocessorHost) {
139 this.coprocessorHost = coprocessorHost;
140 }
141
142
143
144
145
146 public Reference getReference() {
147 return this.reference;
148 }
149
150
151 public boolean isReference() {
152 return this.reference != null;
153 }
154
155
156 public boolean isTopReference() {
157 return this.reference != null && Reference.isTopFileRegion(this.reference.getFileRegion());
158 }
159
160
161 public boolean isLink() {
162 return this.link != null && this.reference == null;
163 }
164
165
166 public HDFSBlocksDistribution getHDFSBlockDistribution() {
167 return this.hdfsBlocksDistribution;
168 }
169
170
171
172
173
174
175
176 public StoreFile.Reader open(final FileSystem fs,
177 final CacheConfig cacheConf, final boolean canUseDropBehind) throws IOException {
178 FSDataInputStreamWrapper in;
179 FileStatus status;
180
181 final boolean doDropBehind = canUseDropBehind && cacheConf.shouldDropBehindCompaction();
182 if (this.link != null) {
183
184 in = new FSDataInputStreamWrapper(fs, this.link, doDropBehind);
185 status = this.link.getFileStatus(fs);
186 } else if (this.reference != null) {
187
188 Path referencePath = getReferredToFile(this.getPath());
189 in = new FSDataInputStreamWrapper(fs, referencePath,
190 doDropBehind);
191 status = fs.getFileStatus(referencePath);
192 } else {
193 in = new FSDataInputStreamWrapper(fs, this.getPath(),
194 doDropBehind);
195 status = fileStatus;
196 }
197 long length = status.getLen();
198 if (this.reference != null) {
199 hdfsBlocksDistribution = computeRefFileHDFSBlockDistribution(fs, reference, status);
200 } else {
201 hdfsBlocksDistribution = FSUtils.computeHDFSBlocksDistribution(fs, status, 0, length);
202 }
203 StoreFile.Reader reader = null;
204 if (this.coprocessorHost != null) {
205 reader = this.coprocessorHost.preStoreFileReaderOpen(fs, this.getPath(), in, length,
206 cacheConf, reference);
207 }
208 if (reader == null) {
209 if (this.reference != null) {
210 reader = new HalfStoreFileReader(fs, this.getPath(), in, length, cacheConf, reference,
211 conf);
212 } else {
213 reader = new StoreFile.Reader(fs, this.getPath(), in, length, cacheConf, conf);
214 }
215 }
216 if (this.coprocessorHost != null) {
217 reader = this.coprocessorHost.postStoreFileReaderOpen(fs, this.getPath(), in, length,
218 cacheConf, reference, reader);
219 }
220 return reader;
221 }
222
223
224
225
226 public HDFSBlocksDistribution computeHDFSBlocksDistribution(final FileSystem fs)
227 throws IOException {
228 FileStatus status = getReferencedFileStatus(fs);
229 if (this.reference != null) {
230 return computeRefFileHDFSBlockDistribution(fs, reference, status);
231 } else {
232 return FSUtils.computeHDFSBlocksDistribution(fs, status, 0, status.getLen());
233 }
234 }
235
236
237
238
239
240
241 public FileStatus getReferencedFileStatus(final FileSystem fs) throws IOException {
242 FileStatus status;
243 if (this.reference != null) {
244 if (this.link != null) {
245
246 status = link.getFileStatus(fs);
247 } else {
248
249 Path referencePath = getReferredToFile(this.getPath());
250 status = fs.getFileStatus(referencePath);
251 }
252 } else {
253 if (this.link != null) {
254
255 status = link.getFileStatus(fs);
256 } else {
257 status = this.fileStatus;
258 }
259 }
260 return status;
261 }
262
263
264 public Path getPath() {
265 return this.fileStatus.getPath();
266 }
267
268
269 public FileStatus getFileStatus() {
270 return this.fileStatus;
271 }
272
273
274 public long getModificationTime() {
275 return this.fileStatus.getModificationTime();
276 }
277
278 @Override
279 public String toString() {
280 return this.getPath() +
281 (isReference() ? "-" + getReferredToFile(this.getPath()) + "-" + reference : "");
282 }
283
284
285
286
287
288 public static boolean isHFile(final Path path) {
289 return isHFile(path.getName());
290 }
291
292 public static boolean isHFile(final String fileName) {
293 Matcher m = HFILE_NAME_PATTERN.matcher(fileName);
294 return m.matches() && m.groupCount() > 0;
295 }
296
297
298
299
300
301 public static boolean isReference(final Path path) {
302 return isReference(path.getName());
303 }
304
305
306
307
308
309 public static boolean isReference(final String name) {
310 Matcher m = REF_NAME_PATTERN.matcher(name);
311 return m.matches() && m.groupCount() > 1;
312 }
313
314
315
316
317
318
319
320
321 public static Path getReferredToFile(final Path p) {
322 Matcher m = REF_NAME_PATTERN.matcher(p.getName());
323 if (m == null || !m.matches()) {
324 LOG.warn("Failed match of store file name " + p.toString());
325 throw new IllegalArgumentException("Failed match of store file name " +
326 p.toString());
327 }
328
329
330 String otherRegion = m.group(2);
331
332 Path tableDir = p.getParent().getParent().getParent();
333 String nameStrippedOfSuffix = m.group(1);
334 LOG.debug("reference '" + p + "' to region=" + otherRegion + " hfile=" + nameStrippedOfSuffix);
335
336
337
338 return new Path(new Path(new Path(tableDir, otherRegion),
339 p.getParent().getName()), nameStrippedOfSuffix);
340 }
341
342
343
344
345
346
347 public static boolean validateStoreFileName(final String fileName) {
348 if (HFileLink.isHFileLink(fileName) || isReference(fileName))
349 return(true);
350 return !fileName.contains("-");
351 }
352
353
354
355
356
357
358 public static boolean isValid(final FileStatus fileStatus)
359 throws IOException {
360 final Path p = fileStatus.getPath();
361
362 if (fileStatus.isDir())
363 return false;
364
365
366
367
368 if (!HFileLink.isHFileLink(p) && fileStatus.getLen() <= 0) {
369 LOG.warn("Skipping " + p + " because it is empty. HBASE-646 DATA LOSS?");
370 return false;
371 }
372
373 return validateStoreFileName(p.getName());
374 }
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389 private static HDFSBlocksDistribution computeRefFileHDFSBlockDistribution(
390 final FileSystem fs, final Reference reference, final FileStatus status)
391 throws IOException {
392 if (status == null) {
393 return null;
394 }
395
396 long start = 0;
397 long length = 0;
398
399 if (Reference.isTopFileRegion(reference.getFileRegion())) {
400 start = status.getLen()/2;
401 length = status.getLen() - status.getLen()/2;
402 } else {
403 start = 0;
404 length = status.getLen()/2;
405 }
406 return FSUtils.computeHDFSBlocksDistribution(fs, status, start, length);
407 }
408 }