1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import com.google.protobuf.InvalidProtocolBufferException;
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.concurrent.Callable;
27 import java.util.concurrent.Executor;
28 import java.util.concurrent.ExecutionException;
29 import java.util.concurrent.ExecutorCompletionService;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.classification.InterfaceAudience;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FSDataInputStream;
36 import org.apache.hadoop.fs.FSDataOutputStream;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.fs.PathFilter;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
43 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
44 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
45 import org.apache.hadoop.hbase.util.FSUtils;
46
47 import com.google.protobuf.HBaseZeroCopyByteString;
48
49
50
51
52
53
54
55
56
57 @InterfaceAudience.Private
58 public class SnapshotManifestV2 {
59 private static final Log LOG = LogFactory.getLog(SnapshotManifestV2.class);
60
61 public static final int DESCRIPTOR_VERSION = 2;
62
63 public static final String SNAPSHOT_MANIFEST_PREFIX = "region-manifest.";
64
65 static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
66 SnapshotRegionManifest.Builder, SnapshotRegionManifest.FamilyFiles.Builder> {
67 private final Configuration conf;
68 private final Path snapshotDir;
69 private final FileSystem fs;
70
71 public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
72 this.snapshotDir = snapshotDir;
73 this.conf = conf;
74 this.fs = fs;
75 }
76
77 public SnapshotRegionManifest.Builder regionOpen(final HRegionInfo regionInfo) {
78 SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
79 manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
80 return manifest;
81 }
82
83 public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
84 SnapshotRegionManifest manifest = region.build();
85 FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
86 try {
87 manifest.writeTo(stream);
88 } finally {
89 stream.close();
90 }
91 }
92
93 public SnapshotRegionManifest.FamilyFiles.Builder familyOpen(
94 final SnapshotRegionManifest.Builder region, final byte[] familyName) {
95 SnapshotRegionManifest.FamilyFiles.Builder family =
96 SnapshotRegionManifest.FamilyFiles.newBuilder();
97 family.setFamilyName(HBaseZeroCopyByteString.wrap(familyName));
98 return family;
99 }
100
101 public void familyClose(final SnapshotRegionManifest.Builder region,
102 final SnapshotRegionManifest.FamilyFiles.Builder family) {
103 region.addFamilyFiles(family.build());
104 }
105
106 public void storeFile(final SnapshotRegionManifest.Builder region,
107 final SnapshotRegionManifest.FamilyFiles.Builder family, final StoreFileInfo storeFile)
108 throws IOException {
109 SnapshotRegionManifest.StoreFile.Builder sfManifest =
110 SnapshotRegionManifest.StoreFile.newBuilder();
111 sfManifest.setName(storeFile.getPath().getName());
112 if (storeFile.isReference()) {
113 sfManifest.setReference(storeFile.getReference().convert());
114 }
115 sfManifest.setFileSize(storeFile.getReferencedFileStatus(fs).getLen());
116 family.addStoreFiles(sfManifest.build());
117 }
118 }
119
120 static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
121 final Executor executor,final FileSystem fs, final Path snapshotDir,
122 final SnapshotDescription desc) throws IOException {
123 FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
124 @Override
125 public boolean accept(Path path) {
126 return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
127 }
128 });
129
130 if (manifestFiles == null || manifestFiles.length == 0) return null;
131
132 final ExecutorCompletionService<SnapshotRegionManifest> completionService =
133 new ExecutorCompletionService<SnapshotRegionManifest>(executor);
134 for (final FileStatus st: manifestFiles) {
135 completionService.submit(new Callable<SnapshotRegionManifest>() {
136 @Override
137 public SnapshotRegionManifest call() throws IOException {
138 FSDataInputStream stream = fs.open(st.getPath());
139 try {
140 return SnapshotRegionManifest.parseFrom(stream);
141 } finally {
142 stream.close();
143 }
144 }
145 });
146 }
147
148 ArrayList<SnapshotRegionManifest> regionsManifest =
149 new ArrayList<SnapshotRegionManifest>(manifestFiles.length);
150 try {
151 for (int i = 0; i < manifestFiles.length; ++i) {
152 regionsManifest.add(completionService.take().get());
153 }
154 } catch (InterruptedException e) {
155 throw new InterruptedIOException(e.getMessage());
156 } catch (ExecutionException e) {
157 Throwable t = e.getCause();
158
159 if(t instanceof InvalidProtocolBufferException) {
160 throw (InvalidProtocolBufferException)t;
161 } else {
162 IOException ex = new IOException("ExecutionException");
163 ex.initCause(e.getCause());
164 throw ex;
165 }
166 }
167 return regionsManifest;
168 }
169
170 static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
171 final SnapshotRegionManifest manifest) throws IOException {
172 fs.delete(getRegionManifestPath(snapshotDir, manifest), true);
173 }
174
175 private static Path getRegionManifestPath(final Path snapshotDir,
176 final SnapshotRegionManifest manifest) {
177 String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
178 return new Path(snapshotDir, SNAPSHOT_MANIFEST_PREFIX + regionName);
179 }
180 }