View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import com.google.protobuf.InvalidProtocolBufferException;
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.concurrent.Callable;
27  import java.util.concurrent.Executor;
28  import java.util.concurrent.ExecutionException;
29  import java.util.concurrent.ExecutorCompletionService;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.hbase.classification.InterfaceAudience;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FSDataInputStream;
36  import org.apache.hadoop.fs.FSDataOutputStream;
37  import org.apache.hadoop.fs.FileStatus;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.fs.PathFilter;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
43  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
44  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
45  import org.apache.hadoop.hbase.util.FSUtils;
46  
47  import com.google.protobuf.HBaseZeroCopyByteString;
48  
49  /**
50   * DO NOT USE DIRECTLY. USE {@link SnapshotManifest}.
51   *
52   * Snapshot v2 layout format
53   *  - Single Manifest file containing all the information of regions
54   *  - In the online-snapshot case each region will write a "region manifest"
55   *      /snapshotName/manifest.regionName
56   */
57  @InterfaceAudience.Private
58  public class SnapshotManifestV2 {
59    private static final Log LOG = LogFactory.getLog(SnapshotManifestV2.class);
60  
61    public static final int DESCRIPTOR_VERSION = 2;
62  
63    public static final String SNAPSHOT_MANIFEST_PREFIX = "region-manifest.";
64  
65    static class ManifestBuilder implements SnapshotManifest.RegionVisitor<
66                      SnapshotRegionManifest.Builder, SnapshotRegionManifest.FamilyFiles.Builder> {
67      private final Configuration conf;
68      private final Path snapshotDir;
69      private final FileSystem fs;
70  
71      public ManifestBuilder(final Configuration conf, final FileSystem fs, final Path snapshotDir) {
72        this.snapshotDir = snapshotDir;
73        this.conf = conf;
74        this.fs = fs;
75      }
76  
77      public SnapshotRegionManifest.Builder regionOpen(final HRegionInfo regionInfo) {
78        SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();
79        manifest.setRegionInfo(HRegionInfo.convert(regionInfo));
80        return manifest;
81      }
82  
83      public void regionClose(final SnapshotRegionManifest.Builder region) throws IOException {
84        SnapshotRegionManifest manifest = region.build();
85        FSDataOutputStream stream = fs.create(getRegionManifestPath(snapshotDir, manifest));
86        try {
87          manifest.writeTo(stream);
88        } finally {
89          stream.close();
90        }
91      }
92  
93      public SnapshotRegionManifest.FamilyFiles.Builder familyOpen(
94          final SnapshotRegionManifest.Builder region, final byte[] familyName) {
95        SnapshotRegionManifest.FamilyFiles.Builder family =
96            SnapshotRegionManifest.FamilyFiles.newBuilder();
97        family.setFamilyName(HBaseZeroCopyByteString.wrap(familyName));
98        return family;
99      }
100 
101     public void familyClose(final SnapshotRegionManifest.Builder region,
102         final SnapshotRegionManifest.FamilyFiles.Builder family) {
103       region.addFamilyFiles(family.build());
104     }
105 
106     public void storeFile(final SnapshotRegionManifest.Builder region,
107         final SnapshotRegionManifest.FamilyFiles.Builder family, final StoreFileInfo storeFile)
108         throws IOException {
109       SnapshotRegionManifest.StoreFile.Builder sfManifest =
110             SnapshotRegionManifest.StoreFile.newBuilder();
111       sfManifest.setName(storeFile.getPath().getName());
112       if (storeFile.isReference()) {
113         sfManifest.setReference(storeFile.getReference().convert());
114       }
115       sfManifest.setFileSize(storeFile.getReferencedFileStatus(fs).getLen());
116       family.addStoreFiles(sfManifest.build());
117     }
118   }
119 
120   static List<SnapshotRegionManifest> loadRegionManifests(final Configuration conf,
121       final Executor executor,final FileSystem fs, final Path snapshotDir,
122       final SnapshotDescription desc) throws IOException {
123     FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
124       @Override
125       public boolean accept(Path path) {
126         return path.getName().startsWith(SNAPSHOT_MANIFEST_PREFIX);
127       }
128     });
129 
130     if (manifestFiles == null || manifestFiles.length == 0) return null;
131 
132     final ExecutorCompletionService<SnapshotRegionManifest> completionService =
133       new ExecutorCompletionService<SnapshotRegionManifest>(executor);
134     for (final FileStatus st: manifestFiles) {
135       completionService.submit(new Callable<SnapshotRegionManifest>() {
136         @Override
137         public SnapshotRegionManifest call() throws IOException {
138           FSDataInputStream stream = fs.open(st.getPath());
139           try {
140             return SnapshotRegionManifest.parseFrom(stream);
141           } finally {
142             stream.close();
143           }
144         }
145       });
146     }
147 
148     ArrayList<SnapshotRegionManifest> regionsManifest =
149         new ArrayList<SnapshotRegionManifest>(manifestFiles.length);
150     try {
151       for (int i = 0; i < manifestFiles.length; ++i) {
152         regionsManifest.add(completionService.take().get());
153       }
154     } catch (InterruptedException e) {
155       throw new InterruptedIOException(e.getMessage());
156     } catch (ExecutionException e) {
157       Throwable t = e.getCause();
158 
159       if(t instanceof InvalidProtocolBufferException) {
160         throw (InvalidProtocolBufferException)t;
161       } else {
162         IOException ex = new IOException("ExecutionException");
163         ex.initCause(e.getCause());
164         throw ex;
165       }
166     }
167     return regionsManifest;
168   }
169 
170   static void deleteRegionManifest(final FileSystem fs, final Path snapshotDir,
171       final SnapshotRegionManifest manifest) throws IOException {
172     fs.delete(getRegionManifestPath(snapshotDir, manifest), true);
173   }
174 
175   private static Path getRegionManifestPath(final Path snapshotDir,
176       final SnapshotRegionManifest manifest) {
177     String regionName = SnapshotManifest.getRegionNameFromManifest(manifest);
178     return new Path(snapshotDir, SNAPSHOT_MANIFEST_PREFIX + regionName);
179   }
180 }