View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.FileNotFoundException;
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.Collections;
25  import java.util.Comparator;
26  import java.util.LinkedList;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  
32  import org.apache.hadoop.classification.InterfaceAudience;
33  import org.apache.hadoop.classification.InterfaceStability;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.conf.Configured;
36  import org.apache.hadoop.fs.FSDataInputStream;
37  import org.apache.hadoop.fs.FSDataOutputStream;
38  import org.apache.hadoop.fs.FileChecksum;
39  import org.apache.hadoop.fs.FileStatus;
40  import org.apache.hadoop.fs.FileSystem;
41  import org.apache.hadoop.fs.FileUtil;
42  import org.apache.hadoop.fs.Path;
43  import org.apache.hadoop.fs.permission.FsPermission;
44  import org.apache.hadoop.hbase.HBaseConfiguration;
45  import org.apache.hadoop.hbase.HConstants;
46  import org.apache.hadoop.hbase.io.HFileLink;
47  import org.apache.hadoop.hbase.io.HLogLink;
48  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
49  import org.apache.hadoop.hbase.regionserver.StoreFile;
50  import org.apache.hadoop.hbase.snapshot.ExportSnapshotException;
51  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
52  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
53  import org.apache.hadoop.hbase.util.Bytes;
54  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
55  import org.apache.hadoop.hbase.util.FSUtils;
56  import org.apache.hadoop.hbase.util.Pair;
57  import org.apache.hadoop.io.NullWritable;
58  import org.apache.hadoop.io.SequenceFile;
59  import org.apache.hadoop.io.Text;
60  import org.apache.hadoop.mapreduce.Job;
61  import org.apache.hadoop.mapreduce.Mapper;
62  import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
63  import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
64  import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
65  import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
66  import org.apache.hadoop.util.StringUtils;
67  import org.apache.hadoop.util.Tool;
68  import org.apache.hadoop.util.ToolRunner;
69  
70  /**
71   * Export the specified snapshot to a given FileSystem.
72   *
73   * The .snapshot/name folder is copied to the destination cluster
74   * and then all the hfiles/hlogs are copied using a Map-Reduce Job in the .archive/ location.
75   * When everything is done, the second cluster can restore the snapshot.
76   */
77  @InterfaceAudience.Public
78  @InterfaceStability.Evolving
79  public final class ExportSnapshot extends Configured implements Tool {
80    private static final Log LOG = LogFactory.getLog(ExportSnapshot.class);
81  
82    private static final String CONF_FILES_USER = "snapshot.export.files.attributes.user";
83    private static final String CONF_FILES_GROUP = "snapshot.export.files.attributes.group";
84    private static final String CONF_FILES_MODE = "snapshot.export.files.attributes.mode";
85    private static final String CONF_CHECKSUM_VERIFY = "snapshot.export.checksum.verify";
86    private static final String CONF_OUTPUT_ROOT = "snapshot.export.output.root";
87    private static final String CONF_INPUT_ROOT = "snapshot.export.input.root";
88    private static final String CONF_STAGING_ROOT = "snapshot.export.staging.root";
89  
90    private static final String INPUT_FOLDER_PREFIX = "export-files.";
91  
92    // Export Map-Reduce Counters, to keep track of the progress
93    public enum Counter { MISSING_FILES, COPY_FAILED, BYTES_EXPECTED, BYTES_COPIED };
94  
95    private static class ExportMapper extends Mapper<Text, NullWritable, NullWritable, NullWritable> {
96      final static int REPORT_SIZE = 1 * 1024 * 1024;
97      final static int BUFFER_SIZE = 64 * 1024;
98  
99      private boolean verifyChecksum;
100     private String filesGroup;
101     private String filesUser;
102     private short filesMode;
103 
104     private FileSystem outputFs;
105     private Path outputArchive;
106     private Path outputRoot;
107 
108     private FileSystem inputFs;
109     private Path inputArchive;
110     private Path inputRoot;
111 
112     @Override
113     public void setup(Context context) {
114       Configuration conf = context.getConfiguration();
115       verifyChecksum = conf.getBoolean(CONF_CHECKSUM_VERIFY, true);
116 
117       filesGroup = conf.get(CONF_FILES_GROUP);
118       filesUser = conf.get(CONF_FILES_USER);
119       filesMode = (short)conf.getInt(CONF_FILES_MODE, 0);
120       outputRoot = new Path(conf.get(CONF_OUTPUT_ROOT));
121       inputRoot = new Path(conf.get(CONF_INPUT_ROOT));
122 
123       inputArchive = new Path(inputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
124       outputArchive = new Path(outputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
125 
126       try {
127         inputFs = FileSystem.get(inputRoot.toUri(), conf);
128       } catch (IOException e) {
129         throw new RuntimeException("Could not get the input FileSystem with root=" + inputRoot, e);
130       }
131 
132       try {
133         outputFs = FileSystem.get(outputRoot.toUri(), conf);
134       } catch (IOException e) {
135         throw new RuntimeException("Could not get the output FileSystem with root="+ outputRoot, e);
136       }
137     }
138 
139     @Override
140     public void map(Text key, NullWritable value, Context context)
141         throws InterruptedException, IOException {
142       Path inputPath = new Path(key.toString());
143       Path outputPath = getOutputPath(inputPath);
144 
145       LOG.info("copy file input=" + inputPath + " output=" + outputPath);
146       if (copyFile(context, inputPath, outputPath)) {
147         LOG.info("copy completed for input=" + inputPath + " output=" + outputPath);
148       }
149     }
150 
151     /**
152      * Returns the location where the inputPath will be copied.
153      *  - hfiles are encoded as hfile links hfile-region-table
154      *  - logs are encoded as serverName/logName
155      */
156     private Path getOutputPath(final Path inputPath) throws IOException {
157       Path path;
158       if (HFileLink.isHFileLink(inputPath) || StoreFile.isReference(inputPath)) {
159         String family = inputPath.getParent().getName();
160         String table = HFileLink.getReferencedTableName(inputPath.getName());
161         String region = HFileLink.getReferencedRegionName(inputPath.getName());
162         String hfile = HFileLink.getReferencedHFileName(inputPath.getName());
163         path = new Path(table, new Path(region, new Path(family, hfile)));
164       } else if (isHLogLinkPath(inputPath)) {
165         String logName = inputPath.getName();
166         path = new Path(new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME), logName);
167       } else {
168         path = inputPath;
169       }
170       return new Path(outputArchive, path);
171     }
172 
173     private boolean copyFile(final Context context, final Path inputPath, final Path outputPath)
174         throws IOException {
175       FSDataInputStream in = openSourceFile(inputPath);
176       if (in == null) {
177         context.getCounter(Counter.MISSING_FILES).increment(1);
178         return false;
179       }
180 
181       try {
182         // Verify if the input file exists
183         FileStatus inputStat = getFileStatus(inputFs, inputPath);
184         if (inputStat == null) return false;
185 
186         // Verify if the output file exists and is the same that we want to copy
187         if (outputFs.exists(outputPath)) {
188           FileStatus outputStat = outputFs.getFileStatus(outputPath);
189           if (sameFile(inputStat, outputStat)) {
190             LOG.info("Skip copy " + inputPath + " to " + outputPath + ", same file.");
191             return true;
192           }
193         }
194 
195         context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen());
196 
197         // Ensure that the output folder is there and copy the file
198         outputFs.mkdirs(outputPath.getParent());
199         FSDataOutputStream out = outputFs.create(outputPath, true);
200         try {
201           if (!copyData(context, inputPath, in, outputPath, out, inputStat.getLen()))
202             return false;
203         } finally {
204           out.close();
205         }
206 
207         // Preserve attributes
208         return preserveAttributes(outputPath, inputStat);
209       } finally {
210         in.close();
211       }
212     }
213 
214     /**
215      * Preserve the files attribute selected by the user copying them from the source file
216      */
217     private boolean preserveAttributes(final Path path, final FileStatus refStat) {
218       FileStatus stat;
219       try {
220         stat = outputFs.getFileStatus(path);
221       } catch (IOException e) {
222         LOG.warn("Unable to get the status for file=" + path);
223         return false;
224       }
225 
226       try {
227         if (filesMode > 0 && stat.getPermission().toShort() != filesMode) {
228           outputFs.setPermission(path, new FsPermission(filesMode));
229         } else if (!stat.getPermission().equals(refStat.getPermission())) {
230           outputFs.setPermission(path, refStat.getPermission());
231         }
232       } catch (IOException e) {
233         LOG.error("Unable to set the permission for file=" + path, e);
234         return false;
235       }
236 
237       try {
238         String user = (filesUser != null) ? filesUser : refStat.getOwner();
239         String group = (filesGroup != null) ? filesGroup : refStat.getGroup();
240         if (!(user.equals(stat.getOwner()) && group.equals(stat.getGroup()))) {
241           outputFs.setOwner(path, user, group);
242         }
243       } catch (IOException e) {
244         LOG.error("Unable to set the owner/group for file=" + path, e);
245         return false;
246       }
247 
248       return true;
249     }
250 
251     private boolean copyData(final Context context,
252         final Path inputPath, final FSDataInputStream in,
253         final Path outputPath, final FSDataOutputStream out,
254         final long inputFileSize) {
255       final String statusMessage = "copied %s/" + StringUtils.humanReadableInt(inputFileSize) +
256                                    " (%.3f%%)";
257 
258       try {
259         byte[] buffer = new byte[BUFFER_SIZE];
260         long totalBytesWritten = 0;
261         int reportBytes = 0;
262         int bytesRead;
263 
264         while ((bytesRead = in.read(buffer)) > 0) {
265           out.write(buffer, 0, bytesRead);
266           totalBytesWritten += bytesRead;
267           reportBytes += bytesRead;
268 
269           if (reportBytes >= REPORT_SIZE) {
270             context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
271             context.setStatus(String.format(statusMessage,
272                               StringUtils.humanReadableInt(totalBytesWritten),
273                               totalBytesWritten/(float)inputFileSize) +
274                               " from " + inputPath + " to " + outputPath);
275             reportBytes = 0;
276           }
277         }
278 
279         context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
280         context.setStatus(String.format(statusMessage,
281                           StringUtils.humanReadableInt(totalBytesWritten),
282                           totalBytesWritten/(float)inputFileSize) +
283                           " from " + inputPath + " to " + outputPath);
284 
285         // Verify that the written size match
286         if (totalBytesWritten != inputFileSize) {
287           LOG.error("number of bytes copied not matching copied=" + totalBytesWritten +
288                     " expected=" + inputFileSize + " for file=" + inputPath);
289           context.getCounter(Counter.COPY_FAILED).increment(1);
290           return false;
291         }
292 
293         return true;
294       } catch (IOException e) {
295         LOG.error("Error copying " + inputPath + " to " + outputPath, e);
296         context.getCounter(Counter.COPY_FAILED).increment(1);
297         return false;
298       }
299     }
300 
301     private FSDataInputStream openSourceFile(final Path path) {
302       try {
303         if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) {
304           return new HFileLink(inputRoot, inputArchive, path).open(inputFs);
305         } else if (isHLogLinkPath(path)) {
306           String serverName = path.getParent().getName();
307           String logName = path.getName();
308           return new HLogLink(inputRoot, serverName, logName).open(inputFs);
309         }
310         return inputFs.open(path);
311       } catch (IOException e) {
312         LOG.error("Unable to open source file=" + path, e);
313         return null;
314       }
315     }
316 
317     private FileStatus getFileStatus(final FileSystem fs, final Path path) {
318       try {
319         if (HFileLink.isHFileLink(path) || StoreFile.isReference(path)) {
320           HFileLink link = new HFileLink(inputRoot, inputArchive, path);
321           return link.getFileStatus(fs);
322         } else if (isHLogLinkPath(path)) {
323           String serverName = path.getParent().getName();
324           String logName = path.getName();
325           return new HLogLink(inputRoot, serverName, logName).getFileStatus(fs);
326         }
327         return fs.getFileStatus(path);
328       } catch (IOException e) {
329         LOG.warn("Unable to get the status for file=" + path);
330         return null;
331       }
332     }
333 
334     private FileChecksum getFileChecksum(final FileSystem fs, final Path path) {
335       try {
336         return fs.getFileChecksum(path);
337       } catch (IOException e) {
338         LOG.warn("Unable to get checksum for file=" + path, e);
339         return null;
340       }
341     }
342 
343     /**
344      * Check if the two files are equal by looking at the file length,
345      * and at the checksum (if user has specified the verifyChecksum flag).
346      */
347     private boolean sameFile(final FileStatus inputStat, final FileStatus outputStat) {
348       // Not matching length
349       if (inputStat.getLen() != outputStat.getLen()) return false;
350 
351       // Mark files as equals, since user asked for no checksum verification
352       if (!verifyChecksum) return true;
353 
354       // If checksums are not available, files are not the same.
355       FileChecksum inChecksum = getFileChecksum(inputFs, inputStat.getPath());
356       if (inChecksum == null) return false;
357 
358       FileChecksum outChecksum = getFileChecksum(outputFs, outputStat.getPath());
359       if (outChecksum == null) return false;
360 
361       return inChecksum.equals(outChecksum);
362     }
363 
364     /**
365      * HLog files are encoded as serverName/logName
366      * and since all the other files should be in /hbase/table/..path..
367      * we can rely on the depth, for now.
368      */
369     private static boolean isHLogLinkPath(final Path path) {
370       return path.depth() == 2;
371     }
372   }
373 
374   /**
375    * Extract the list of files (HFiles/HLogs) to copy using Map-Reduce.
376    * @return list of files referenced by the snapshot (pair of path and size)
377    */
378   private List<Pair<Path, Long>> getSnapshotFiles(final FileSystem fs, final Path snapshotDir)
379       throws IOException {
380     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
381 
382     final List<Pair<Path, Long>> files = new ArrayList<Pair<Path, Long>>();
383     final String table = snapshotDesc.getTable();
384     final Configuration conf = getConf();
385 
386     // Get snapshot files
387     SnapshotReferenceUtil.visitReferencedFiles(fs, snapshotDir,
388       new SnapshotReferenceUtil.FileVisitor() {
389         public void storeFile (final String region, final String family, final String hfile)
390             throws IOException {
391           Path path = HFileLink.createPath(table, region, family, hfile);
392           long size = new HFileLink(conf, path).getFileStatus(fs).getLen();
393           files.add(new Pair<Path, Long>(path, size));
394         }
395 
396         public void recoveredEdits (final String region, final String logfile)
397             throws IOException {
398           // copied with the snapshot referenecs
399         }
400 
401         public void logFile (final String server, final String logfile)
402             throws IOException {
403           long size = new HLogLink(conf, server, logfile).getFileStatus(fs).getLen();
404           files.add(new Pair<Path, Long>(new Path(server, logfile), size));
405         }
406     });
407 
408     return files;
409   }
410 
411   /**
412    * Given a list of file paths and sizes, create around ngroups in as balanced a way as possible.
413    * The groups created will have similar amounts of bytes.
414    * <p>
415    * The algorithm used is pretty straightforward; the file list is sorted by size,
416    * and then each group fetch the bigger file available, iterating through groups
417    * alternating the direction.
418    */
419   static List<List<Path>> getBalancedSplits(final List<Pair<Path, Long>> files, int ngroups) {
420     // Sort files by size, from small to big
421     Collections.sort(files, new Comparator<Pair<Path, Long>>() {
422       public int compare(Pair<Path, Long> a, Pair<Path, Long> b) {
423         long r = a.getSecond() - b.getSecond();
424         return (r < 0) ? -1 : ((r > 0) ? 1 : 0);
425       }
426     });
427 
428     // create balanced groups
429     List<List<Path>> fileGroups = new LinkedList<List<Path>>();
430     long[] sizeGroups = new long[ngroups];
431     int hi = files.size() - 1;
432     int lo = 0;
433 
434     List<Path> group;
435     int dir = 1;
436     int g = 0;
437 
438     while (hi >= lo) {
439       if (g == fileGroups.size()) {
440         group = new LinkedList<Path>();
441         fileGroups.add(group);
442       } else {
443         group = fileGroups.get(g);
444       }
445 
446       Pair<Path, Long> fileInfo = files.get(hi--);
447 
448       // add the hi one
449       sizeGroups[g] += fileInfo.getSecond();
450       group.add(fileInfo.getFirst());
451 
452       // change direction when at the end or the beginning
453       g += dir;
454       if (g == ngroups) {
455         dir = -1;
456         g = ngroups - 1;
457       } else if (g < 0) {
458         dir = 1;
459         g = 0;
460       }
461     }
462 
463     if (LOG.isDebugEnabled()) {
464       for (int i = 0; i < sizeGroups.length; ++i) {
465         LOG.debug("export split=" + i + " size=" + StringUtils.humanReadableInt(sizeGroups[i]));
466       }
467     }
468 
469     return fileGroups;
470   }
471 
472   private static Path getInputFolderPath(final FileSystem fs, final Configuration conf)
473       throws IOException, InterruptedException {
474     String stagingName = "exportSnapshot-" + EnvironmentEdgeManager.currentTimeMillis();
475     Path stagingDir = new Path(conf.get(CONF_STAGING_ROOT, fs.getWorkingDirectory().toString())
476         , stagingName);
477     fs.mkdirs(stagingDir);
478     return new Path(stagingDir, INPUT_FOLDER_PREFIX +
479       String.valueOf(EnvironmentEdgeManager.currentTimeMillis()));
480   }
481 
482   /**
483    * Create the input files, with the path to copy, for the MR job.
484    * Each input files contains n files, and each input file has a similar amount data to copy.
485    * The number of input files created are based on the number of mappers provided as argument
486    * and the number of the files to copy.
487    */
488   private static Path[] createInputFiles(final Configuration conf,
489       final List<Pair<Path, Long>> snapshotFiles, int mappers)
490       throws IOException, InterruptedException {
491     FileSystem fs = FileSystem.get(conf);
492     Path inputFolderPath = getInputFolderPath(fs, conf);
493     LOG.debug("Input folder location: " + inputFolderPath);
494 
495     List<List<Path>> splits = getBalancedSplits(snapshotFiles, mappers);
496     Path[] inputFiles = new Path[splits.size()];
497 
498     Text key = new Text();
499     for (int i = 0; i < inputFiles.length; i++) {
500       List<Path> files = splits.get(i);
501       inputFiles[i] = new Path(inputFolderPath, String.format("export-%d.seq", i));
502       SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, inputFiles[i],
503         Text.class, NullWritable.class);
504       LOG.debug("Input split: " + i);
505       try {
506         for (Path file: files) {
507           LOG.debug(file.toString());
508           key.set(file.toString());
509           writer.append(key, NullWritable.get());
510         }
511       } finally {
512         writer.close();
513       }
514     }
515 
516     return inputFiles;
517   }
518 
519   /**
520    * Run Map-Reduce Job to perform the files copy.
521    */
522   private boolean runCopyJob(final Path inputRoot, final Path outputRoot,
523       final List<Pair<Path, Long>> snapshotFiles, final boolean verifyChecksum,
524       final String filesUser, final String filesGroup, final int filesMode,
525       final int mappers) throws IOException, InterruptedException, ClassNotFoundException {
526     Configuration conf = getConf();
527     if (filesGroup != null) conf.set(CONF_FILES_GROUP, filesGroup);
528     if (filesUser != null) conf.set(CONF_FILES_USER, filesUser);
529     conf.setInt(CONF_FILES_MODE, filesMode);
530     conf.setBoolean(CONF_CHECKSUM_VERIFY, verifyChecksum);
531     conf.set(CONF_OUTPUT_ROOT, outputRoot.toString());
532     conf.set(CONF_INPUT_ROOT, inputRoot.toString());
533     conf.setInt("mapreduce.job.maps", mappers);
534 
535     // job.setMapSpeculativeExecution(false)
536     conf.setBoolean("mapreduce.map.speculative", false);
537     conf.setBoolean("mapreduce.reduce.speculative", false);
538     conf.setBoolean("mapred.map.tasks.speculative.execution", false);
539     conf.setBoolean("mapred.reduce.tasks.speculative.execution", false);
540 
541     Job job = new Job(conf);
542     job.setJobName("ExportSnapshot");
543     job.setJarByClass(ExportSnapshot.class);
544     job.setMapperClass(ExportMapper.class);
545     job.setInputFormatClass(SequenceFileInputFormat.class);
546     job.setOutputFormatClass(NullOutputFormat.class);
547     job.setNumReduceTasks(0);
548     for (Path path: createInputFiles(conf, snapshotFiles, mappers)) {
549       LOG.debug("Add Input Path=" + path);
550       SequenceFileInputFormat.addInputPath(job, path);
551     }
552 
553     return job.waitForCompletion(true);
554   }
555 
556   /**
557    * Execute the export snapshot by copying the snapshot metadata, hfiles and hlogs.
558    * @return 0 on success, and != 0 upon failure.
559    */
560   @Override
561   public int run(String[] args) throws Exception {
562     boolean verifyChecksum = true;
563     String snapshotName = null;
564     String filesGroup = null;
565     String filesUser = null;
566     Path outputRoot = null;
567     int filesMode = 0;
568     int mappers = getConf().getInt("mapreduce.job.maps", 1);
569 
570     // Process command line args
571     for (int i = 0; i < args.length; i++) {
572       String cmd = args[i];
573       try {
574         if (cmd.equals("-snapshot")) {
575           snapshotName = args[++i];
576         } else if (cmd.equals("-copy-to")) {
577           outputRoot = new Path(args[++i]);
578         } else if (cmd.equals("-no-checksum-verify")) {
579           verifyChecksum = false;
580         } else if (cmd.equals("-mappers")) {
581           mappers = Integer.parseInt(args[++i]);
582         } else if (cmd.equals("-chuser")) {
583           filesUser = args[++i];
584         } else if (cmd.equals("-chgroup")) {
585           filesGroup = args[++i];
586         } else if (cmd.equals("-chmod")) {
587           filesMode = Integer.parseInt(args[++i], 8);
588         } else if (cmd.equals("-h") || cmd.equals("--help")) {
589           printUsageAndExit();
590         } else {
591           System.err.println("UNEXPECTED: " + cmd);
592           printUsageAndExit();
593         }
594       } catch (Exception e) {
595         printUsageAndExit();
596       }
597     }
598 
599     // Check user options
600     if (snapshotName == null) {
601       System.err.println("Snapshot name not provided.");
602       printUsageAndExit();
603     }
604 
605     if (outputRoot == null) {
606       System.err.println("Destination file-system not provided.");
607       printUsageAndExit();
608     }
609 
610     Configuration conf = getConf();
611     Path inputRoot = FSUtils.getRootDir(conf);
612     FileSystem inputFs = FileSystem.get(conf);
613     FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);
614 
615     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
616     Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshotName, outputRoot);
617     Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, outputRoot);
618 
619     // Check if the snapshot already exists
620     if (outputFs.exists(outputSnapshotDir)) {
621       System.err.println("The snapshot '" + snapshotName +
622         "' already exists in the destination: " + outputSnapshotDir);
623       return 1;
624     }
625 
626     // Check if the snapshot already in-progress
627     if (outputFs.exists(snapshotTmpDir)) {
628       System.err.println("A snapshot with the same name '" + snapshotName + "' may be in-progress");
629       System.err.println("Please check " + snapshotTmpDir + ". If the snapshot has completed, ");
630       System.err.println("consider removing " + snapshotTmpDir + " before retrying export");
631       return 1;
632     }
633 
634     // Step 0 - Extract snapshot files to copy
635     final List<Pair<Path, Long>> files = getSnapshotFiles(inputFs, snapshotDir);
636 
637     // Step 1 - Copy fs1:/.snapshot/<snapshot> to  fs2:/.snapshot/.tmp/<snapshot>
638     // The snapshot references must be copied before the hfiles otherwise the cleaner
639     // will remove them because they are unreferenced.
640     try {
641       FileUtil.copy(inputFs, snapshotDir, outputFs, snapshotTmpDir, false, false, conf);
642     } catch (IOException e) {
643       System.err.println("Failed to copy the snapshot directory: from=" + snapshotDir +
644         " to=" + snapshotTmpDir);
645       e.printStackTrace(System.err);
646       return 1;
647     }
648 
649     // Step 2 - Start MR Job to copy files
650     // The snapshot references must be copied before the files otherwise the files gets removed
651     // by the HFileArchiver, since they have no references.
652     try {
653       if (files.size() == 0) {
654         LOG.warn("There are 0 store file to be copied. There may be no data in the table.");
655       } else {
656         if (!runCopyJob(inputRoot, outputRoot, files, verifyChecksum,
657             filesUser, filesGroup, filesMode, mappers)) {
658           throw new ExportSnapshotException("Snapshot export failed!");
659         }
660       }
661 
662       // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
663       if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
664         System.err.println("Snapshot export failed!");
665         System.err.println("Unable to rename snapshot directory from=" +
666                            snapshotTmpDir + " to=" + outputSnapshotDir);
667         return 1;
668       }
669 
670       return 0;
671     } catch (Exception e) {
672       System.err.println("Snapshot export failed!");
673       e.printStackTrace(System.err);
674       outputFs.delete(outputSnapshotDir, true);
675       return 1;
676     }
677   }
678 
679   // ExportSnapshot
680   private void printUsageAndExit() {
681     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
682     System.err.println(" where [options] are:");
683     System.err.println("  -h|-help                Show this help and exit.");
684     System.err.println("  -snapshot NAME          Snapshot to restore.");
685     System.err.println("  -copy-to NAME           Remote destination hdfs://");
686     System.err.println("  -no-checksum-verify     Do not verify checksum.");
687     System.err.println("  -chuser USERNAME        Change the owner of the files to the specified one.");
688     System.err.println("  -chgroup GROUP          Change the group of the files to the specified one.");
689     System.err.println("  -chmod MODE             Change the permission of the files to the specified one.");
690     System.err.println("  -mappers                Number of mappers to use during the copy (mapreduce.job.maps).");
691     System.err.println();
692     System.err.println("Examples:");
693     System.err.println("  hbase " + getClass() + " \\");
694     System.err.println("    -snapshot MySnapshot -copy-to hdfs:///srv2:8082/hbase \\");
695     System.err.println("    -chuser MyUser -chgroup MyGroup -chmod 700 -mappers 16");
696     System.exit(1);
697   }
698 
699   /**
700    * The guts of the {@link #main} method.
701    * Call this method to avoid the {@link #main(String[])} System.exit.
702    * @param args
703    * @return errCode
704    * @throws Exception
705    */
706   static int innerMain(final Configuration conf, final String [] args) throws Exception {
707     return ToolRunner.run(conf, new ExportSnapshot(), args);
708   }
709 
710   public static void main(String[] args) throws Exception {
711      System.exit(innerMain(HBaseConfiguration.create(), args));
712   }
713 }