View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.snapshot;
20  
21  import java.io.BufferedInputStream;
22  import java.io.FileNotFoundException;
23  import java.io.DataInput;
24  import java.io.DataOutput;
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.net.URI;
28  import java.util.ArrayList;
29  import java.util.Collections;
30  import java.util.Comparator;
31  import java.util.LinkedList;
32  import java.util.List;
33  import java.util.Random;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.hbase.classification.InterfaceAudience;
38  import org.apache.hadoop.hbase.classification.InterfaceStability;
39  import org.apache.hadoop.conf.Configuration;
40  import org.apache.hadoop.conf.Configured;
41  import org.apache.hadoop.fs.FSDataInputStream;
42  import org.apache.hadoop.fs.FSDataOutputStream;
43  import org.apache.hadoop.fs.FileChecksum;
44  import org.apache.hadoop.fs.FileStatus;
45  import org.apache.hadoop.fs.FileSystem;
46  import org.apache.hadoop.fs.FileUtil;
47  import org.apache.hadoop.fs.Path;
48  import org.apache.hadoop.fs.permission.FsPermission;
49  import org.apache.hadoop.hbase.TableName;
50  import org.apache.hadoop.hbase.HBaseConfiguration;
51  import org.apache.hadoop.hbase.HConstants;
52  import org.apache.hadoop.hbase.HRegionInfo;
53  import org.apache.hadoop.hbase.io.FileLink;
54  import org.apache.hadoop.hbase.io.HFileLink;
55  import org.apache.hadoop.hbase.io.HLogLink;
56  import org.apache.hadoop.hbase.io.hadoopbackport.ThrottledInputStream;
57  import org.apache.hadoop.hbase.mapreduce.JobUtil;
58  import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
59  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
60  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
61  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
62  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
63  import org.apache.hadoop.hbase.util.FSUtils;
64  import org.apache.hadoop.hbase.util.Pair;
65  import org.apache.hadoop.io.BytesWritable;
66  import org.apache.hadoop.io.NullWritable;
67  import org.apache.hadoop.io.SequenceFile;
68  import org.apache.hadoop.io.Writable;
69  import org.apache.hadoop.mapreduce.Job;
70  import org.apache.hadoop.mapreduce.JobContext;
71  import org.apache.hadoop.mapreduce.Mapper;
72  import org.apache.hadoop.mapreduce.InputFormat;
73  import org.apache.hadoop.mapreduce.InputSplit;
74  import org.apache.hadoop.mapreduce.RecordReader;
75  import org.apache.hadoop.mapreduce.TaskAttemptContext;
76  import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
77  import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
78  import org.apache.hadoop.mapreduce.security.TokenCache;
79  import org.apache.hadoop.util.StringUtils;
80  import org.apache.hadoop.util.Tool;
81  import org.apache.hadoop.util.ToolRunner;
82  
83  /**
84   * Export the specified snapshot to a given FileSystem.
85   *
86   * The .snapshot/name folder is copied to the destination cluster
87   * and then all the hfiles/hlogs are copied using a Map-Reduce Job in the .archive/ location.
88   * When everything is done, the second cluster can restore the snapshot.
89   */
90  @InterfaceAudience.Public
91  @InterfaceStability.Evolving
92  public class ExportSnapshot extends Configured implements Tool {
93    private static final Log LOG = LogFactory.getLog(ExportSnapshot.class);
94  
95    private static final String MR_NUM_MAPS = "mapreduce.job.maps";
96    private static final String CONF_NUM_SPLITS = "snapshot.export.format.splits";
97    private static final String CONF_SNAPSHOT_NAME = "snapshot.export.format.snapshot.name";
98    private static final String CONF_SNAPSHOT_DIR = "snapshot.export.format.snapshot.dir";
99    private static final String CONF_FILES_USER = "snapshot.export.files.attributes.user";
100   private static final String CONF_FILES_GROUP = "snapshot.export.files.attributes.group";
101   private static final String CONF_FILES_MODE = "snapshot.export.files.attributes.mode";
102   private static final String CONF_CHECKSUM_VERIFY = "snapshot.export.checksum.verify";
103   private static final String CONF_OUTPUT_ROOT = "snapshot.export.output.root";
104   private static final String CONF_INPUT_ROOT = "snapshot.export.input.root";
105   private static final String CONF_BANDWIDTH_MB = "snapshot.export.map.bandwidth.mb";
106   private static final String CONF_BUFFER_SIZE = "snapshot.export.buffer.size";
107   private static final String CONF_MAP_GROUP = "snapshot.export.default.map.group";
108   protected static final String CONF_SKIP_TMP = "snapshot.export.skip.tmp";
109 
110   static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
111   static final String CONF_TEST_RETRY = "test.snapshot.export.failure.retry";
112 
113   private static final String INPUT_FOLDER_PREFIX = "export-files.";
114 
115   // Export Map-Reduce Counters, to keep track of the progress
116   public enum Counter { MISSING_FILES, COPY_FAILED, BYTES_EXPECTED, BYTES_COPIED, FILES_COPIED };
117 
118   private static class ExportMapper extends Mapper<BytesWritable, NullWritable,
119                                                    NullWritable, NullWritable> {
120     final static int REPORT_SIZE = 1 * 1024 * 1024;
121     final static int BUFFER_SIZE = 64 * 1024;
122 
123     private boolean testFailures;
124     private Random random;
125 
126     private boolean verifyChecksum;
127     private String filesGroup;
128     private String filesUser;
129     private short filesMode;
130     private int bufferSize;
131 
132     private FileSystem outputFs;
133     private Path outputArchive;
134     private Path outputRoot;
135 
136     private FileSystem inputFs;
137     private Path inputArchive;
138     private Path inputRoot;
139 
140     @Override
141     public void setup(Context context) throws IOException {
142       Configuration conf = context.getConfiguration();
143       verifyChecksum = conf.getBoolean(CONF_CHECKSUM_VERIFY, true);
144 
145       filesGroup = conf.get(CONF_FILES_GROUP);
146       filesUser = conf.get(CONF_FILES_USER);
147       filesMode = (short)conf.getInt(CONF_FILES_MODE, 0);
148       outputRoot = new Path(conf.get(CONF_OUTPUT_ROOT));
149       inputRoot = new Path(conf.get(CONF_INPUT_ROOT));
150 
151       inputArchive = new Path(inputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
152       outputArchive = new Path(outputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
153 
154       testFailures = conf.getBoolean(CONF_TEST_FAILURE, false);
155 
156       try {
157         inputFs = FileSystem.get(inputRoot.toUri(), conf);
158       } catch (IOException e) {
159         throw new IOException("Could not get the input FileSystem with root=" + inputRoot, e);
160       }
161 
162       try {
163         outputFs = FileSystem.get(outputRoot.toUri(), conf);
164       } catch (IOException e) {
165         throw new IOException("Could not get the output FileSystem with root="+ outputRoot, e);
166       }
167 
168       // Use the default block size of the outputFs if bigger
169       int defaultBlockSize = Math.max((int) outputFs.getDefaultBlockSize(), BUFFER_SIZE);
170       bufferSize = conf.getInt(CONF_BUFFER_SIZE, defaultBlockSize);
171       LOG.info("Using bufferSize=" + StringUtils.humanReadableInt(bufferSize));
172     }
173 
174     byte[] copyBytes(BytesWritable  bw) {
175       byte[] result = new byte[bw.getLength()];
176       System.arraycopy(bw.getBytes(), 0, result, 0, bw.getLength());
177       return result;
178     }
179 
180     @Override
181     public void map(BytesWritable key, NullWritable value, Context context)
182         throws InterruptedException, IOException {
183       SnapshotFileInfo inputInfo = SnapshotFileInfo.parseFrom(copyBytes(key));
184       Path outputPath = getOutputPath(inputInfo);
185 
186       copyFile(context, inputInfo, outputPath);
187     }
188 
189     /**
190      * Returns the location where the inputPath will be copied.
191      */
192     private Path getOutputPath(final SnapshotFileInfo inputInfo) throws IOException {
193       Path path = null;
194       switch (inputInfo.getType()) {
195         case HFILE:
196           Path inputPath = new Path(inputInfo.getHfile());
197           String family = inputPath.getParent().getName();
198           TableName table =HFileLink.getReferencedTableName(inputPath.getName());
199           String region = HFileLink.getReferencedRegionName(inputPath.getName());
200           String hfile = HFileLink.getReferencedHFileName(inputPath.getName());
201           path = new Path(FSUtils.getTableDir(new Path("./"), table),
202               new Path(region, new Path(family, hfile)));
203           break;
204         case WAL:
205           Path oldLogsDir = new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME);
206           path = new Path(oldLogsDir, inputInfo.getWalName());
207           break;
208         default:
209           throw new IOException("Invalid File Type: " + inputInfo.getType().toString());
210       }
211       return new Path(outputArchive, path);
212     }
213 
214     /*
215      * Used by TestExportSnapshot to simulate a failure
216      */
217     private void injectTestFailure(final Context context, final SnapshotFileInfo inputInfo)
218         throws IOException {
219       if (testFailures) {
220         if (context.getConfiguration().getBoolean(CONF_TEST_RETRY, false)) {
221           if (random == null) {
222             random = new Random();
223           }
224 
225           // FLAKY-TEST-WARN: lower is better, we can get some runs without the
226           // retry, but at least we reduce the number of test failures due to
227           // this test exception from the same map task.
228           if (random.nextFloat() < 0.03) {
229             throw new IOException("TEST RETRY FAILURE: Unable to copy input=" + inputInfo
230                                   + " time=" + System.currentTimeMillis());
231           }
232         } else {
233           context.getCounter(Counter.COPY_FAILED).increment(1);
234           throw new IOException("TEST FAILURE: Unable to copy input=" + inputInfo);
235         }
236       }
237     }
238 
239     private void copyFile(final Context context, final SnapshotFileInfo inputInfo,
240         final Path outputPath) throws IOException {
241       injectTestFailure(context, inputInfo);
242 
243       // Get the file information
244       FileStatus inputStat = getSourceFileStatus(context, inputInfo);
245 
246       // Verify if the output file exists and is the same that we want to copy
247       if (outputFs.exists(outputPath)) {
248         FileStatus outputStat = outputFs.getFileStatus(outputPath);
249         if (outputStat != null && sameFile(inputStat, outputStat)) {
250           LOG.info("Skip copy " + inputStat.getPath() + " to " + outputPath + ", same file.");
251           return;
252         }
253       }
254 
255       InputStream in = openSourceFile(context, inputInfo);
256       int bandwidthMB = context.getConfiguration().getInt(CONF_BANDWIDTH_MB, 100);
257       if (Integer.MAX_VALUE != bandwidthMB) {
258         in = new ThrottledInputStream(new BufferedInputStream(in), bandwidthMB * 1024 * 1024);
259       }
260 
261       try {
262         context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen());
263 
264         // Ensure that the output folder is there and copy the file
265         outputFs.mkdirs(outputPath.getParent());
266         FSDataOutputStream out = outputFs.create(outputPath, true);
267         try {
268           copyData(context, inputStat.getPath(), in, outputPath, out, inputStat.getLen());
269         } finally {
270           out.close();
271         }
272 
273         // Try to Preserve attributes
274         if (!preserveAttributes(outputPath, inputStat)) {
275           LOG.warn("You may have to run manually chown on: " + outputPath);
276         }
277       } finally {
278         in.close();
279       }
280     }
281 
282     /**
283      * Try to Preserve the files attribute selected by the user copying them from the source file
284      * This is only required when you are exporting as a different user than "hbase" or on a system
285      * that doesn't have the "hbase" user.
286      *
287      * This is not considered a blocking failure since the user can force a chmod with the user
288      * that knows is available on the system.
289      */
290     private boolean preserveAttributes(final Path path, final FileStatus refStat) {
291       FileStatus stat;
292       try {
293         stat = outputFs.getFileStatus(path);
294       } catch (IOException e) {
295         LOG.warn("Unable to get the status for file=" + path);
296         return false;
297       }
298 
299       try {
300         if (filesMode > 0 && stat.getPermission().toShort() != filesMode) {
301           outputFs.setPermission(path, new FsPermission(filesMode));
302         } else if (refStat != null && !stat.getPermission().equals(refStat.getPermission())) {
303           outputFs.setPermission(path, refStat.getPermission());
304         }
305       } catch (IOException e) {
306         LOG.warn("Unable to set the permission for file="+ stat.getPath() +": "+ e.getMessage());
307         return false;
308       }
309 
310       boolean hasRefStat = (refStat != null);
311       String user = stringIsNotEmpty(filesUser) || !hasRefStat ? filesUser : refStat.getOwner();
312       String group = stringIsNotEmpty(filesGroup) || !hasRefStat ? filesGroup : refStat.getGroup();
313       if (stringIsNotEmpty(user) || stringIsNotEmpty(group)) {
314         try {
315           if (!(user.equals(stat.getOwner()) && group.equals(stat.getGroup()))) {
316             outputFs.setOwner(path, user, group);
317           }
318         } catch (IOException e) {
319           LOG.warn("Unable to set the owner/group for file="+ stat.getPath() +": "+ e.getMessage());
320           LOG.warn("The user/group may not exist on the destination cluster: user=" +
321                    user + " group=" + group);
322           return false;
323         }
324       }
325 
326       return true;
327     }
328 
329     private boolean stringIsNotEmpty(final String str) {
330       return str != null && str.length() > 0;
331     }
332 
333     private void copyData(final Context context,
334         final Path inputPath, final InputStream in,
335         final Path outputPath, final FSDataOutputStream out,
336         final long inputFileSize)
337         throws IOException {
338       final String statusMessage = "copied %s/" + StringUtils.humanReadableInt(inputFileSize) +
339                                    " (%.1f%%)";
340 
341       try {
342         byte[] buffer = new byte[bufferSize];
343         long totalBytesWritten = 0;
344         int reportBytes = 0;
345         int bytesRead;
346 
347         long stime = System.currentTimeMillis();
348         while ((bytesRead = in.read(buffer)) > 0) {
349           out.write(buffer, 0, bytesRead);
350           totalBytesWritten += bytesRead;
351           reportBytes += bytesRead;
352 
353           if (reportBytes >= REPORT_SIZE) {
354             context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
355             context.setStatus(String.format(statusMessage,
356                               StringUtils.humanReadableInt(totalBytesWritten),
357                               (totalBytesWritten/(float)inputFileSize) * 100.0f) +
358                               " from " + inputPath + " to " + outputPath);
359             reportBytes = 0;
360           }
361         }
362         long etime = System.currentTimeMillis();
363 
364         context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
365         context.setStatus(String.format(statusMessage,
366                           StringUtils.humanReadableInt(totalBytesWritten),
367                           (totalBytesWritten/(float)inputFileSize) * 100.0f) +
368                           " from " + inputPath + " to " + outputPath);
369 
370         // Verify that the written size match
371         if (totalBytesWritten != inputFileSize) {
372           String msg = "number of bytes copied not matching copied=" + totalBytesWritten +
373                        " expected=" + inputFileSize + " for file=" + inputPath;
374           throw new IOException(msg);
375         }
376 
377         LOG.info("copy completed for input=" + inputPath + " output=" + outputPath);
378         LOG.info("size=" + totalBytesWritten +
379             " (" + StringUtils.humanReadableInt(totalBytesWritten) + ")" +
380             " time=" + StringUtils.formatTimeDiff(etime, stime) +
381             String.format(" %.3fM/sec", (totalBytesWritten / ((etime - stime)/1000.0))/1048576.0));
382         context.getCounter(Counter.FILES_COPIED).increment(1);
383       } catch (IOException e) {
384         LOG.error("Error copying " + inputPath + " to " + outputPath, e);
385         context.getCounter(Counter.COPY_FAILED).increment(1);
386         throw e;
387       }
388     }
389 
390     /**
391      * Try to open the "source" file.
392      * Throws an IOException if the communication with the inputFs fail or
393      * if the file is not found.
394      */
395     private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo)
396         throws IOException {
397       try {
398         FileLink link = null;
399         switch (fileInfo.getType()) {
400           case HFILE:
401             Path inputPath = new Path(fileInfo.getHfile());
402             link = new HFileLink(inputRoot, inputArchive, inputPath);
403             break;
404           case WAL:
405             String serverName = fileInfo.getWalServer();
406             String logName = fileInfo.getWalName();
407             link = new HLogLink(inputRoot, serverName, logName);
408             break;
409           default:
410             throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
411         }
412         return link.open(inputFs);
413       } catch (IOException e) {
414         context.getCounter(Counter.MISSING_FILES).increment(1);
415         LOG.error("Unable to open source file=" + fileInfo.toString(), e);
416         throw e;
417       }
418     }
419 
420     private FileStatus getSourceFileStatus(Context context, final SnapshotFileInfo fileInfo)
421         throws IOException {
422       try {
423         FileLink link = null;
424         switch (fileInfo.getType()) {
425           case HFILE:
426             Path inputPath = new Path(fileInfo.getHfile());
427             link = new HFileLink(inputRoot, inputArchive, inputPath);
428             break;
429           case WAL:
430             link = new HLogLink(inputRoot, fileInfo.getWalServer(), fileInfo.getWalName());
431             break;
432           default:
433             throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
434         }
435         return link.getFileStatus(inputFs);
436       } catch (FileNotFoundException e) {
437         context.getCounter(Counter.MISSING_FILES).increment(1);
438         LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e);
439         throw e;
440       } catch (IOException e) {
441         LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e);
442         throw e;
443       }
444     }
445 
446     private FileChecksum getFileChecksum(final FileSystem fs, final Path path) {
447       try {
448         return fs.getFileChecksum(path);
449       } catch (IOException e) {
450         LOG.warn("Unable to get checksum for file=" + path, e);
451         return null;
452       }
453     }
454 
455     /**
456      * Check if the two files are equal by looking at the file length,
457      * and at the checksum (if user has specified the verifyChecksum flag).
458      */
459     private boolean sameFile(final FileStatus inputStat, final FileStatus outputStat) {
460       // Not matching length
461       if (inputStat.getLen() != outputStat.getLen()) return false;
462 
463       // Mark files as equals, since user asked for no checksum verification
464       if (!verifyChecksum) return true;
465 
466       // If checksums are not available, files are not the same.
467       FileChecksum inChecksum = getFileChecksum(inputFs, inputStat.getPath());
468       if (inChecksum == null) return false;
469 
470       FileChecksum outChecksum = getFileChecksum(outputFs, outputStat.getPath());
471       if (outChecksum == null) return false;
472 
473       return inChecksum.equals(outChecksum);
474     }
475   }
476 
477   // ==========================================================================
478   //  Input Format
479   // ==========================================================================
480 
481   /**
482    * Extract the list of files (HFiles/HLogs) to copy using Map-Reduce.
483    * @return list of files referenced by the snapshot (pair of path and size)
484    */
485   private static List<Pair<SnapshotFileInfo, Long>> getSnapshotFiles(final Configuration conf,
486       final FileSystem fs, final Path snapshotDir) throws IOException {
487     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
488 
489     final List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<Pair<SnapshotFileInfo, Long>>();
490     final TableName table = TableName.valueOf(snapshotDesc.getTable());
491 
492     // Get snapshot files
493     LOG.info("Loading Snapshot '" + snapshotDesc.getName() + "' hfile list");
494     SnapshotReferenceUtil.visitReferencedFiles(conf, fs, snapshotDir, snapshotDesc,
495       new SnapshotReferenceUtil.SnapshotVisitor() {
496         @Override
497         public void storeFile(final HRegionInfo regionInfo, final String family,
498             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
499           if (storeFile.hasReference()) {
500             // copied as part of the manifest
501           } else {
502             String region = regionInfo.getEncodedName();
503             String hfile = storeFile.getName();
504             Path path = HFileLink.createPath(table, region, family, hfile);
505 
506             SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
507               .setType(SnapshotFileInfo.Type.HFILE)
508               .setHfile(path.toString())
509               .build();
510 
511             long size;
512             if (storeFile.hasFileSize()) {
513               size = storeFile.getFileSize();
514             } else {
515               size = new HFileLink(conf, path).getFileStatus(fs).getLen();
516             }
517             files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, size));
518           }
519         }
520 
521         @Override
522         public void logFile (final String server, final String logfile)
523             throws IOException {
524           SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
525             .setType(SnapshotFileInfo.Type.WAL)
526             .setWalServer(server)
527             .setWalName(logfile)
528             .build();
529 
530           long size = new HLogLink(conf, server, logfile).getFileStatus(fs).getLen();
531           files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, size));
532         }
533     });
534 
535     return files;
536   }
537 
538   /**
539    * Given a list of file paths and sizes, create around ngroups in as balanced a way as possible.
540    * The groups created will have similar amounts of bytes.
541    * <p>
542    * The algorithm used is pretty straightforward; the file list is sorted by size,
543    * and then each group fetch the bigger file available, iterating through groups
544    * alternating the direction.
545    */
546   static List<List<Pair<SnapshotFileInfo, Long>>> getBalancedSplits(
547       final List<Pair<SnapshotFileInfo, Long>> files, final int ngroups) {
548     // Sort files by size, from small to big
549     Collections.sort(files, new Comparator<Pair<SnapshotFileInfo, Long>>() {
550       public int compare(Pair<SnapshotFileInfo, Long> a, Pair<SnapshotFileInfo, Long> b) {
551         long r = a.getSecond() - b.getSecond();
552         return (r < 0) ? -1 : ((r > 0) ? 1 : 0);
553       }
554     });
555 
556     // create balanced groups
557     List<List<Pair<SnapshotFileInfo, Long>>> fileGroups =
558       new LinkedList<List<Pair<SnapshotFileInfo, Long>>>();
559     long[] sizeGroups = new long[ngroups];
560     int hi = files.size() - 1;
561     int lo = 0;
562 
563     List<Pair<SnapshotFileInfo, Long>> group;
564     int dir = 1;
565     int g = 0;
566 
567     while (hi >= lo) {
568       if (g == fileGroups.size()) {
569         group = new LinkedList<Pair<SnapshotFileInfo, Long>>();
570         fileGroups.add(group);
571       } else {
572         group = fileGroups.get(g);
573       }
574 
575       Pair<SnapshotFileInfo, Long> fileInfo = files.get(hi--);
576 
577       // add the hi one
578       sizeGroups[g] += fileInfo.getSecond();
579       group.add(fileInfo);
580 
581       // change direction when at the end or the beginning
582       g += dir;
583       if (g == ngroups) {
584         dir = -1;
585         g = ngroups - 1;
586       } else if (g < 0) {
587         dir = 1;
588         g = 0;
589       }
590     }
591 
592     if (LOG.isDebugEnabled()) {
593       for (int i = 0; i < sizeGroups.length; ++i) {
594         LOG.debug("export split=" + i + " size=" + StringUtils.humanReadableInt(sizeGroups[i]));
595       }
596     }
597 
598     return fileGroups;
599   }
600 
601   private static class ExportSnapshotInputFormat extends InputFormat<BytesWritable, NullWritable> {
602     @Override
603     public RecordReader<BytesWritable, NullWritable> createRecordReader(InputSplit split,
604         TaskAttemptContext tac) throws IOException, InterruptedException {
605       return new ExportSnapshotRecordReader(((ExportSnapshotInputSplit)split).getSplitKeys());
606     }
607 
608     @Override
609     public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
610       Configuration conf = context.getConfiguration();
611       String snapshotName = conf.get(CONF_SNAPSHOT_NAME);
612       Path snapshotDir = new Path(conf.get(CONF_SNAPSHOT_DIR));
613       FileSystem fs = FileSystem.get(snapshotDir.toUri(), conf);
614 
615       List<Pair<SnapshotFileInfo, Long>> snapshotFiles = getSnapshotFiles(conf, fs, snapshotDir);
616       int mappers = conf.getInt(CONF_NUM_SPLITS, 0);
617       if (mappers == 0 && snapshotFiles.size() > 0) {
618         mappers = 1 + (snapshotFiles.size() / conf.getInt(CONF_MAP_GROUP, 10));
619         mappers = Math.min(mappers, snapshotFiles.size());
620         conf.setInt(CONF_NUM_SPLITS, mappers);
621         conf.setInt(MR_NUM_MAPS, mappers);
622       }
623 
624       List<List<Pair<SnapshotFileInfo, Long>>> groups = getBalancedSplits(snapshotFiles, mappers);
625       List<InputSplit> splits = new ArrayList(groups.size());
626       for (List<Pair<SnapshotFileInfo, Long>> files: groups) {
627         splits.add(new ExportSnapshotInputSplit(files));
628       }
629       return splits;
630     }
631 
632     private static class ExportSnapshotInputSplit extends InputSplit implements Writable {
633       private List<Pair<BytesWritable, Long>> files;
634       private long length;
635 
636       public ExportSnapshotInputSplit() {
637         this.files = null;
638       }
639 
640       public ExportSnapshotInputSplit(final List<Pair<SnapshotFileInfo, Long>> snapshotFiles) {
641         this.files = new ArrayList(snapshotFiles.size());
642         for (Pair<SnapshotFileInfo, Long> fileInfo: snapshotFiles) {
643           this.files.add(new Pair<BytesWritable, Long>(
644             new BytesWritable(fileInfo.getFirst().toByteArray()), fileInfo.getSecond()));
645           this.length += fileInfo.getSecond();
646         }
647       }
648 
649       private List<Pair<BytesWritable, Long>> getSplitKeys() {
650         return files;
651       }
652 
653       @Override
654       public long getLength() throws IOException, InterruptedException {
655         return length;
656       }
657 
658       @Override
659       public String[] getLocations() throws IOException, InterruptedException {
660         return new String[] {};
661       }
662 
663       @Override
664       public void readFields(DataInput in) throws IOException {
665         int count = in.readInt();
666         files = new ArrayList<Pair<BytesWritable, Long>>(count);
667         length = 0;
668         for (int i = 0; i < count; ++i) {
669           BytesWritable fileInfo = new BytesWritable();
670           fileInfo.readFields(in);
671           long size = in.readLong();
672           files.add(new Pair<BytesWritable, Long>(fileInfo, size));
673           length += size;
674         }
675       }
676 
677       @Override
678       public void write(DataOutput out) throws IOException {
679         out.writeInt(files.size());
680         for (final Pair<BytesWritable, Long> fileInfo: files) {
681           fileInfo.getFirst().write(out);
682           out.writeLong(fileInfo.getSecond());
683         }
684       }
685     }
686 
687     private static class ExportSnapshotRecordReader
688         extends RecordReader<BytesWritable, NullWritable> {
689       private final List<Pair<BytesWritable, Long>> files;
690       private long totalSize = 0;
691       private long procSize = 0;
692       private int index = -1;
693 
694       ExportSnapshotRecordReader(final List<Pair<BytesWritable, Long>> files) {
695         this.files = files;
696         for (Pair<BytesWritable, Long> fileInfo: files) {
697           totalSize += fileInfo.getSecond();
698         }
699       }
700 
701       @Override
702       public void close() { }
703 
704       @Override
705       public BytesWritable getCurrentKey() { return files.get(index).getFirst(); }
706 
707       @Override
708       public NullWritable getCurrentValue() { return NullWritable.get(); }
709 
710       @Override
711       public float getProgress() { return (float)procSize / totalSize; }
712 
713       @Override
714       public void initialize(InputSplit split, TaskAttemptContext tac) { }
715 
716       @Override
717       public boolean nextKeyValue() {
718         if (index >= 0) {
719           procSize += files.get(index).getSecond();
720         }
721         return(++index < files.size());
722       }
723     }
724   }
725 
726   // ==========================================================================
727   //  Tool
728   // ==========================================================================
729 
730   /**
731    * Run Map-Reduce Job to perform the files copy.
732    */
733   private void runCopyJob(final Path inputRoot, final Path outputRoot,
734       final String snapshotName, final Path snapshotDir, final boolean verifyChecksum,
735       final String filesUser, final String filesGroup, final int filesMode,
736       final int mappers, final int bandwidthMB)
737           throws IOException, InterruptedException, ClassNotFoundException {
738     Configuration conf = getConf();
739     if (filesGroup != null) conf.set(CONF_FILES_GROUP, filesGroup);
740     if (filesUser != null) conf.set(CONF_FILES_USER, filesUser);
741     if (mappers > 0) {
742       conf.setInt(CONF_NUM_SPLITS, mappers);
743       conf.setInt(MR_NUM_MAPS, mappers);
744     }
745     conf.setInt(CONF_FILES_MODE, filesMode);
746     conf.setBoolean(CONF_CHECKSUM_VERIFY, verifyChecksum);
747     conf.set(CONF_OUTPUT_ROOT, outputRoot.toString());
748     conf.set(CONF_INPUT_ROOT, inputRoot.toString());
749     conf.setInt(CONF_BANDWIDTH_MB, bandwidthMB);
750     conf.set(CONF_SNAPSHOT_NAME, snapshotName);
751     conf.set(CONF_SNAPSHOT_DIR, snapshotDir.toString());
752 
753     Job job = new Job(conf);
754     job.setJobName("ExportSnapshot-" + snapshotName);
755     job.setJarByClass(ExportSnapshot.class);
756     TableMapReduceUtil.addDependencyJars(job);
757     job.setMapperClass(ExportMapper.class);
758     job.setInputFormatClass(ExportSnapshotInputFormat.class);
759     job.setOutputFormatClass(NullOutputFormat.class);
760     job.setMapSpeculativeExecution(false);
761     job.setNumReduceTasks(0);
762 
763     // Acquire the delegation Tokens
764     TokenCache.obtainTokensForNamenodes(job.getCredentials(),
765       new Path[] { inputRoot, outputRoot }, conf);
766 
767     // Run the MR Job
768     if (!job.waitForCompletion(true)) {
769       // TODO: Replace the fixed string with job.getStatus().getFailureInfo()
770       // when it will be available on all the supported versions.
771       throw new ExportSnapshotException("Copy Files Map-Reduce Job failed");
772     }
773   }
774 
775   private void verifySnapshot(final Configuration baseConf,
776       final FileSystem fs, final Path rootDir, final Path snapshotDir) throws IOException {
777     // Update the conf with the current root dir, since may be a different cluster
778     Configuration conf = new Configuration(baseConf);
779     FSUtils.setRootDir(conf, rootDir);
780     FSUtils.setFsDefault(conf, FSUtils.getRootDir(conf));
781     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
782     SnapshotReferenceUtil.verifySnapshot(conf, fs, snapshotDir, snapshotDesc);
783   }
784 
785   /**
786    * Execute the export snapshot by copying the snapshot metadata, hfiles and hlogs.
787    * @return 0 on success, and != 0 upon failure.
788    */
789   @Override
790   public int run(String[] args) throws IOException {
791     boolean verifyTarget = true;
792     boolean verifyChecksum = true;
793     String snapshotName = null;
794     String targetName = null;
795     boolean overwrite = false;
796     String filesGroup = null;
797     String filesUser = null;
798     Path outputRoot = null;
799     int bandwidthMB = Integer.MAX_VALUE;
800     int filesMode = 0;
801     int mappers = 0;
802 
803     Configuration conf = getConf();
804     Path inputRoot = FSUtils.getRootDir(conf);
805 
806     // Process command line args
807     for (int i = 0; i < args.length; i++) {
808       String cmd = args[i];
809       if (cmd.equals("-snapshot")) {
810         snapshotName = args[++i];
811       } else if (cmd.equals("-target")) {
812         targetName = args[++i];
813       } else if (cmd.equals("-copy-to")) {
814         outputRoot = new Path(args[++i]);
815       } else if (cmd.equals("-copy-from")) {
816         inputRoot = new Path(args[++i]);
817         FSUtils.setRootDir(conf, inputRoot);
818       } else if (cmd.equals("-no-checksum-verify")) {
819         verifyChecksum = false;
820       } else if (cmd.equals("-no-target-verify")) {
821         verifyTarget = false;
822       } else if (cmd.equals("-mappers")) {
823         mappers = Integer.parseInt(args[++i]);
824       } else if (cmd.equals("-chuser")) {
825         filesUser = args[++i];
826       } else if (cmd.equals("-chgroup")) {
827         filesGroup = args[++i];
828       } else if (cmd.equals("-bandwidth")) {
829         bandwidthMB = Integer.parseInt(args[++i]);
830       } else if (cmd.equals("-chmod")) {
831         filesMode = Integer.parseInt(args[++i], 8);
832       } else if (cmd.equals("-overwrite")) {
833         overwrite = true;
834       } else if (cmd.equals("-h") || cmd.equals("--help")) {
835         printUsageAndExit();
836       } else {
837         System.err.println("UNEXPECTED: " + cmd);
838         printUsageAndExit();
839       }
840     }
841 
842     // Check user options
843     if (snapshotName == null) {
844       System.err.println("Snapshot name not provided.");
845       printUsageAndExit();
846     }
847 
848     if (outputRoot == null) {
849       System.err.println("Destination file-system not provided.");
850       printUsageAndExit();
851     }
852 
853     if (targetName == null) {
854       targetName = snapshotName;
855     }
856 
857     FileSystem inputFs = FileSystem.get(inputRoot.toUri(), conf);
858     LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot);
859     FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);
860     LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString());
861 
862     boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
863 
864     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
865     Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot);
866     Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot);
867     Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
868 
869     // Check if the snapshot already exists
870     if (outputFs.exists(outputSnapshotDir)) {
871       if (overwrite) {
872         if (!outputFs.delete(outputSnapshotDir, true)) {
873           System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir);
874           return 1;
875         }
876       } else {
877         System.err.println("The snapshot '" + targetName +
878           "' already exists in the destination: " + outputSnapshotDir);
879         return 1;
880       }
881     }
882 
883     if (!skipTmp) {
884       // Check if the snapshot already in-progress
885       if (outputFs.exists(snapshotTmpDir)) {
886         if (overwrite) {
887           if (!outputFs.delete(snapshotTmpDir, true)) {
888             System.err.println("Unable to remove existing snapshot tmp directory: "+snapshotTmpDir);
889             return 1;
890           }
891         } else {
892           System.err.println("A snapshot with the same name '"+ targetName +"' may be in-progress");
893           System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");
894           System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");
895           return 1;
896         }
897       }
898     }
899 
900     // Step 1 - Copy fs1:/.snapshot/<snapshot> to  fs2:/.snapshot/.tmp/<snapshot>
901     // The snapshot references must be copied before the hfiles otherwise the cleaner
902     // will remove them because they are unreferenced.
903     try {
904       LOG.info("Copy Snapshot Manifest");
905       FileUtil.copy(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, false, false, conf);
906     } catch (IOException e) {
907       throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" +
908         snapshotDir + " to=" + initialOutputSnapshotDir, e);
909     }
910 
911     // Write a new .snapshotinfo if the target name is different from the source name
912     if (!targetName.equals(snapshotName)) {
913       SnapshotDescription snapshotDesc =
914         SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir)
915           .toBuilder()
916           .setName(targetName)
917           .build();
918       SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, snapshotTmpDir, outputFs);
919     }
920 
921     // Step 2 - Start MR Job to copy files
922     // The snapshot references must be copied before the files otherwise the files gets removed
923     // by the HFileArchiver, since they have no references.
924     try {
925       runCopyJob(inputRoot, outputRoot, snapshotName, snapshotDir, verifyChecksum,
926                  filesUser, filesGroup, filesMode, mappers, bandwidthMB);
927 
928       LOG.info("Finalize the Snapshot Export");
929       if (!skipTmp) {
930         // Step 3 - Rename fs2:/.snapshot/.tmp/<snapshot> fs2:/.snapshot/<snapshot>
931         if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
932           throw new ExportSnapshotException("Unable to rename snapshot directory from=" +
933             snapshotTmpDir + " to=" + outputSnapshotDir);
934         }
935       }
936 
937       // Step 4 - Verify snapshot integrity
938       if (verifyTarget) {
939         LOG.info("Verify snapshot integrity");
940         verifySnapshot(conf, outputFs, outputRoot, outputSnapshotDir);
941       }
942 
943       LOG.info("Export Completed: " + targetName);
944       return 0;
945     } catch (Exception e) {
946       LOG.error("Snapshot export failed", e);
947       if (!skipTmp) {
948         outputFs.delete(snapshotTmpDir, true);
949       }
950       outputFs.delete(outputSnapshotDir, true);
951       return 1;
952     }
953   }
954 
955   // ExportSnapshot
956   private void printUsageAndExit() {
957     System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
958     System.err.println(" where [options] are:");
959     System.err.println("  -h|-help                Show this help and exit.");
960     System.err.println("  -snapshot NAME          Snapshot to restore.");
961     System.err.println("  -copy-to NAME           Remote destination hdfs://");
962     System.err.println("  -copy-from NAME         Input folder hdfs:// (default hbase.rootdir)");
963     System.err.println("  -no-checksum-verify     Do not verify checksum, use name+length only.");
964     System.err.println("  -no-target-verify       Do not verify the integrity of the \\" +
965         "exported snapshot.");
966     System.err.println("  -overwrite              Rewrite the snapshot manifest if already exists");
967     System.err.println("  -chuser USERNAME        Change the owner of the files to the specified one.");
968     System.err.println("  -chgroup GROUP          Change the group of the files to the specified one.");
969     System.err.println("  -chmod MODE             Change the permission of the files to the specified one.");
970     System.err.println("  -mappers                Number of mappers to use during the copy (mapreduce.job.maps).");
971     System.err.println();
972     System.err.println("Examples:");
973     System.err.println("  hbase " + getClass().getName() + " \\");
974     System.err.println("    -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase \\");
975     System.err.println("    -chuser MyUser -chgroup MyGroup -chmod 700 -mappers 16");
976     System.err.println();
977     System.err.println("  hbase " + getClass().getName() + " \\");
978     System.err.println("    -snapshot MySnapshot -copy-from hdfs://srv2:8082/hbase \\");
979     System.err.println("    -copy-to hdfs://srv1:50070/hbase \\");
980     System.exit(1);
981   }
982 
983   /**
984    * The guts of the {@link #main} method.
985    * Call this method to avoid the {@link #main(String[])} System.exit.
986    * @param args
987    * @return errCode
988    * @throws Exception
989    */
990   static int innerMain(final Configuration conf, final String [] args) throws Exception {
991     return ToolRunner.run(conf, new ExportSnapshot(), args);
992   }
993 
994   public static void main(String[] args) throws Exception {
995     System.exit(innerMain(HBaseConfiguration.create(), args));
996   }
997 }