1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.BufferedInputStream;
22 import java.io.FileNotFoundException;
23 import java.io.DataInput;
24 import java.io.DataOutput;
25 import java.io.IOException;
26 import java.io.InputStream;
27 import java.net.URI;
28 import java.util.ArrayList;
29 import java.util.Collections;
30 import java.util.Comparator;
31 import java.util.LinkedList;
32 import java.util.List;
33 import java.util.Random;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.hbase.classification.InterfaceAudience;
38 import org.apache.hadoop.hbase.classification.InterfaceStability;
39 import org.apache.hadoop.conf.Configuration;
40 import org.apache.hadoop.conf.Configured;
41 import org.apache.hadoop.fs.FSDataInputStream;
42 import org.apache.hadoop.fs.FSDataOutputStream;
43 import org.apache.hadoop.fs.FileChecksum;
44 import org.apache.hadoop.fs.FileStatus;
45 import org.apache.hadoop.fs.FileSystem;
46 import org.apache.hadoop.fs.FileUtil;
47 import org.apache.hadoop.fs.Path;
48 import org.apache.hadoop.fs.permission.FsPermission;
49 import org.apache.hadoop.hbase.TableName;
50 import org.apache.hadoop.hbase.HBaseConfiguration;
51 import org.apache.hadoop.hbase.HConstants;
52 import org.apache.hadoop.hbase.HRegionInfo;
53 import org.apache.hadoop.hbase.io.FileLink;
54 import org.apache.hadoop.hbase.io.HFileLink;
55 import org.apache.hadoop.hbase.io.HLogLink;
56 import org.apache.hadoop.hbase.io.hadoopbackport.ThrottledInputStream;
57 import org.apache.hadoop.hbase.mapreduce.JobUtil;
58 import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
59 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
60 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
61 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
62 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
63 import org.apache.hadoop.hbase.util.FSUtils;
64 import org.apache.hadoop.hbase.util.Pair;
65 import org.apache.hadoop.io.BytesWritable;
66 import org.apache.hadoop.io.NullWritable;
67 import org.apache.hadoop.io.SequenceFile;
68 import org.apache.hadoop.io.Writable;
69 import org.apache.hadoop.mapreduce.Job;
70 import org.apache.hadoop.mapreduce.JobContext;
71 import org.apache.hadoop.mapreduce.Mapper;
72 import org.apache.hadoop.mapreduce.InputFormat;
73 import org.apache.hadoop.mapreduce.InputSplit;
74 import org.apache.hadoop.mapreduce.RecordReader;
75 import org.apache.hadoop.mapreduce.TaskAttemptContext;
76 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
77 import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
78 import org.apache.hadoop.mapreduce.security.TokenCache;
79 import org.apache.hadoop.util.StringUtils;
80 import org.apache.hadoop.util.Tool;
81 import org.apache.hadoop.util.ToolRunner;
82
83
84
85
86
87
88
89
90 @InterfaceAudience.Public
91 @InterfaceStability.Evolving
92 public class ExportSnapshot extends Configured implements Tool {
93 private static final Log LOG = LogFactory.getLog(ExportSnapshot.class);
94
95 private static final String MR_NUM_MAPS = "mapreduce.job.maps";
96 private static final String CONF_NUM_SPLITS = "snapshot.export.format.splits";
97 private static final String CONF_SNAPSHOT_NAME = "snapshot.export.format.snapshot.name";
98 private static final String CONF_SNAPSHOT_DIR = "snapshot.export.format.snapshot.dir";
99 private static final String CONF_FILES_USER = "snapshot.export.files.attributes.user";
100 private static final String CONF_FILES_GROUP = "snapshot.export.files.attributes.group";
101 private static final String CONF_FILES_MODE = "snapshot.export.files.attributes.mode";
102 private static final String CONF_CHECKSUM_VERIFY = "snapshot.export.checksum.verify";
103 private static final String CONF_OUTPUT_ROOT = "snapshot.export.output.root";
104 private static final String CONF_INPUT_ROOT = "snapshot.export.input.root";
105 private static final String CONF_BANDWIDTH_MB = "snapshot.export.map.bandwidth.mb";
106 private static final String CONF_BUFFER_SIZE = "snapshot.export.buffer.size";
107 private static final String CONF_MAP_GROUP = "snapshot.export.default.map.group";
108 protected static final String CONF_SKIP_TMP = "snapshot.export.skip.tmp";
109
110 static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
111 static final String CONF_TEST_RETRY = "test.snapshot.export.failure.retry";
112
113 private static final String INPUT_FOLDER_PREFIX = "export-files.";
114
115
116 public enum Counter { MISSING_FILES, COPY_FAILED, BYTES_EXPECTED, BYTES_COPIED, FILES_COPIED };
117
118 private static class ExportMapper extends Mapper<BytesWritable, NullWritable,
119 NullWritable, NullWritable> {
120 final static int REPORT_SIZE = 1 * 1024 * 1024;
121 final static int BUFFER_SIZE = 64 * 1024;
122
123 private boolean testFailures;
124 private Random random;
125
126 private boolean verifyChecksum;
127 private String filesGroup;
128 private String filesUser;
129 private short filesMode;
130 private int bufferSize;
131
132 private FileSystem outputFs;
133 private Path outputArchive;
134 private Path outputRoot;
135
136 private FileSystem inputFs;
137 private Path inputArchive;
138 private Path inputRoot;
139
140 @Override
141 public void setup(Context context) throws IOException {
142 Configuration conf = context.getConfiguration();
143 verifyChecksum = conf.getBoolean(CONF_CHECKSUM_VERIFY, true);
144
145 filesGroup = conf.get(CONF_FILES_GROUP);
146 filesUser = conf.get(CONF_FILES_USER);
147 filesMode = (short)conf.getInt(CONF_FILES_MODE, 0);
148 outputRoot = new Path(conf.get(CONF_OUTPUT_ROOT));
149 inputRoot = new Path(conf.get(CONF_INPUT_ROOT));
150
151 inputArchive = new Path(inputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
152 outputArchive = new Path(outputRoot, HConstants.HFILE_ARCHIVE_DIRECTORY);
153
154 testFailures = conf.getBoolean(CONF_TEST_FAILURE, false);
155
156 try {
157 inputFs = FileSystem.get(inputRoot.toUri(), conf);
158 } catch (IOException e) {
159 throw new IOException("Could not get the input FileSystem with root=" + inputRoot, e);
160 }
161
162 try {
163 outputFs = FileSystem.get(outputRoot.toUri(), conf);
164 } catch (IOException e) {
165 throw new IOException("Could not get the output FileSystem with root="+ outputRoot, e);
166 }
167
168
169 int defaultBlockSize = Math.max((int) outputFs.getDefaultBlockSize(), BUFFER_SIZE);
170 bufferSize = conf.getInt(CONF_BUFFER_SIZE, defaultBlockSize);
171 LOG.info("Using bufferSize=" + StringUtils.humanReadableInt(bufferSize));
172 }
173
174 byte[] copyBytes(BytesWritable bw) {
175 byte[] result = new byte[bw.getLength()];
176 System.arraycopy(bw.getBytes(), 0, result, 0, bw.getLength());
177 return result;
178 }
179
180 @Override
181 public void map(BytesWritable key, NullWritable value, Context context)
182 throws InterruptedException, IOException {
183 SnapshotFileInfo inputInfo = SnapshotFileInfo.parseFrom(copyBytes(key));
184 Path outputPath = getOutputPath(inputInfo);
185
186 copyFile(context, inputInfo, outputPath);
187 }
188
189
190
191
192 private Path getOutputPath(final SnapshotFileInfo inputInfo) throws IOException {
193 Path path = null;
194 switch (inputInfo.getType()) {
195 case HFILE:
196 Path inputPath = new Path(inputInfo.getHfile());
197 String family = inputPath.getParent().getName();
198 TableName table =HFileLink.getReferencedTableName(inputPath.getName());
199 String region = HFileLink.getReferencedRegionName(inputPath.getName());
200 String hfile = HFileLink.getReferencedHFileName(inputPath.getName());
201 path = new Path(FSUtils.getTableDir(new Path("./"), table),
202 new Path(region, new Path(family, hfile)));
203 break;
204 case WAL:
205 Path oldLogsDir = new Path(outputRoot, HConstants.HREGION_OLDLOGDIR_NAME);
206 path = new Path(oldLogsDir, inputInfo.getWalName());
207 break;
208 default:
209 throw new IOException("Invalid File Type: " + inputInfo.getType().toString());
210 }
211 return new Path(outputArchive, path);
212 }
213
214
215
216
217 private void injectTestFailure(final Context context, final SnapshotFileInfo inputInfo)
218 throws IOException {
219 if (testFailures) {
220 if (context.getConfiguration().getBoolean(CONF_TEST_RETRY, false)) {
221 if (random == null) {
222 random = new Random();
223 }
224
225
226
227
228 if (random.nextFloat() < 0.03) {
229 throw new IOException("TEST RETRY FAILURE: Unable to copy input=" + inputInfo
230 + " time=" + System.currentTimeMillis());
231 }
232 } else {
233 context.getCounter(Counter.COPY_FAILED).increment(1);
234 throw new IOException("TEST FAILURE: Unable to copy input=" + inputInfo);
235 }
236 }
237 }
238
239 private void copyFile(final Context context, final SnapshotFileInfo inputInfo,
240 final Path outputPath) throws IOException {
241 injectTestFailure(context, inputInfo);
242
243
244 FileStatus inputStat = getSourceFileStatus(context, inputInfo);
245
246
247 if (outputFs.exists(outputPath)) {
248 FileStatus outputStat = outputFs.getFileStatus(outputPath);
249 if (outputStat != null && sameFile(inputStat, outputStat)) {
250 LOG.info("Skip copy " + inputStat.getPath() + " to " + outputPath + ", same file.");
251 return;
252 }
253 }
254
255 InputStream in = openSourceFile(context, inputInfo);
256 int bandwidthMB = context.getConfiguration().getInt(CONF_BANDWIDTH_MB, 100);
257 if (Integer.MAX_VALUE != bandwidthMB) {
258 in = new ThrottledInputStream(new BufferedInputStream(in), bandwidthMB * 1024 * 1024);
259 }
260
261 try {
262 context.getCounter(Counter.BYTES_EXPECTED).increment(inputStat.getLen());
263
264
265 outputFs.mkdirs(outputPath.getParent());
266 FSDataOutputStream out = outputFs.create(outputPath, true);
267 try {
268 copyData(context, inputStat.getPath(), in, outputPath, out, inputStat.getLen());
269 } finally {
270 out.close();
271 }
272
273
274 if (!preserveAttributes(outputPath, inputStat)) {
275 LOG.warn("You may have to run manually chown on: " + outputPath);
276 }
277 } finally {
278 in.close();
279 }
280 }
281
282
283
284
285
286
287
288
289
290 private boolean preserveAttributes(final Path path, final FileStatus refStat) {
291 FileStatus stat;
292 try {
293 stat = outputFs.getFileStatus(path);
294 } catch (IOException e) {
295 LOG.warn("Unable to get the status for file=" + path);
296 return false;
297 }
298
299 try {
300 if (filesMode > 0 && stat.getPermission().toShort() != filesMode) {
301 outputFs.setPermission(path, new FsPermission(filesMode));
302 } else if (refStat != null && !stat.getPermission().equals(refStat.getPermission())) {
303 outputFs.setPermission(path, refStat.getPermission());
304 }
305 } catch (IOException e) {
306 LOG.warn("Unable to set the permission for file="+ stat.getPath() +": "+ e.getMessage());
307 return false;
308 }
309
310 boolean hasRefStat = (refStat != null);
311 String user = stringIsNotEmpty(filesUser) || !hasRefStat ? filesUser : refStat.getOwner();
312 String group = stringIsNotEmpty(filesGroup) || !hasRefStat ? filesGroup : refStat.getGroup();
313 if (stringIsNotEmpty(user) || stringIsNotEmpty(group)) {
314 try {
315 if (!(user.equals(stat.getOwner()) && group.equals(stat.getGroup()))) {
316 outputFs.setOwner(path, user, group);
317 }
318 } catch (IOException e) {
319 LOG.warn("Unable to set the owner/group for file="+ stat.getPath() +": "+ e.getMessage());
320 LOG.warn("The user/group may not exist on the destination cluster: user=" +
321 user + " group=" + group);
322 return false;
323 }
324 }
325
326 return true;
327 }
328
329 private boolean stringIsNotEmpty(final String str) {
330 return str != null && str.length() > 0;
331 }
332
333 private void copyData(final Context context,
334 final Path inputPath, final InputStream in,
335 final Path outputPath, final FSDataOutputStream out,
336 final long inputFileSize)
337 throws IOException {
338 final String statusMessage = "copied %s/" + StringUtils.humanReadableInt(inputFileSize) +
339 " (%.1f%%)";
340
341 try {
342 byte[] buffer = new byte[bufferSize];
343 long totalBytesWritten = 0;
344 int reportBytes = 0;
345 int bytesRead;
346
347 long stime = System.currentTimeMillis();
348 while ((bytesRead = in.read(buffer)) > 0) {
349 out.write(buffer, 0, bytesRead);
350 totalBytesWritten += bytesRead;
351 reportBytes += bytesRead;
352
353 if (reportBytes >= REPORT_SIZE) {
354 context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
355 context.setStatus(String.format(statusMessage,
356 StringUtils.humanReadableInt(totalBytesWritten),
357 (totalBytesWritten/(float)inputFileSize) * 100.0f) +
358 " from " + inputPath + " to " + outputPath);
359 reportBytes = 0;
360 }
361 }
362 long etime = System.currentTimeMillis();
363
364 context.getCounter(Counter.BYTES_COPIED).increment(reportBytes);
365 context.setStatus(String.format(statusMessage,
366 StringUtils.humanReadableInt(totalBytesWritten),
367 (totalBytesWritten/(float)inputFileSize) * 100.0f) +
368 " from " + inputPath + " to " + outputPath);
369
370
371 if (totalBytesWritten != inputFileSize) {
372 String msg = "number of bytes copied not matching copied=" + totalBytesWritten +
373 " expected=" + inputFileSize + " for file=" + inputPath;
374 throw new IOException(msg);
375 }
376
377 LOG.info("copy completed for input=" + inputPath + " output=" + outputPath);
378 LOG.info("size=" + totalBytesWritten +
379 " (" + StringUtils.humanReadableInt(totalBytesWritten) + ")" +
380 " time=" + StringUtils.formatTimeDiff(etime, stime) +
381 String.format(" %.3fM/sec", (totalBytesWritten / ((etime - stime)/1000.0))/1048576.0));
382 context.getCounter(Counter.FILES_COPIED).increment(1);
383 } catch (IOException e) {
384 LOG.error("Error copying " + inputPath + " to " + outputPath, e);
385 context.getCounter(Counter.COPY_FAILED).increment(1);
386 throw e;
387 }
388 }
389
390
391
392
393
394
395 private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo)
396 throws IOException {
397 try {
398 FileLink link = null;
399 switch (fileInfo.getType()) {
400 case HFILE:
401 Path inputPath = new Path(fileInfo.getHfile());
402 link = new HFileLink(inputRoot, inputArchive, inputPath);
403 break;
404 case WAL:
405 String serverName = fileInfo.getWalServer();
406 String logName = fileInfo.getWalName();
407 link = new HLogLink(inputRoot, serverName, logName);
408 break;
409 default:
410 throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
411 }
412 return link.open(inputFs);
413 } catch (IOException e) {
414 context.getCounter(Counter.MISSING_FILES).increment(1);
415 LOG.error("Unable to open source file=" + fileInfo.toString(), e);
416 throw e;
417 }
418 }
419
420 private FileStatus getSourceFileStatus(Context context, final SnapshotFileInfo fileInfo)
421 throws IOException {
422 try {
423 FileLink link = null;
424 switch (fileInfo.getType()) {
425 case HFILE:
426 Path inputPath = new Path(fileInfo.getHfile());
427 link = new HFileLink(inputRoot, inputArchive, inputPath);
428 break;
429 case WAL:
430 link = new HLogLink(inputRoot, fileInfo.getWalServer(), fileInfo.getWalName());
431 break;
432 default:
433 throw new IOException("Invalid File Type: " + fileInfo.getType().toString());
434 }
435 return link.getFileStatus(inputFs);
436 } catch (FileNotFoundException e) {
437 context.getCounter(Counter.MISSING_FILES).increment(1);
438 LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e);
439 throw e;
440 } catch (IOException e) {
441 LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e);
442 throw e;
443 }
444 }
445
446 private FileChecksum getFileChecksum(final FileSystem fs, final Path path) {
447 try {
448 return fs.getFileChecksum(path);
449 } catch (IOException e) {
450 LOG.warn("Unable to get checksum for file=" + path, e);
451 return null;
452 }
453 }
454
455
456
457
458
459 private boolean sameFile(final FileStatus inputStat, final FileStatus outputStat) {
460
461 if (inputStat.getLen() != outputStat.getLen()) return false;
462
463
464 if (!verifyChecksum) return true;
465
466
467 FileChecksum inChecksum = getFileChecksum(inputFs, inputStat.getPath());
468 if (inChecksum == null) return false;
469
470 FileChecksum outChecksum = getFileChecksum(outputFs, outputStat.getPath());
471 if (outChecksum == null) return false;
472
473 return inChecksum.equals(outChecksum);
474 }
475 }
476
477
478
479
480
481
482
483
484
485 private static List<Pair<SnapshotFileInfo, Long>> getSnapshotFiles(final Configuration conf,
486 final FileSystem fs, final Path snapshotDir) throws IOException {
487 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
488
489 final List<Pair<SnapshotFileInfo, Long>> files = new ArrayList<Pair<SnapshotFileInfo, Long>>();
490 final TableName table = TableName.valueOf(snapshotDesc.getTable());
491
492
493 LOG.info("Loading Snapshot '" + snapshotDesc.getName() + "' hfile list");
494 SnapshotReferenceUtil.visitReferencedFiles(conf, fs, snapshotDir, snapshotDesc,
495 new SnapshotReferenceUtil.SnapshotVisitor() {
496 @Override
497 public void storeFile(final HRegionInfo regionInfo, final String family,
498 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
499 if (storeFile.hasReference()) {
500
501 } else {
502 String region = regionInfo.getEncodedName();
503 String hfile = storeFile.getName();
504 Path path = HFileLink.createPath(table, region, family, hfile);
505
506 SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
507 .setType(SnapshotFileInfo.Type.HFILE)
508 .setHfile(path.toString())
509 .build();
510
511 long size;
512 if (storeFile.hasFileSize()) {
513 size = storeFile.getFileSize();
514 } else {
515 size = new HFileLink(conf, path).getFileStatus(fs).getLen();
516 }
517 files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, size));
518 }
519 }
520
521 @Override
522 public void logFile (final String server, final String logfile)
523 throws IOException {
524 SnapshotFileInfo fileInfo = SnapshotFileInfo.newBuilder()
525 .setType(SnapshotFileInfo.Type.WAL)
526 .setWalServer(server)
527 .setWalName(logfile)
528 .build();
529
530 long size = new HLogLink(conf, server, logfile).getFileStatus(fs).getLen();
531 files.add(new Pair<SnapshotFileInfo, Long>(fileInfo, size));
532 }
533 });
534
535 return files;
536 }
537
538
539
540
541
542
543
544
545
546 static List<List<Pair<SnapshotFileInfo, Long>>> getBalancedSplits(
547 final List<Pair<SnapshotFileInfo, Long>> files, final int ngroups) {
548
549 Collections.sort(files, new Comparator<Pair<SnapshotFileInfo, Long>>() {
550 public int compare(Pair<SnapshotFileInfo, Long> a, Pair<SnapshotFileInfo, Long> b) {
551 long r = a.getSecond() - b.getSecond();
552 return (r < 0) ? -1 : ((r > 0) ? 1 : 0);
553 }
554 });
555
556
557 List<List<Pair<SnapshotFileInfo, Long>>> fileGroups =
558 new LinkedList<List<Pair<SnapshotFileInfo, Long>>>();
559 long[] sizeGroups = new long[ngroups];
560 int hi = files.size() - 1;
561 int lo = 0;
562
563 List<Pair<SnapshotFileInfo, Long>> group;
564 int dir = 1;
565 int g = 0;
566
567 while (hi >= lo) {
568 if (g == fileGroups.size()) {
569 group = new LinkedList<Pair<SnapshotFileInfo, Long>>();
570 fileGroups.add(group);
571 } else {
572 group = fileGroups.get(g);
573 }
574
575 Pair<SnapshotFileInfo, Long> fileInfo = files.get(hi--);
576
577
578 sizeGroups[g] += fileInfo.getSecond();
579 group.add(fileInfo);
580
581
582 g += dir;
583 if (g == ngroups) {
584 dir = -1;
585 g = ngroups - 1;
586 } else if (g < 0) {
587 dir = 1;
588 g = 0;
589 }
590 }
591
592 if (LOG.isDebugEnabled()) {
593 for (int i = 0; i < sizeGroups.length; ++i) {
594 LOG.debug("export split=" + i + " size=" + StringUtils.humanReadableInt(sizeGroups[i]));
595 }
596 }
597
598 return fileGroups;
599 }
600
601 private static class ExportSnapshotInputFormat extends InputFormat<BytesWritable, NullWritable> {
602 @Override
603 public RecordReader<BytesWritable, NullWritable> createRecordReader(InputSplit split,
604 TaskAttemptContext tac) throws IOException, InterruptedException {
605 return new ExportSnapshotRecordReader(((ExportSnapshotInputSplit)split).getSplitKeys());
606 }
607
608 @Override
609 public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
610 Configuration conf = context.getConfiguration();
611 String snapshotName = conf.get(CONF_SNAPSHOT_NAME);
612 Path snapshotDir = new Path(conf.get(CONF_SNAPSHOT_DIR));
613 FileSystem fs = FileSystem.get(snapshotDir.toUri(), conf);
614
615 List<Pair<SnapshotFileInfo, Long>> snapshotFiles = getSnapshotFiles(conf, fs, snapshotDir);
616 int mappers = conf.getInt(CONF_NUM_SPLITS, 0);
617 if (mappers == 0 && snapshotFiles.size() > 0) {
618 mappers = 1 + (snapshotFiles.size() / conf.getInt(CONF_MAP_GROUP, 10));
619 mappers = Math.min(mappers, snapshotFiles.size());
620 conf.setInt(CONF_NUM_SPLITS, mappers);
621 conf.setInt(MR_NUM_MAPS, mappers);
622 }
623
624 List<List<Pair<SnapshotFileInfo, Long>>> groups = getBalancedSplits(snapshotFiles, mappers);
625 List<InputSplit> splits = new ArrayList(groups.size());
626 for (List<Pair<SnapshotFileInfo, Long>> files: groups) {
627 splits.add(new ExportSnapshotInputSplit(files));
628 }
629 return splits;
630 }
631
632 private static class ExportSnapshotInputSplit extends InputSplit implements Writable {
633 private List<Pair<BytesWritable, Long>> files;
634 private long length;
635
636 public ExportSnapshotInputSplit() {
637 this.files = null;
638 }
639
640 public ExportSnapshotInputSplit(final List<Pair<SnapshotFileInfo, Long>> snapshotFiles) {
641 this.files = new ArrayList(snapshotFiles.size());
642 for (Pair<SnapshotFileInfo, Long> fileInfo: snapshotFiles) {
643 this.files.add(new Pair<BytesWritable, Long>(
644 new BytesWritable(fileInfo.getFirst().toByteArray()), fileInfo.getSecond()));
645 this.length += fileInfo.getSecond();
646 }
647 }
648
649 private List<Pair<BytesWritable, Long>> getSplitKeys() {
650 return files;
651 }
652
653 @Override
654 public long getLength() throws IOException, InterruptedException {
655 return length;
656 }
657
658 @Override
659 public String[] getLocations() throws IOException, InterruptedException {
660 return new String[] {};
661 }
662
663 @Override
664 public void readFields(DataInput in) throws IOException {
665 int count = in.readInt();
666 files = new ArrayList<Pair<BytesWritable, Long>>(count);
667 length = 0;
668 for (int i = 0; i < count; ++i) {
669 BytesWritable fileInfo = new BytesWritable();
670 fileInfo.readFields(in);
671 long size = in.readLong();
672 files.add(new Pair<BytesWritable, Long>(fileInfo, size));
673 length += size;
674 }
675 }
676
677 @Override
678 public void write(DataOutput out) throws IOException {
679 out.writeInt(files.size());
680 for (final Pair<BytesWritable, Long> fileInfo: files) {
681 fileInfo.getFirst().write(out);
682 out.writeLong(fileInfo.getSecond());
683 }
684 }
685 }
686
687 private static class ExportSnapshotRecordReader
688 extends RecordReader<BytesWritable, NullWritable> {
689 private final List<Pair<BytesWritable, Long>> files;
690 private long totalSize = 0;
691 private long procSize = 0;
692 private int index = -1;
693
694 ExportSnapshotRecordReader(final List<Pair<BytesWritable, Long>> files) {
695 this.files = files;
696 for (Pair<BytesWritable, Long> fileInfo: files) {
697 totalSize += fileInfo.getSecond();
698 }
699 }
700
701 @Override
702 public void close() { }
703
704 @Override
705 public BytesWritable getCurrentKey() { return files.get(index).getFirst(); }
706
707 @Override
708 public NullWritable getCurrentValue() { return NullWritable.get(); }
709
710 @Override
711 public float getProgress() { return (float)procSize / totalSize; }
712
713 @Override
714 public void initialize(InputSplit split, TaskAttemptContext tac) { }
715
716 @Override
717 public boolean nextKeyValue() {
718 if (index >= 0) {
719 procSize += files.get(index).getSecond();
720 }
721 return(++index < files.size());
722 }
723 }
724 }
725
726
727
728
729
730
731
732
733 private void runCopyJob(final Path inputRoot, final Path outputRoot,
734 final String snapshotName, final Path snapshotDir, final boolean verifyChecksum,
735 final String filesUser, final String filesGroup, final int filesMode,
736 final int mappers, final int bandwidthMB)
737 throws IOException, InterruptedException, ClassNotFoundException {
738 Configuration conf = getConf();
739 if (filesGroup != null) conf.set(CONF_FILES_GROUP, filesGroup);
740 if (filesUser != null) conf.set(CONF_FILES_USER, filesUser);
741 if (mappers > 0) {
742 conf.setInt(CONF_NUM_SPLITS, mappers);
743 conf.setInt(MR_NUM_MAPS, mappers);
744 }
745 conf.setInt(CONF_FILES_MODE, filesMode);
746 conf.setBoolean(CONF_CHECKSUM_VERIFY, verifyChecksum);
747 conf.set(CONF_OUTPUT_ROOT, outputRoot.toString());
748 conf.set(CONF_INPUT_ROOT, inputRoot.toString());
749 conf.setInt(CONF_BANDWIDTH_MB, bandwidthMB);
750 conf.set(CONF_SNAPSHOT_NAME, snapshotName);
751 conf.set(CONF_SNAPSHOT_DIR, snapshotDir.toString());
752
753 Job job = new Job(conf);
754 job.setJobName("ExportSnapshot-" + snapshotName);
755 job.setJarByClass(ExportSnapshot.class);
756 TableMapReduceUtil.addDependencyJars(job);
757 job.setMapperClass(ExportMapper.class);
758 job.setInputFormatClass(ExportSnapshotInputFormat.class);
759 job.setOutputFormatClass(NullOutputFormat.class);
760 job.setMapSpeculativeExecution(false);
761 job.setNumReduceTasks(0);
762
763
764 TokenCache.obtainTokensForNamenodes(job.getCredentials(),
765 new Path[] { inputRoot, outputRoot }, conf);
766
767
768 if (!job.waitForCompletion(true)) {
769
770
771 throw new ExportSnapshotException("Copy Files Map-Reduce Job failed");
772 }
773 }
774
775 private void verifySnapshot(final Configuration baseConf,
776 final FileSystem fs, final Path rootDir, final Path snapshotDir) throws IOException {
777
778 Configuration conf = new Configuration(baseConf);
779 FSUtils.setRootDir(conf, rootDir);
780 FSUtils.setFsDefault(conf, FSUtils.getRootDir(conf));
781 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
782 SnapshotReferenceUtil.verifySnapshot(conf, fs, snapshotDir, snapshotDesc);
783 }
784
785
786
787
788
789 @Override
790 public int run(String[] args) throws IOException {
791 boolean verifyTarget = true;
792 boolean verifyChecksum = true;
793 String snapshotName = null;
794 String targetName = null;
795 boolean overwrite = false;
796 String filesGroup = null;
797 String filesUser = null;
798 Path outputRoot = null;
799 int bandwidthMB = Integer.MAX_VALUE;
800 int filesMode = 0;
801 int mappers = 0;
802
803 Configuration conf = getConf();
804 Path inputRoot = FSUtils.getRootDir(conf);
805
806
807 for (int i = 0; i < args.length; i++) {
808 String cmd = args[i];
809 if (cmd.equals("-snapshot")) {
810 snapshotName = args[++i];
811 } else if (cmd.equals("-target")) {
812 targetName = args[++i];
813 } else if (cmd.equals("-copy-to")) {
814 outputRoot = new Path(args[++i]);
815 } else if (cmd.equals("-copy-from")) {
816 inputRoot = new Path(args[++i]);
817 FSUtils.setRootDir(conf, inputRoot);
818 } else if (cmd.equals("-no-checksum-verify")) {
819 verifyChecksum = false;
820 } else if (cmd.equals("-no-target-verify")) {
821 verifyTarget = false;
822 } else if (cmd.equals("-mappers")) {
823 mappers = Integer.parseInt(args[++i]);
824 } else if (cmd.equals("-chuser")) {
825 filesUser = args[++i];
826 } else if (cmd.equals("-chgroup")) {
827 filesGroup = args[++i];
828 } else if (cmd.equals("-bandwidth")) {
829 bandwidthMB = Integer.parseInt(args[++i]);
830 } else if (cmd.equals("-chmod")) {
831 filesMode = Integer.parseInt(args[++i], 8);
832 } else if (cmd.equals("-overwrite")) {
833 overwrite = true;
834 } else if (cmd.equals("-h") || cmd.equals("--help")) {
835 printUsageAndExit();
836 } else {
837 System.err.println("UNEXPECTED: " + cmd);
838 printUsageAndExit();
839 }
840 }
841
842
843 if (snapshotName == null) {
844 System.err.println("Snapshot name not provided.");
845 printUsageAndExit();
846 }
847
848 if (outputRoot == null) {
849 System.err.println("Destination file-system not provided.");
850 printUsageAndExit();
851 }
852
853 if (targetName == null) {
854 targetName = snapshotName;
855 }
856
857 FileSystem inputFs = FileSystem.get(inputRoot.toUri(), conf);
858 LOG.debug("inputFs=" + inputFs.getUri().toString() + " inputRoot=" + inputRoot);
859 FileSystem outputFs = FileSystem.get(outputRoot.toUri(), conf);
860 LOG.debug("outputFs=" + outputFs.getUri().toString() + " outputRoot=" + outputRoot.toString());
861
862 boolean skipTmp = conf.getBoolean(CONF_SKIP_TMP, false);
863
864 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, inputRoot);
865 Path snapshotTmpDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(targetName, outputRoot);
866 Path outputSnapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(targetName, outputRoot);
867 Path initialOutputSnapshotDir = skipTmp ? outputSnapshotDir : snapshotTmpDir;
868
869
870 if (outputFs.exists(outputSnapshotDir)) {
871 if (overwrite) {
872 if (!outputFs.delete(outputSnapshotDir, true)) {
873 System.err.println("Unable to remove existing snapshot directory: " + outputSnapshotDir);
874 return 1;
875 }
876 } else {
877 System.err.println("The snapshot '" + targetName +
878 "' already exists in the destination: " + outputSnapshotDir);
879 return 1;
880 }
881 }
882
883 if (!skipTmp) {
884
885 if (outputFs.exists(snapshotTmpDir)) {
886 if (overwrite) {
887 if (!outputFs.delete(snapshotTmpDir, true)) {
888 System.err.println("Unable to remove existing snapshot tmp directory: "+snapshotTmpDir);
889 return 1;
890 }
891 } else {
892 System.err.println("A snapshot with the same name '"+ targetName +"' may be in-progress");
893 System.err.println("Please check "+snapshotTmpDir+". If the snapshot has completed, ");
894 System.err.println("consider removing "+snapshotTmpDir+" by using the -overwrite option");
895 return 1;
896 }
897 }
898 }
899
900
901
902
903 try {
904 LOG.info("Copy Snapshot Manifest");
905 FileUtil.copy(inputFs, snapshotDir, outputFs, initialOutputSnapshotDir, false, false, conf);
906 } catch (IOException e) {
907 throw new ExportSnapshotException("Failed to copy the snapshot directory: from=" +
908 snapshotDir + " to=" + initialOutputSnapshotDir, e);
909 }
910
911
912 if (!targetName.equals(snapshotName)) {
913 SnapshotDescription snapshotDesc =
914 SnapshotDescriptionUtils.readSnapshotInfo(inputFs, snapshotDir)
915 .toBuilder()
916 .setName(targetName)
917 .build();
918 SnapshotDescriptionUtils.writeSnapshotInfo(snapshotDesc, snapshotTmpDir, outputFs);
919 }
920
921
922
923
924 try {
925 runCopyJob(inputRoot, outputRoot, snapshotName, snapshotDir, verifyChecksum,
926 filesUser, filesGroup, filesMode, mappers, bandwidthMB);
927
928 LOG.info("Finalize the Snapshot Export");
929 if (!skipTmp) {
930
931 if (!outputFs.rename(snapshotTmpDir, outputSnapshotDir)) {
932 throw new ExportSnapshotException("Unable to rename snapshot directory from=" +
933 snapshotTmpDir + " to=" + outputSnapshotDir);
934 }
935 }
936
937
938 if (verifyTarget) {
939 LOG.info("Verify snapshot integrity");
940 verifySnapshot(conf, outputFs, outputRoot, outputSnapshotDir);
941 }
942
943 LOG.info("Export Completed: " + targetName);
944 return 0;
945 } catch (Exception e) {
946 LOG.error("Snapshot export failed", e);
947 if (!skipTmp) {
948 outputFs.delete(snapshotTmpDir, true);
949 }
950 outputFs.delete(outputSnapshotDir, true);
951 return 1;
952 }
953 }
954
955
956 private void printUsageAndExit() {
957 System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
958 System.err.println(" where [options] are:");
959 System.err.println(" -h|-help Show this help and exit.");
960 System.err.println(" -snapshot NAME Snapshot to restore.");
961 System.err.println(" -copy-to NAME Remote destination hdfs://");
962 System.err.println(" -copy-from NAME Input folder hdfs:// (default hbase.rootdir)");
963 System.err.println(" -no-checksum-verify Do not verify checksum, use name+length only.");
964 System.err.println(" -no-target-verify Do not verify the integrity of the \\" +
965 "exported snapshot.");
966 System.err.println(" -overwrite Rewrite the snapshot manifest if already exists");
967 System.err.println(" -chuser USERNAME Change the owner of the files to the specified one.");
968 System.err.println(" -chgroup GROUP Change the group of the files to the specified one.");
969 System.err.println(" -chmod MODE Change the permission of the files to the specified one.");
970 System.err.println(" -mappers Number of mappers to use during the copy (mapreduce.job.maps).");
971 System.err.println();
972 System.err.println("Examples:");
973 System.err.println(" hbase " + getClass().getName() + " \\");
974 System.err.println(" -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase \\");
975 System.err.println(" -chuser MyUser -chgroup MyGroup -chmod 700 -mappers 16");
976 System.err.println();
977 System.err.println(" hbase " + getClass().getName() + " \\");
978 System.err.println(" -snapshot MySnapshot -copy-from hdfs://srv2:8082/hbase \\");
979 System.err.println(" -copy-to hdfs://srv1:50070/hbase \\");
980 System.exit(1);
981 }
982
983
984
985
986
987
988
989
990 static int innerMain(final Configuration conf, final String [] args) throws Exception {
991 return ToolRunner.run(conf, new ExportSnapshot(), args);
992 }
993
994 public static void main(String[] args) throws Exception {
995 System.exit(innerMain(HBaseConfiguration.create(), args));
996 }
997 }