1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.InputStream;
22 import java.io.IOException;
23 import java.io.OutputStream;
24 import java.util.Arrays;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.LinkedList;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Set;
31 import java.util.TreeMap;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.classification.InterfaceAudience;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.TableName;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.backup.HFileArchiver;
45 import org.apache.hadoop.hbase.catalog.CatalogTracker;
46 import org.apache.hadoop.hbase.catalog.MetaEditor;
47 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
48 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
49 import org.apache.hadoop.hbase.io.HFileLink;
50 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
51 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
52 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
53 import org.apache.hadoop.hbase.regionserver.HRegion;
54 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
55 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.FSUtils;
58 import org.apache.hadoop.hbase.util.FSVisitor;
59 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
60 import org.apache.hadoop.hbase.util.Pair;
61 import org.apache.hadoop.io.IOUtils;
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104 @InterfaceAudience.Private
105 public class RestoreSnapshotHelper {
106 private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
107
108 private final Map<byte[], byte[]> regionsMap =
109 new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
110
111 private final Map<String, Pair<String, String> > parentsMap =
112 new HashMap<String, Pair<String, String> >();
113
114 private final ForeignExceptionDispatcher monitor;
115 private final MonitoredTask status;
116
117 private final SnapshotDescription snapshotDesc;
118 private final TableName snapshotTable;
119 private final Path snapshotDir;
120
121 private final HTableDescriptor tableDesc;
122 private final Path rootDir;
123 private final Path tableDir;
124
125 private final Configuration conf;
126 private final FileSystem fs;
127
128 public RestoreSnapshotHelper(final Configuration conf,
129 final FileSystem fs,
130 final SnapshotDescription snapshotDescription,
131 final Path snapshotDir,
132 final HTableDescriptor tableDescriptor,
133 final Path rootDir,
134 final ForeignExceptionDispatcher monitor,
135 final MonitoredTask status)
136 {
137 this.fs = fs;
138 this.conf = conf;
139 this.snapshotDesc = snapshotDescription;
140 this.snapshotTable = TableName.valueOf(snapshotDescription.getTable());
141 this.snapshotDir = snapshotDir;
142 this.tableDesc = tableDescriptor;
143 this.rootDir = rootDir;
144 this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
145 this.monitor = monitor;
146 this.status = status;
147 }
148
149
150
151
152
153 public RestoreMetaChanges restoreHdfsRegions() throws IOException {
154 LOG.debug("starting restore");
155 Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
156 if (snapshotRegionNames == null) {
157 LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
158 return null;
159 }
160
161 RestoreMetaChanges metaChanges = new RestoreMetaChanges(parentsMap);
162
163
164
165 List<HRegionInfo> tableRegions = getTableRegions();
166 if (tableRegions != null) {
167 monitor.rethrowException();
168 for (HRegionInfo regionInfo: tableRegions) {
169 String regionName = regionInfo.getEncodedName();
170 if (snapshotRegionNames.contains(regionName)) {
171 LOG.info("region to restore: " + regionName);
172 snapshotRegionNames.remove(regionName);
173 metaChanges.addRegionToRestore(regionInfo);
174 } else {
175 LOG.info("region to remove: " + regionName);
176 metaChanges.addRegionToRemove(regionInfo);
177 }
178 }
179
180
181 monitor.rethrowException();
182 status.setStatus("Restoring table regions...");
183 restoreHdfsRegions(metaChanges.getRegionsToRestore());
184 status.setStatus("Finished restoring all table regions.");
185
186
187 monitor.rethrowException();
188 status.setStatus("Starting to delete excess regions from table");
189 removeHdfsRegions(metaChanges.getRegionsToRemove());
190 status.setStatus("Finished deleting excess regions from table.");
191 }
192
193
194 if (snapshotRegionNames.size() > 0) {
195 List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();
196
197 monitor.rethrowException();
198 for (String regionName: snapshotRegionNames) {
199 LOG.info("region to add: " + regionName);
200 Path regionDir = new Path(snapshotDir, regionName);
201 regionsToAdd.add(HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir));
202 }
203
204
205 monitor.rethrowException();
206 status.setStatus("Cloning regions...");
207 HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
208 metaChanges.setNewRegions(clonedRegions);
209 status.setStatus("Finished cloning regions.");
210 }
211
212
213 monitor.rethrowException();
214 status.setStatus("Restoring WALs to table...");
215 restoreWALs();
216 status.setStatus("Finished restoring WALs to table.");
217
218 return metaChanges;
219 }
220
221
222
223
224 public static class RestoreMetaChanges {
225 private final Map<String, Pair<String, String> > parentsMap;
226
227 private List<HRegionInfo> regionsToRestore = null;
228 private List<HRegionInfo> regionsToRemove = null;
229 private List<HRegionInfo> regionsToAdd = null;
230
231 RestoreMetaChanges(final Map<String, Pair<String, String> > parentsMap) {
232 this.parentsMap = parentsMap;
233 }
234
235
236
237
238 public boolean hasRegionsToAdd() {
239 return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
240 }
241
242
243
244
245
246
247
248 public List<HRegionInfo> getRegionsToAdd() {
249 return this.regionsToAdd;
250 }
251
252
253
254
255 public boolean hasRegionsToRestore() {
256 return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
257 }
258
259
260
261
262
263
264 public List<HRegionInfo> getRegionsToRestore() {
265 return this.regionsToRestore;
266 }
267
268
269
270
271 public boolean hasRegionsToRemove() {
272 return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
273 }
274
275
276
277
278
279
280
281 public List<HRegionInfo> getRegionsToRemove() {
282 return this.regionsToRemove;
283 }
284
285 void setNewRegions(final HRegionInfo[] hris) {
286 if (hris != null) {
287 regionsToAdd = Arrays.asList(hris);
288 } else {
289 regionsToAdd = null;
290 }
291 }
292
293 void addRegionToRemove(final HRegionInfo hri) {
294 if (regionsToRemove == null) {
295 regionsToRemove = new LinkedList<HRegionInfo>();
296 }
297 regionsToRemove.add(hri);
298 }
299
300 void addRegionToRestore(final HRegionInfo hri) {
301 if (regionsToRestore == null) {
302 regionsToRestore = new LinkedList<HRegionInfo>();
303 }
304 regionsToRestore.add(hri);
305 }
306
307 public void updateMetaParentRegions(final CatalogTracker catalogTracker,
308 final List<HRegionInfo> regionInfos) throws IOException {
309 if (regionInfos == null || parentsMap.isEmpty()) return;
310
311
312 Map<String, HRegionInfo> regionsByName = new HashMap<String, HRegionInfo>(regionInfos.size());
313 List<HRegionInfo> parentRegions = new LinkedList();
314 for (HRegionInfo regionInfo: regionInfos) {
315 if (regionInfo.isSplitParent()) {
316 parentRegions.add(regionInfo);
317 } else {
318 regionsByName.put(regionInfo.getEncodedName(), regionInfo);
319 }
320 }
321
322
323 for (HRegionInfo regionInfo: parentRegions) {
324 Pair<String, String> daughters = parentsMap.get(regionInfo.getEncodedName());
325 if (daughters == null) {
326
327
328 LOG.warn("Skip update of unreferenced offline parent: " + regionInfo);
329 continue;
330 }
331
332
333 if (daughters.getSecond() == null) {
334 daughters.setSecond(daughters.getFirst());
335 }
336
337 LOG.debug("Update splits parent " + regionInfo.getEncodedName() + " -> " + daughters);
338 MetaEditor.addRegionToMeta(catalogTracker, regionInfo,
339 regionsByName.get(daughters.getFirst()),
340 regionsByName.get(daughters.getSecond()));
341 }
342 }
343 }
344
345
346
347
348 private void removeHdfsRegions(final List<HRegionInfo> regions) throws IOException {
349 if (regions != null && regions.size() > 0) {
350 for (HRegionInfo hri: regions) {
351 HFileArchiver.archiveRegion(conf, fs, hri);
352 }
353 }
354 }
355
356
357
358
359 private void restoreHdfsRegions(final List<HRegionInfo> regions) throws IOException {
360 if (regions == null || regions.size() == 0) return;
361 for (HRegionInfo hri: regions) restoreRegion(hri);
362 }
363
364
365
366
367
368 private void restoreRegion(HRegionInfo regionInfo) throws IOException {
369 Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName());
370 Map<String, List<String>> snapshotFiles =
371 SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir);
372 Path regionDir = new Path(tableDir, regionInfo.getEncodedName());
373 String tableName = tableDesc.getTableName().getNameAsString();
374
375
376 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
377 byte[] family = Bytes.toBytes(familyDir.getName());
378 Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
379 List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
380 if (snapshotFamilyFiles != null) {
381 List<String> hfilesToAdd = new LinkedList<String>();
382 for (String hfileName: snapshotFamilyFiles) {
383 if (familyFiles.contains(hfileName)) {
384
385 familyFiles.remove(hfileName);
386 } else {
387
388 hfilesToAdd.add(hfileName);
389 }
390 }
391
392
393 for (String hfileName: familyFiles) {
394 Path hfile = new Path(familyDir, hfileName);
395 LOG.trace("Removing hfile=" + hfile +
396 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
397 HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
398 }
399
400
401 for (String hfileName: hfilesToAdd) {
402 LOG.trace("Adding HFileLink " + hfileName +
403 " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
404 restoreStoreFile(familyDir, regionInfo, hfileName);
405 }
406 } else {
407
408 LOG.trace("Removing family=" + Bytes.toString(family) +
409 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
410 HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family);
411 fs.delete(familyDir, true);
412 }
413 }
414
415
416 for (Map.Entry<String, List<String>> familyEntry: snapshotFiles.entrySet()) {
417 Path familyDir = new Path(regionDir, familyEntry.getKey());
418 if (!fs.mkdirs(familyDir)) {
419 throw new IOException("Unable to create familyDir=" + familyDir);
420 }
421
422 for (String hfileName: familyEntry.getValue()) {
423 LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName);
424 restoreStoreFile(familyDir, regionInfo, hfileName);
425 }
426 }
427 }
428
429
430
431
432 private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
433 Set<String> familyFiles = new HashSet<String>();
434
435 FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
436 if (hfiles == null) return familyFiles;
437
438 for (FileStatus hfileRef: hfiles) {
439 String hfileName = hfileRef.getPath().getName();
440 familyFiles.add(hfileName);
441 }
442
443 return familyFiles;
444 }
445
446
447
448
449
450 private HRegionInfo[] cloneHdfsRegions(final List<HRegionInfo> regions) throws IOException {
451 if (regions == null || regions.size() == 0) return null;
452
453 final Map<String, HRegionInfo> snapshotRegions =
454 new HashMap<String, HRegionInfo>(regions.size());
455
456
457 HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
458 for (int i = 0; i < clonedRegionsInfo.length; ++i) {
459
460 HRegionInfo snapshotRegionInfo = regions.get(i);
461 clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
462
463
464 String snapshotRegionName = snapshotRegionInfo.getEncodedName();
465 String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
466 regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
467 LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
468
469
470 snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
471 }
472
473
474 ModifyRegionUtils.createRegions(conf, rootDir,
475 tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
476 public void fillRegion(final HRegion region) throws IOException {
477 cloneRegion(region, snapshotRegions.get(region.getRegionInfo().getEncodedName()));
478 }
479 });
480
481 return clonedRegionsInfo;
482 }
483
484
485
486
487
488
489
490
491
492
493
494
495 private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
496 throws IOException {
497 final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
498 final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
499 final String tableName = tableDesc.getTableName().getNameAsString();
500 SnapshotReferenceUtil.visitRegionStoreFiles(fs, snapshotRegionDir,
501 new FSVisitor.StoreFileVisitor() {
502 public void storeFile (final String region, final String family, final String hfile)
503 throws IOException {
504 LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
505 Path familyDir = new Path(regionDir, family);
506 restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
507 }
508 });
509 }
510
511
512
513
514
515
516
517
518
519
520
521
522
523 private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
524 final String hfileName) throws IOException {
525 if (HFileLink.isHFileLink(hfileName)) {
526 HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
527 } else if (StoreFileInfo.isReference(hfileName)) {
528 restoreReferenceFile(familyDir, regionInfo, hfileName);
529 } else {
530 HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
531 }
532 }
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552 private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
553 final String hfileName) throws IOException {
554
555 Path refPath = StoreFileInfo.getReferredToFile(new Path(new Path(new Path(
556 snapshotTable.getNameAsString(), regionInfo.getEncodedName()), familyDir.getName()),
557 hfileName));
558 String snapshotRegionName = refPath.getParent().getParent().getName();
559 String fileName = refPath.getName();
560
561
562 String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
563 if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
564
565
566 Path linkPath = null;
567 String refLink = fileName;
568 if (!HFileLink.isHFileLink(fileName)) {
569 refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
570 linkPath = new Path(familyDir,
571 HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
572 }
573
574 Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
575
576
577 InputStream in;
578 if (linkPath != null) {
579 in = new HFileLink(conf, linkPath).open(fs);
580 } else {
581 linkPath = new Path(new Path(HRegion.getRegionDir(snapshotDir, regionInfo.getEncodedName()),
582 familyDir.getName()), hfileName);
583 in = fs.open(linkPath);
584 }
585 OutputStream out = fs.create(outPath);
586 IOUtils.copyBytes(in, out, conf);
587
588
589 String regionName = Bytes.toString(regionsMap.get(regionInfo.getEncodedNameAsBytes()));
590 LOG.debug("Restore reference " + regionName + " to " + clonedRegionName);
591 synchronized (parentsMap) {
592 Pair<String, String> daughters = parentsMap.get(clonedRegionName);
593 if (daughters == null) {
594 daughters = new Pair<String, String>(regionName, null);
595 parentsMap.put(clonedRegionName, daughters);
596 } else if (!regionName.equals(daughters.getFirst())) {
597 daughters.setSecond(regionName);
598 }
599 }
600 }
601
602
603
604
605
606
607
608
609
610 public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
611 HRegionInfo regionInfo = new HRegionInfo(tableDesc.getTableName(),
612 snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
613 snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
614 regionInfo.setOffline(snapshotRegionInfo.isOffline());
615 return regionInfo;
616 }
617
618
619
620
621
622
623
624
625
626
627 private void restoreWALs() throws IOException {
628 final SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
629 snapshotTable, regionsMap);
630 try {
631
632 SnapshotReferenceUtil.visitRecoveredEdits(fs, snapshotDir,
633 new FSVisitor.RecoveredEditsVisitor() {
634 public void recoveredEdits (final String region, final String logfile) throws IOException {
635 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
636 logSplitter.splitRecoveredEdit(path);
637 }
638 });
639
640
641 SnapshotReferenceUtil.visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
642 public void logFile (final String server, final String logfile) throws IOException {
643 logSplitter.splitLog(server, logfile);
644 }
645 });
646 } finally {
647 logSplitter.close();
648 }
649 }
650
651
652
653
654 private List<HRegionInfo> getTableRegions() throws IOException {
655 LOG.debug("get table regions: " + tableDir);
656 FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
657 if (regionDirs == null) return null;
658
659 List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
660 for (FileStatus regionDir: regionDirs) {
661 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
662 regions.add(hri);
663 }
664 LOG.debug("found " + regions.size() + " regions for table=" +
665 tableDesc.getTableName().getNameAsString());
666 return regions;
667 }
668
669
670
671
672
673
674
675
676
677 public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor,
678 final TableName tableName) throws IOException {
679 HTableDescriptor htd = new HTableDescriptor(tableName);
680 for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
681 htd.addFamily(hcd);
682 }
683 for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
684 snapshotTableDescriptor.getValues().entrySet()) {
685 htd.setValue(e.getKey(), e.getValue());
686 }
687 for (Map.Entry<String, String> e: snapshotTableDescriptor.getConfiguration().entrySet()) {
688 htd.setConfiguration(e.getKey(), e.getValue());
689 }
690 return htd;
691 }
692 }