1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.InputStream;
22 import java.io.IOException;
23 import java.io.OutputStream;
24 import java.util.Arrays;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.LinkedList;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Set;
31 import java.util.TreeMap;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.classification.InterfaceAudience;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.TableName;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.backup.HFileArchiver;
45 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
46 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
47 import org.apache.hadoop.hbase.io.HFileLink;
48 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
49 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
50 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
51 import org.apache.hadoop.hbase.regionserver.HRegion;
52 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
53 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
54 import org.apache.hadoop.hbase.util.Bytes;
55 import org.apache.hadoop.hbase.util.FSUtils;
56 import org.apache.hadoop.hbase.util.FSVisitor;
57 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
58 import org.apache.hadoop.io.IOUtils;
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101 @InterfaceAudience.Private
102 public class RestoreSnapshotHelper {
103 private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
104
105 private final Map<byte[], byte[]> regionsMap =
106 new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
107
108 private final ForeignExceptionDispatcher monitor;
109 private final MonitoredTask status;
110
111 private final SnapshotDescription snapshotDesc;
112 private final TableName snapshotTable;
113 private final Path snapshotDir;
114
115 private final HTableDescriptor tableDesc;
116 private final Path rootDir;
117 private final Path tableDir;
118
119 private final Configuration conf;
120 private final FileSystem fs;
121
122 public RestoreSnapshotHelper(final Configuration conf,
123 final FileSystem fs,
124 final SnapshotDescription snapshotDescription,
125 final Path snapshotDir,
126 final HTableDescriptor tableDescriptor,
127 final Path rootDir,
128 final ForeignExceptionDispatcher monitor,
129 final MonitoredTask status)
130 {
131 this.fs = fs;
132 this.conf = conf;
133 this.snapshotDesc = snapshotDescription;
134 this.snapshotTable = TableName.valueOf(snapshotDescription.getTable());
135 this.snapshotDir = snapshotDir;
136 this.tableDesc = tableDescriptor;
137 this.rootDir = rootDir;
138 this.tableDir = FSUtils.getTableDir(rootDir, tableDesc.getTableName());
139 this.monitor = monitor;
140 this.status = status;
141 }
142
143
144
145
146
147 public RestoreMetaChanges restoreHdfsRegions() throws IOException {
148 LOG.debug("starting restore");
149 Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
150 if (snapshotRegionNames == null) {
151 LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
152 return null;
153 }
154
155 RestoreMetaChanges metaChanges = new RestoreMetaChanges();
156
157
158
159 List<HRegionInfo> tableRegions = getTableRegions();
160 if (tableRegions != null) {
161 monitor.rethrowException();
162 for (HRegionInfo regionInfo: tableRegions) {
163 String regionName = regionInfo.getEncodedName();
164 if (snapshotRegionNames.contains(regionName)) {
165 LOG.info("region to restore: " + regionName);
166 snapshotRegionNames.remove(regionName);
167 metaChanges.addRegionToRestore(regionInfo);
168 } else {
169 LOG.info("region to remove: " + regionName);
170 metaChanges.addRegionToRemove(regionInfo);
171 }
172 }
173
174
175 monitor.rethrowException();
176 status.setStatus("Restoring table regions...");
177 restoreHdfsRegions(metaChanges.getRegionsToRestore());
178 status.setStatus("Finished restoring all table regions.");
179
180
181 monitor.rethrowException();
182 status.setStatus("Starting to delete excess regions from table");
183 removeHdfsRegions(metaChanges.getRegionsToRemove());
184 status.setStatus("Finished deleting excess regions from table.");
185 }
186
187
188 if (snapshotRegionNames.size() > 0) {
189 List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();
190
191 monitor.rethrowException();
192 for (String regionName: snapshotRegionNames) {
193 LOG.info("region to add: " + regionName);
194 Path regionDir = new Path(snapshotDir, regionName);
195 regionsToAdd.add(HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir));
196 }
197
198
199 monitor.rethrowException();
200 status.setStatus("Cloning regions...");
201 HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
202 metaChanges.setNewRegions(clonedRegions);
203 status.setStatus("Finished cloning regions.");
204 }
205
206
207 monitor.rethrowException();
208 status.setStatus("Restoring WALs to table...");
209 restoreWALs();
210 status.setStatus("Finished restoring WALs to table.");
211
212 return metaChanges;
213 }
214
215
216
217
218 public static class RestoreMetaChanges {
219 private List<HRegionInfo> regionsToRestore = null;
220 private List<HRegionInfo> regionsToRemove = null;
221 private List<HRegionInfo> regionsToAdd = null;
222
223
224
225
226 public boolean hasRegionsToAdd() {
227 return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
228 }
229
230
231
232
233
234
235
236 public List<HRegionInfo> getRegionsToAdd() {
237 return this.regionsToAdd;
238 }
239
240
241
242
243 public boolean hasRegionsToRestore() {
244 return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
245 }
246
247
248
249
250
251
252 public List<HRegionInfo> getRegionsToRestore() {
253 return this.regionsToRestore;
254 }
255
256
257
258
259 public boolean hasRegionsToRemove() {
260 return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
261 }
262
263
264
265
266
267
268
269 public List<HRegionInfo> getRegionsToRemove() {
270 return this.regionsToRemove;
271 }
272
273 void setNewRegions(final HRegionInfo[] hris) {
274 if (hris != null) {
275 regionsToAdd = Arrays.asList(hris);
276 } else {
277 regionsToAdd = null;
278 }
279 }
280
281 void addRegionToRemove(final HRegionInfo hri) {
282 if (regionsToRemove == null) {
283 regionsToRemove = new LinkedList<HRegionInfo>();
284 }
285 regionsToRemove.add(hri);
286 }
287
288 void addRegionToRestore(final HRegionInfo hri) {
289 if (regionsToRestore == null) {
290 regionsToRestore = new LinkedList<HRegionInfo>();
291 }
292 regionsToRestore.add(hri);
293 }
294 }
295
296
297
298
299 private void removeHdfsRegions(final List<HRegionInfo> regions) throws IOException {
300 if (regions != null && regions.size() > 0) {
301 for (HRegionInfo hri: regions) {
302 HFileArchiver.archiveRegion(conf, fs, hri);
303 }
304 }
305 }
306
307
308
309
310 private void restoreHdfsRegions(final List<HRegionInfo> regions) throws IOException {
311 if (regions == null || regions.size() == 0) return;
312 for (HRegionInfo hri: regions) restoreRegion(hri);
313 }
314
315
316
317
318
319 private void restoreRegion(HRegionInfo regionInfo) throws IOException {
320 Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName());
321 Map<String, List<String>> snapshotFiles =
322 SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir);
323 Path regionDir = new Path(tableDir, regionInfo.getEncodedName());
324 String tableName = tableDesc.getTableName().getNameAsString();
325
326
327 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
328 byte[] family = Bytes.toBytes(familyDir.getName());
329 Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
330 List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
331 if (snapshotFamilyFiles != null) {
332 List<String> hfilesToAdd = new LinkedList<String>();
333 for (String hfileName: snapshotFamilyFiles) {
334 if (familyFiles.contains(hfileName)) {
335
336 familyFiles.remove(hfileName);
337 } else {
338
339 hfilesToAdd.add(hfileName);
340 }
341 }
342
343
344 for (String hfileName: hfilesToAdd) {
345 LOG.trace("Adding HFileLink " + hfileName +
346 " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
347 restoreStoreFile(familyDir, regionInfo, hfileName);
348 }
349
350
351 for (String hfileName: familyFiles) {
352 Path hfile = new Path(familyDir, hfileName);
353 LOG.trace("Removing hfile=" + hfile +
354 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
355 HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
356 }
357 } else {
358
359 LOG.trace("Removing family=" + Bytes.toString(family) +
360 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
361 HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family);
362 fs.delete(familyDir, true);
363 }
364 }
365
366
367 for (Map.Entry<String, List<String>> familyEntry: snapshotFiles.entrySet()) {
368 Path familyDir = new Path(regionDir, familyEntry.getKey());
369 if (!fs.mkdirs(familyDir)) {
370 throw new IOException("Unable to create familyDir=" + familyDir);
371 }
372
373 for (String hfileName: familyEntry.getValue()) {
374 LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName);
375 restoreStoreFile(familyDir, regionInfo, hfileName);
376 }
377 }
378 }
379
380
381
382
383 private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
384 Set<String> familyFiles = new HashSet<String>();
385
386 FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
387 if (hfiles == null) return familyFiles;
388
389 for (FileStatus hfileRef: hfiles) {
390 String hfileName = hfileRef.getPath().getName();
391 familyFiles.add(hfileName);
392 }
393
394 return familyFiles;
395 }
396
397
398
399
400
401 private HRegionInfo[] cloneHdfsRegions(final List<HRegionInfo> regions) throws IOException {
402 if (regions == null || regions.size() == 0) return null;
403
404 final Map<String, HRegionInfo> snapshotRegions =
405 new HashMap<String, HRegionInfo>(regions.size());
406
407
408 HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
409 for (int i = 0; i < clonedRegionsInfo.length; ++i) {
410
411 HRegionInfo snapshotRegionInfo = regions.get(i);
412 clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
413
414
415 String snapshotRegionName = snapshotRegionInfo.getEncodedName();
416 String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
417 regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
418 LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
419
420
421 snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
422 }
423
424
425 ModifyRegionUtils.createRegions(conf, rootDir,
426 tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
427 public void fillRegion(final HRegion region) throws IOException {
428 cloneRegion(region, snapshotRegions.get(region.getRegionInfo().getEncodedName()));
429 }
430 });
431
432 return clonedRegionsInfo;
433 }
434
435
436
437
438
439
440
441
442
443
444
445
446 private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
447 throws IOException {
448 final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
449 final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
450 final String tableName = tableDesc.getTableName().getNameAsString();
451 SnapshotReferenceUtil.visitRegionStoreFiles(fs, snapshotRegionDir,
452 new FSVisitor.StoreFileVisitor() {
453 public void storeFile (final String region, final String family, final String hfile)
454 throws IOException {
455 LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
456 Path familyDir = new Path(regionDir, family);
457 restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
458 }
459 });
460 }
461
462
463
464
465
466
467
468
469
470
471
472
473
474 private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
475 final String hfileName) throws IOException {
476 if (HFileLink.isHFileLink(hfileName)) {
477 HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
478 } else if (StoreFileInfo.isReference(hfileName)) {
479 restoreReferenceFile(familyDir, regionInfo, hfileName);
480 } else {
481 HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
482 }
483 }
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503 private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
504 final String hfileName) throws IOException {
505
506 Path refPath = StoreFileInfo.getReferredToFile(new Path(new Path(new Path(
507 snapshotTable.getNameAsString(), regionInfo.getEncodedName()), familyDir.getName()),
508 hfileName));
509 String snapshotRegionName = refPath.getParent().getParent().getName();
510 String fileName = refPath.getName();
511
512
513 String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
514 if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
515
516
517 String refLink = fileName;
518 if (!HFileLink.isHFileLink(fileName)) {
519 refLink = HFileLink.createHFileLinkName(snapshotTable, snapshotRegionName, fileName);
520 }
521 Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
522
523
524 Path linkPath = new Path(familyDir,
525 HFileLink.createHFileLinkName(snapshotTable, regionInfo.getEncodedName(), hfileName));
526 InputStream in = new HFileLink(conf, linkPath).open(fs);
527 OutputStream out = fs.create(outPath);
528 IOUtils.copyBytes(in, out, conf);
529 }
530
531
532
533
534
535
536
537
538
539 public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
540 return new HRegionInfo(tableDesc.getTableName(),
541 snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
542 snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
543 }
544
545
546
547
548
549
550
551
552
553
554 private void restoreWALs() throws IOException {
555 final SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
556 snapshotTable, regionsMap);
557 try {
558
559 SnapshotReferenceUtil.visitRecoveredEdits(fs, snapshotDir,
560 new FSVisitor.RecoveredEditsVisitor() {
561 public void recoveredEdits (final String region, final String logfile) throws IOException {
562 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
563 logSplitter.splitRecoveredEdit(path);
564 }
565 });
566
567
568 SnapshotReferenceUtil.visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
569 public void logFile (final String server, final String logfile) throws IOException {
570 logSplitter.splitLog(server, logfile);
571 }
572 });
573 } finally {
574 logSplitter.close();
575 }
576 }
577
578
579
580
581 private List<HRegionInfo> getTableRegions() throws IOException {
582 LOG.debug("get table regions: " + tableDir);
583 FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
584 if (regionDirs == null) return null;
585
586 List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
587 for (FileStatus regionDir: regionDirs) {
588 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
589 regions.add(hri);
590 }
591 LOG.debug("found " + regions.size() + " regions for table=" +
592 tableDesc.getTableName().getNameAsString());
593 return regions;
594 }
595
596
597
598
599
600
601
602
603
604 public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor,
605 final TableName tableName) throws IOException {
606 HTableDescriptor htd = new HTableDescriptor(tableName);
607 for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
608 htd.addFamily(hcd);
609 }
610 for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
611 snapshotTableDescriptor.getValues().entrySet()) {
612 htd.setValue(e.getKey(), e.getValue());
613 }
614 for (Map.Entry<String, String> e: snapshotTableDescriptor.getConfiguration().entrySet()) {
615 htd.setConfiguration(e.getKey(), e.getValue());
616 }
617 return htd;
618 }
619 }