1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.InputStream;
22 import java.io.IOException;
23 import java.io.OutputStream;
24 import java.util.Arrays;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.LinkedList;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Set;
31 import java.util.TreeMap;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.classification.InterfaceAudience;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.backup.HFileArchiver;
44 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
45 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
46 import org.apache.hadoop.hbase.io.HFileLink;
47 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
48 import org.apache.hadoop.hbase.regionserver.HRegion;
49 import org.apache.hadoop.hbase.regionserver.StoreFile;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.FSUtils;
52 import org.apache.hadoop.hbase.util.FSVisitor;
53 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
54 import org.apache.hadoop.io.IOUtils;
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97 @InterfaceAudience.Private
98 public class RestoreSnapshotHelper {
99 private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
100
101 private final Map<byte[], byte[]> regionsMap =
102 new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
103
104 private final ForeignExceptionDispatcher monitor;
105 private final MonitoredTask status;
106
107 private final SnapshotDescription snapshotDesc;
108 private final Path snapshotDir;
109
110 private final HTableDescriptor tableDesc;
111 private final Path tableDir;
112
113 private final Configuration conf;
114 private final FileSystem fs;
115
116 public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs,
117 final SnapshotDescription snapshotDescription, final Path snapshotDir,
118 final HTableDescriptor tableDescriptor, final Path tableDir,
119 final ForeignExceptionDispatcher monitor, final MonitoredTask status)
120 {
121 this.fs = fs;
122 this.conf = conf;
123 this.snapshotDesc = snapshotDescription;
124 this.snapshotDir = snapshotDir;
125 this.tableDesc = tableDescriptor;
126 this.tableDir = tableDir;
127 this.monitor = monitor;
128 this.status = status;
129 }
130
131
132
133
134
135 public RestoreMetaChanges restoreHdfsRegions() throws IOException {
136 LOG.debug("starting restore");
137 Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
138 if (snapshotRegionNames == null) {
139 LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
140 return null;
141 }
142
143 RestoreMetaChanges metaChanges = new RestoreMetaChanges();
144
145
146
147 List<HRegionInfo> tableRegions = getTableRegions();
148 if (tableRegions != null) {
149 monitor.rethrowException();
150 for (HRegionInfo regionInfo: tableRegions) {
151 String regionName = regionInfo.getEncodedName();
152 if (snapshotRegionNames.contains(regionName)) {
153 LOG.info("region to restore: " + regionName);
154 snapshotRegionNames.remove(regionName);
155 metaChanges.addRegionToRestore(regionInfo);
156 } else {
157 LOG.info("region to remove: " + regionName);
158 metaChanges.addRegionToRemove(regionInfo);
159 }
160 }
161
162
163 monitor.rethrowException();
164 status.setStatus("Restoring table regions...");
165 restoreHdfsRegions(metaChanges.getRegionsToRestore());
166 status.setStatus("Finished restoring all table regions.");
167
168
169 monitor.rethrowException();
170 status.setStatus("Starting to delete excess regions from table");
171 removeHdfsRegions(metaChanges.getRegionsToRemove());
172 status.setStatus("Finished deleting excess regions from table.");
173 }
174
175
176 if (snapshotRegionNames.size() > 0) {
177 List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();
178
179 monitor.rethrowException();
180 for (String regionName: snapshotRegionNames) {
181 LOG.info("region to add: " + regionName);
182 Path regionDir = new Path(snapshotDir, regionName);
183 regionsToAdd.add(HRegion.loadDotRegionInfoFileContent(fs, regionDir));
184 }
185
186
187 monitor.rethrowException();
188 status.setStatus("Cloning regions...");
189 HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
190 metaChanges.setNewRegions(clonedRegions);
191 status.setStatus("Finished cloning regions.");
192 }
193
194
195 monitor.rethrowException();
196 status.setStatus("Restoring WALs to table...");
197 restoreWALs();
198 status.setStatus("Finished restoring WALs to table.");
199
200 return metaChanges;
201 }
202
203
204
205
206 public static class RestoreMetaChanges {
207 private List<HRegionInfo> regionsToRestore = null;
208 private List<HRegionInfo> regionsToRemove = null;
209 private List<HRegionInfo> regionsToAdd = null;
210
211
212
213
214 public boolean hasRegionsToAdd() {
215 return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
216 }
217
218
219
220
221
222
223
224 public List<HRegionInfo> getRegionsToAdd() {
225 return this.regionsToAdd;
226 }
227
228
229
230
231 public boolean hasRegionsToRestore() {
232 return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
233 }
234
235
236
237
238
239
240 public List<HRegionInfo> getRegionsToRestore() {
241 return this.regionsToRestore;
242 }
243
244
245
246
247 public boolean hasRegionsToRemove() {
248 return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
249 }
250
251
252
253
254
255
256
257 public List<HRegionInfo> getRegionsToRemove() {
258 return this.regionsToRemove;
259 }
260
261 void setNewRegions(final HRegionInfo[] hris) {
262 if (hris != null) {
263 regionsToAdd = Arrays.asList(hris);
264 } else {
265 regionsToAdd = null;
266 }
267 }
268
269 void addRegionToRemove(final HRegionInfo hri) {
270 if (regionsToRemove == null) {
271 regionsToRemove = new LinkedList<HRegionInfo>();
272 }
273 regionsToRemove.add(hri);
274 }
275
276 void addRegionToRestore(final HRegionInfo hri) {
277 if (regionsToRestore == null) {
278 regionsToRestore = new LinkedList<HRegionInfo>();
279 }
280 regionsToRestore.add(hri);
281 }
282 }
283
284
285
286
287 private void removeHdfsRegions(final List<HRegionInfo> regions) throws IOException {
288 if (regions != null && regions.size() > 0) {
289 for (HRegionInfo hri: regions) {
290 HFileArchiver.archiveRegion(conf, fs, hri);
291 }
292 }
293 }
294
295
296
297
298 private void restoreHdfsRegions(final List<HRegionInfo> regions) throws IOException {
299 if (regions == null || regions.size() == 0) return;
300 for (HRegionInfo hri: regions) restoreRegion(hri);
301 }
302
303
304
305
306
307 private void restoreRegion(HRegionInfo regionInfo) throws IOException {
308 Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName());
309 Map<String, List<String>> snapshotFiles =
310 SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir);
311 Path regionDir = new Path(tableDir, regionInfo.getEncodedName());
312 String tableName = tableDesc.getNameAsString();
313
314
315 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
316 byte[] family = Bytes.toBytes(familyDir.getName());
317 Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
318 List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
319 if (snapshotFamilyFiles != null) {
320 List<String> hfilesToAdd = new LinkedList<String>();
321 for (String hfileName: snapshotFamilyFiles) {
322 if (familyFiles.contains(hfileName)) {
323
324 familyFiles.remove(hfileName);
325 } else {
326
327 hfilesToAdd.add(hfileName);
328 }
329 }
330
331
332 for (String hfileName: hfilesToAdd) {
333 LOG.trace("Adding HFileLink " + hfileName +
334 " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
335 restoreStoreFile(familyDir, regionInfo, hfileName);
336 }
337
338
339 for (String hfileName: familyFiles) {
340 Path hfile = new Path(familyDir, hfileName);
341 LOG.trace("Removing hfile=" + hfile +
342 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
343 HFileArchiver.archiveStoreFile(fs, regionInfo, conf, tableDir, family, hfile);
344 }
345 } else {
346
347 LOG.trace("Removing family=" + Bytes.toString(family) +
348 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
349 HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family);
350 fs.delete(familyDir, true);
351 }
352 }
353
354
355 for (Map.Entry<String, List<String>> familyEntry: snapshotFiles.entrySet()) {
356 Path familyDir = new Path(regionDir, familyEntry.getKey());
357 if (!fs.mkdirs(familyDir)) {
358 throw new IOException("Unable to create familyDir=" + familyDir);
359 }
360
361 for (String hfileName: familyEntry.getValue()) {
362 LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName);
363 restoreStoreFile(familyDir, regionInfo, hfileName);
364 }
365 }
366 }
367
368
369
370
371 private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
372 Set<String> familyFiles = new HashSet<String>();
373
374 FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
375 if (hfiles == null) return familyFiles;
376
377 for (FileStatus hfileRef: hfiles) {
378 String hfileName = hfileRef.getPath().getName();
379 familyFiles.add(hfileName);
380 }
381
382 return familyFiles;
383 }
384
385
386
387
388
389 private HRegionInfo[] cloneHdfsRegions(final List<HRegionInfo> regions) throws IOException {
390 if (regions == null || regions.size() == 0) return null;
391
392 final Map<String, HRegionInfo> snapshotRegions =
393 new HashMap<String, HRegionInfo>(regions.size());
394
395
396 HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
397 for (int i = 0; i < clonedRegionsInfo.length; ++i) {
398
399 HRegionInfo snapshotRegionInfo = regions.get(i);
400 clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
401
402
403 String snapshotRegionName = snapshotRegionInfo.getEncodedName();
404 String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
405 regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
406 LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
407
408
409 snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
410 }
411
412
413 ModifyRegionUtils.createRegions(conf, tableDir.getParent(),
414 tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
415 public void fillRegion(final HRegion region) throws IOException {
416 cloneRegion(region, snapshotRegions.get(region.getRegionInfo().getEncodedName()));
417 }
418 });
419
420 return clonedRegionsInfo;
421 }
422
423
424
425
426
427
428
429
430
431
432
433
434 private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
435 throws IOException {
436 final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
437 final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
438 final String tableName = tableDesc.getNameAsString();
439 SnapshotReferenceUtil.visitRegionStoreFiles(fs, snapshotRegionDir,
440 new FSVisitor.StoreFileVisitor() {
441 public void storeFile (final String region, final String family, final String hfile)
442 throws IOException {
443 LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
444 Path familyDir = new Path(regionDir, family);
445 restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
446 }
447 });
448 }
449
450
451
452
453
454
455
456
457
458
459
460
461
462 private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
463 final String hfileName) throws IOException {
464 if (HFileLink.isHFileLink(hfileName)) {
465 HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
466 } else if (StoreFile.isReference(hfileName)) {
467 restoreReferenceFile(familyDir, regionInfo, hfileName);
468 } else {
469 HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
470 }
471 }
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491 private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
492 final String hfileName) throws IOException {
493
494 String tableName = snapshotDesc.getTable();
495 Path refPath = StoreFile.getReferredToFile(new Path(new Path(new Path(tableName,
496 regionInfo.getEncodedName()), familyDir.getName()), hfileName));
497 String snapshotRegionName = refPath.getParent().getParent().getName();
498 String fileName = refPath.getName();
499
500
501 String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
502 if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
503
504
505 String refLink = fileName;
506 if (!HFileLink.isHFileLink(fileName)) {
507 refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName);
508 }
509 Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
510
511
512 Path linkPath = new Path(familyDir,
513 HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName));
514 InputStream in = new HFileLink(conf, linkPath).open(fs);
515 OutputStream out = fs.create(outPath);
516 IOUtils.copyBytes(in, out, conf);
517 }
518
519
520
521
522
523
524
525
526
527 public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
528 return new HRegionInfo(tableDesc.getName(),
529 snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
530 snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
531 }
532
533
534
535
536
537
538
539
540
541
542 private void restoreWALs() throws IOException {
543 final SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
544 Bytes.toBytes(snapshotDesc.getTable()), regionsMap);
545 try {
546
547 SnapshotReferenceUtil.visitRecoveredEdits(fs, snapshotDir,
548 new FSVisitor.RecoveredEditsVisitor() {
549 public void recoveredEdits (final String region, final String logfile) throws IOException {
550 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
551 logSplitter.splitRecoveredEdit(path);
552 }
553 });
554
555
556 SnapshotReferenceUtil.visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
557 public void logFile (final String server, final String logfile) throws IOException {
558 logSplitter.splitLog(server, logfile);
559 }
560 });
561 } finally {
562 logSplitter.close();
563 }
564 }
565
566
567
568
569 private List<HRegionInfo> getTableRegions() throws IOException {
570 LOG.debug("get table regions: " + tableDir);
571 FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
572 if (regionDirs == null) return null;
573
574 List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
575 for (FileStatus regionDir: regionDirs) {
576 HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir.getPath());
577 regions.add(hri);
578 }
579 LOG.debug("found " + regions.size() + " regions for table=" + tableDesc.getNameAsString());
580 return regions;
581 }
582
583
584
585
586
587
588
589
590
591 public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor,
592 final byte[] tableName) throws IOException {
593 HTableDescriptor htd = new HTableDescriptor(tableName);
594 for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
595 htd.addFamily(hcd);
596 }
597 return htd;
598 }
599 }