1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.InputStream;
22 import java.io.IOException;
23 import java.io.OutputStream;
24 import java.util.Arrays;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.LinkedList;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Set;
31 import java.util.TreeMap;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.classification.InterfaceAudience;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.backup.HFileArchiver;
44 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
45 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
46 import org.apache.hadoop.hbase.io.HFileLink;
47 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
48 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
49 import org.apache.hadoop.hbase.regionserver.HRegion;
50 import org.apache.hadoop.hbase.regionserver.StoreFile;
51 import org.apache.hadoop.hbase.util.Bytes;
52 import org.apache.hadoop.hbase.util.FSUtils;
53 import org.apache.hadoop.hbase.util.FSVisitor;
54 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
55 import org.apache.hadoop.io.IOUtils;
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98 @InterfaceAudience.Private
99 public class RestoreSnapshotHelper {
100 private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
101
102 private final Map<byte[], byte[]> regionsMap =
103 new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
104
105 private final ForeignExceptionDispatcher monitor;
106 private final MonitoredTask status;
107
108 private final SnapshotDescription snapshotDesc;
109 private final Path snapshotDir;
110
111 private final HTableDescriptor tableDesc;
112 private final Path tableDir;
113
114 private final Configuration conf;
115 private final FileSystem fs;
116
117 public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs,
118 final SnapshotDescription snapshotDescription, final Path snapshotDir,
119 final HTableDescriptor tableDescriptor, final Path tableDir,
120 final ForeignExceptionDispatcher monitor, final MonitoredTask status)
121 {
122 this.fs = fs;
123 this.conf = conf;
124 this.snapshotDesc = snapshotDescription;
125 this.snapshotDir = snapshotDir;
126 this.tableDesc = tableDescriptor;
127 this.tableDir = tableDir;
128 this.monitor = monitor;
129 this.status = status;
130 }
131
132
133
134
135
136 public RestoreMetaChanges restoreHdfsRegions() throws IOException {
137 LOG.debug("starting restore");
138 Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
139 if (snapshotRegionNames == null) {
140 LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
141 return null;
142 }
143
144 RestoreMetaChanges metaChanges = new RestoreMetaChanges();
145
146
147
148 List<HRegionInfo> tableRegions = getTableRegions();
149 if (tableRegions != null) {
150 monitor.rethrowException();
151 for (HRegionInfo regionInfo: tableRegions) {
152 String regionName = regionInfo.getEncodedName();
153 if (snapshotRegionNames.contains(regionName)) {
154 LOG.info("region to restore: " + regionName);
155 snapshotRegionNames.remove(regionName);
156 metaChanges.addRegionToRestore(regionInfo);
157 } else {
158 LOG.info("region to remove: " + regionName);
159 metaChanges.addRegionToRemove(regionInfo);
160 }
161 }
162
163
164 monitor.rethrowException();
165 status.setStatus("Restoring table regions...");
166 restoreHdfsRegions(metaChanges.getRegionsToRestore());
167 status.setStatus("Finished restoring all table regions.");
168
169
170 monitor.rethrowException();
171 status.setStatus("Starting to delete excess regions from table");
172 removeHdfsRegions(metaChanges.getRegionsToRemove());
173 status.setStatus("Finished deleting excess regions from table.");
174 }
175
176
177 if (snapshotRegionNames.size() > 0) {
178 List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();
179
180 monitor.rethrowException();
181 for (String regionName: snapshotRegionNames) {
182 LOG.info("region to add: " + regionName);
183 Path regionDir = new Path(snapshotDir, regionName);
184 regionsToAdd.add(HRegion.loadDotRegionInfoFileContent(fs, regionDir));
185 }
186
187
188 monitor.rethrowException();
189 status.setStatus("Cloning regions...");
190 HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
191 metaChanges.setNewRegions(clonedRegions);
192 status.setStatus("Finished cloning regions.");
193 }
194
195
196 monitor.rethrowException();
197 status.setStatus("Restoring WALs to table...");
198 restoreWALs();
199 status.setStatus("Finished restoring WALs to table.");
200
201 return metaChanges;
202 }
203
204
205
206
207 public static class RestoreMetaChanges {
208 private List<HRegionInfo> regionsToRestore = null;
209 private List<HRegionInfo> regionsToRemove = null;
210 private List<HRegionInfo> regionsToAdd = null;
211
212
213
214
215 public boolean hasRegionsToAdd() {
216 return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
217 }
218
219
220
221
222
223
224
225 public List<HRegionInfo> getRegionsToAdd() {
226 return this.regionsToAdd;
227 }
228
229
230
231
232 public boolean hasRegionsToRestore() {
233 return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
234 }
235
236
237
238
239
240
241 public List<HRegionInfo> getRegionsToRestore() {
242 return this.regionsToRestore;
243 }
244
245
246
247
248 public boolean hasRegionsToRemove() {
249 return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
250 }
251
252
253
254
255
256
257
258 public List<HRegionInfo> getRegionsToRemove() {
259 return this.regionsToRemove;
260 }
261
262 void setNewRegions(final HRegionInfo[] hris) {
263 if (hris != null) {
264 regionsToAdd = Arrays.asList(hris);
265 } else {
266 regionsToAdd = null;
267 }
268 }
269
270 void addRegionToRemove(final HRegionInfo hri) {
271 if (regionsToRemove == null) {
272 regionsToRemove = new LinkedList<HRegionInfo>();
273 }
274 regionsToRemove.add(hri);
275 }
276
277 void addRegionToRestore(final HRegionInfo hri) {
278 if (regionsToRestore == null) {
279 regionsToRestore = new LinkedList<HRegionInfo>();
280 }
281 regionsToRestore.add(hri);
282 }
283 }
284
285
286
287
288 private void removeHdfsRegions(final List<HRegionInfo> regions) throws IOException {
289 if (regions != null && regions.size() > 0) {
290 for (HRegionInfo hri: regions) {
291 HFileArchiver.archiveRegion(conf, fs, hri);
292 }
293 }
294 }
295
296
297
298
299 private void restoreHdfsRegions(final List<HRegionInfo> regions) throws IOException {
300 if (regions == null || regions.size() == 0) return;
301 for (HRegionInfo hri: regions) restoreRegion(hri);
302 }
303
304
305
306
307
308 private void restoreRegion(HRegionInfo regionInfo) throws IOException {
309 Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName());
310 Map<String, List<String>> snapshotFiles =
311 SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir);
312 Path regionDir = new Path(tableDir, regionInfo.getEncodedName());
313 String tableName = tableDesc.getNameAsString();
314
315
316 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
317 byte[] family = Bytes.toBytes(familyDir.getName());
318 Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
319 List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
320 if (snapshotFamilyFiles != null) {
321 List<String> hfilesToAdd = new LinkedList<String>();
322 for (String hfileName: snapshotFamilyFiles) {
323 if (familyFiles.contains(hfileName)) {
324
325 familyFiles.remove(hfileName);
326 } else {
327
328 hfilesToAdd.add(hfileName);
329 }
330 }
331
332
333 for (String hfileName: hfilesToAdd) {
334 LOG.trace("Adding HFileLink " + hfileName +
335 " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
336 restoreStoreFile(familyDir, regionInfo, hfileName);
337 }
338
339
340 for (String hfileName: familyFiles) {
341 Path hfile = new Path(familyDir, hfileName);
342 LOG.trace("Removing hfile=" + hfile +
343 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
344 HFileArchiver.archiveStoreFile(fs, regionInfo, conf, tableDir, family, hfile);
345 }
346 } else {
347
348 LOG.trace("Removing family=" + Bytes.toString(family) +
349 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
350 HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family);
351 fs.delete(familyDir, true);
352 }
353 }
354
355
356 for (Map.Entry<String, List<String>> familyEntry: snapshotFiles.entrySet()) {
357 Path familyDir = new Path(regionDir, familyEntry.getKey());
358 if (!fs.mkdirs(familyDir)) {
359 throw new IOException("Unable to create familyDir=" + familyDir);
360 }
361
362 for (String hfileName: familyEntry.getValue()) {
363 LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName);
364 restoreStoreFile(familyDir, regionInfo, hfileName);
365 }
366 }
367 }
368
369
370
371
372 private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
373 Set<String> familyFiles = new HashSet<String>();
374
375 FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
376 if (hfiles == null) return familyFiles;
377
378 for (FileStatus hfileRef: hfiles) {
379 String hfileName = hfileRef.getPath().getName();
380 familyFiles.add(hfileName);
381 }
382
383 return familyFiles;
384 }
385
386
387
388
389
390 private HRegionInfo[] cloneHdfsRegions(final List<HRegionInfo> regions) throws IOException {
391 if (regions == null || regions.size() == 0) return null;
392
393 final Map<String, HRegionInfo> snapshotRegions =
394 new HashMap<String, HRegionInfo>(regions.size());
395
396
397 HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
398 for (int i = 0; i < clonedRegionsInfo.length; ++i) {
399
400 HRegionInfo snapshotRegionInfo = regions.get(i);
401 clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
402
403
404 String snapshotRegionName = snapshotRegionInfo.getEncodedName();
405 String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
406 regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
407 LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
408
409
410 snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
411 }
412
413
414 ModifyRegionUtils.createRegions(conf, tableDir.getParent(),
415 tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
416 public void fillRegion(final HRegion region) throws IOException {
417 cloneRegion(region, snapshotRegions.get(region.getRegionInfo().getEncodedName()));
418 }
419 });
420
421 return clonedRegionsInfo;
422 }
423
424
425
426
427
428
429
430
431
432
433
434
435 private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
436 throws IOException {
437 final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
438 final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
439 final String tableName = tableDesc.getNameAsString();
440 SnapshotReferenceUtil.visitRegionStoreFiles(fs, snapshotRegionDir,
441 new FSVisitor.StoreFileVisitor() {
442 public void storeFile (final String region, final String family, final String hfile)
443 throws IOException {
444 LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
445 Path familyDir = new Path(regionDir, family);
446 restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
447 }
448 });
449 }
450
451
452
453
454
455
456
457
458
459
460
461
462
463 private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
464 final String hfileName) throws IOException {
465 if (HFileLink.isHFileLink(hfileName)) {
466 HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
467 } else if (StoreFile.isReference(hfileName)) {
468 restoreReferenceFile(familyDir, regionInfo, hfileName);
469 } else {
470 HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
471 }
472 }
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492 private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
493 final String hfileName) throws IOException {
494
495 String tableName = snapshotDesc.getTable();
496 Path refPath = StoreFile.getReferredToFile(new Path(new Path(new Path(tableName,
497 regionInfo.getEncodedName()), familyDir.getName()), hfileName));
498 String snapshotRegionName = refPath.getParent().getParent().getName();
499 String fileName = refPath.getName();
500
501
502 String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
503 if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
504
505
506 String refLink = fileName;
507 if (!HFileLink.isHFileLink(fileName)) {
508 refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName);
509 }
510 Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
511
512
513 Path linkPath = new Path(familyDir,
514 HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName));
515 InputStream in = new HFileLink(conf, linkPath).open(fs);
516 OutputStream out = fs.create(outPath);
517 IOUtils.copyBytes(in, out, conf);
518 }
519
520
521
522
523
524
525
526
527
528 public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
529 return new HRegionInfo(tableDesc.getName(),
530 snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
531 snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
532 }
533
534
535
536
537
538
539
540
541
542
543 private void restoreWALs() throws IOException {
544 final SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
545 Bytes.toBytes(snapshotDesc.getTable()), regionsMap);
546 try {
547
548 SnapshotReferenceUtil.visitRecoveredEdits(fs, snapshotDir,
549 new FSVisitor.RecoveredEditsVisitor() {
550 public void recoveredEdits (final String region, final String logfile) throws IOException {
551 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
552 logSplitter.splitRecoveredEdit(path);
553 }
554 });
555
556
557 SnapshotReferenceUtil.visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
558 public void logFile (final String server, final String logfile) throws IOException {
559 logSplitter.splitLog(server, logfile);
560 }
561 });
562 } finally {
563 logSplitter.close();
564 }
565 }
566
567
568
569
570 private List<HRegionInfo> getTableRegions() throws IOException {
571 LOG.debug("get table regions: " + tableDir);
572 FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
573 if (regionDirs == null) return null;
574
575 List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
576 for (FileStatus regionDir: regionDirs) {
577 HRegionInfo hri = HRegion.loadDotRegionInfoFileContent(fs, regionDir.getPath());
578 regions.add(hri);
579 }
580 LOG.debug("found " + regions.size() + " regions for table=" + tableDesc.getNameAsString());
581 return regions;
582 }
583
584
585
586
587
588
589
590
591
592 public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor,
593 final byte[] tableName) throws IOException {
594 HTableDescriptor htd = new HTableDescriptor(tableName);
595 for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
596 htd.addFamily(hcd);
597 }
598 for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> e:
599 snapshotTableDescriptor.getValues().entrySet()) {
600 htd.setValue(e.getKey(), e.getValue());
601 }
602 return htd;
603 }
604 }