1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.InputStream;
22 import java.io.IOException;
23 import java.io.OutputStream;
24 import java.util.Arrays;
25 import java.util.HashMap;
26 import java.util.HashSet;
27 import java.util.LinkedList;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Set;
31 import java.util.TreeMap;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.classification.InterfaceAudience;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.backup.HFileArchiver;
44 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
45 import org.apache.hadoop.hbase.io.HFileLink;
46 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
47 import org.apache.hadoop.hbase.regionserver.HRegion;
48 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
49 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.FSUtils;
52 import org.apache.hadoop.hbase.util.FSVisitor;
53 import org.apache.hadoop.hbase.util.ModifyRegionUtils;
54 import org.apache.hadoop.io.IOUtils;
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97 @InterfaceAudience.Private
98 public class RestoreSnapshotHelper {
99 private static final Log LOG = LogFactory.getLog(RestoreSnapshotHelper.class);
100
101 private final Map<byte[], byte[]> regionsMap =
102 new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
103
104 private final ForeignExceptionDispatcher monitor;
105
106 private final SnapshotDescription snapshotDesc;
107 private final Path snapshotDir;
108
109 private final HTableDescriptor tableDesc;
110 private final Path tableDir;
111
112 private final Configuration conf;
113 private final FileSystem fs;
114
115 public RestoreSnapshotHelper(final Configuration conf, final FileSystem fs,
116 final SnapshotDescription snapshotDescription, final Path snapshotDir,
117 final HTableDescriptor tableDescriptor, final Path tableDir,
118 final ForeignExceptionDispatcher monitor)
119 {
120 this.fs = fs;
121 this.conf = conf;
122 this.snapshotDesc = snapshotDescription;
123 this.snapshotDir = snapshotDir;
124 this.tableDesc = tableDescriptor;
125 this.tableDir = tableDir;
126 this.monitor = monitor;
127 }
128
129
130
131
132
133 public RestoreMetaChanges restoreHdfsRegions() throws IOException {
134 LOG.debug("starting restore");
135 Set<String> snapshotRegionNames = SnapshotReferenceUtil.getSnapshotRegionNames(fs, snapshotDir);
136 if (snapshotRegionNames == null) {
137 LOG.warn("Nothing to restore. Snapshot " + snapshotDesc + " looks empty");
138 return null;
139 }
140
141 RestoreMetaChanges metaChanges = new RestoreMetaChanges();
142
143
144
145 List<HRegionInfo> tableRegions = getTableRegions();
146 if (tableRegions != null) {
147 monitor.rethrowException();
148 for (HRegionInfo regionInfo: tableRegions) {
149 String regionName = regionInfo.getEncodedName();
150 if (snapshotRegionNames.contains(regionName)) {
151 LOG.info("region to restore: " + regionName);
152 snapshotRegionNames.remove(regionName);
153 metaChanges.addRegionToRestore(regionInfo);
154 } else {
155 LOG.info("region to remove: " + regionName);
156 metaChanges.addRegionToRemove(regionInfo);
157 }
158 }
159
160
161 monitor.rethrowException();
162 restoreHdfsRegions(metaChanges.getRegionsToRestore());
163
164
165 monitor.rethrowException();
166 removeHdfsRegions(metaChanges.getRegionsToRemove());
167 }
168
169
170 if (snapshotRegionNames.size() > 0) {
171 List<HRegionInfo> regionsToAdd = new LinkedList<HRegionInfo>();
172
173 monitor.rethrowException();
174 for (String regionName: snapshotRegionNames) {
175 LOG.info("region to add: " + regionName);
176 Path regionDir = new Path(snapshotDir, regionName);
177 regionsToAdd.add(HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir));
178 }
179
180
181 monitor.rethrowException();
182 HRegionInfo[] clonedRegions = cloneHdfsRegions(regionsToAdd);
183 metaChanges.setNewRegions(clonedRegions);
184 }
185
186
187 monitor.rethrowException();
188 restoreWALs();
189
190 return metaChanges;
191 }
192
193
194
195
196 public static class RestoreMetaChanges {
197 private List<HRegionInfo> regionsToRestore = null;
198 private List<HRegionInfo> regionsToRemove = null;
199 private List<HRegionInfo> regionsToAdd = null;
200
201
202
203
204 public boolean hasRegionsToAdd() {
205 return this.regionsToAdd != null && this.regionsToAdd.size() > 0;
206 }
207
208
209
210
211
212
213
214 public List<HRegionInfo> getRegionsToAdd() {
215 return this.regionsToAdd;
216 }
217
218
219
220
221 public boolean hasRegionsToRestore() {
222 return this.regionsToRestore != null && this.regionsToRestore.size() > 0;
223 }
224
225
226
227
228
229
230 public List<HRegionInfo> getRegionsToRestore() {
231 return this.regionsToRestore;
232 }
233
234
235
236
237 public boolean hasRegionsToRemove() {
238 return this.regionsToRemove != null && this.regionsToRemove.size() > 0;
239 }
240
241
242
243
244
245
246
247 public List<HRegionInfo> getRegionsToRemove() {
248 return this.regionsToRemove;
249 }
250
251 void setNewRegions(final HRegionInfo[] hris) {
252 if (hris != null) {
253 regionsToAdd = Arrays.asList(hris);
254 } else {
255 regionsToAdd = null;
256 }
257 }
258
259 void addRegionToRemove(final HRegionInfo hri) {
260 if (regionsToRemove == null) {
261 regionsToRemove = new LinkedList<HRegionInfo>();
262 }
263 regionsToRemove.add(hri);
264 }
265
266 void addRegionToRestore(final HRegionInfo hri) {
267 if (regionsToRestore == null) {
268 regionsToRestore = new LinkedList<HRegionInfo>();
269 }
270 regionsToRestore.add(hri);
271 }
272 }
273
274
275
276
277 private void removeHdfsRegions(final List<HRegionInfo> regions) throws IOException {
278 if (regions != null && regions.size() > 0) {
279 for (HRegionInfo hri: regions) {
280 HFileArchiver.archiveRegion(conf, fs, hri);
281 }
282 }
283 }
284
285
286
287
288 private void restoreHdfsRegions(final List<HRegionInfo> regions) throws IOException {
289 if (regions == null || regions.size() == 0) return;
290 for (HRegionInfo hri: regions) restoreRegion(hri);
291 }
292
293
294
295
296
297 private void restoreRegion(HRegionInfo regionInfo) throws IOException {
298 Path snapshotRegionDir = new Path(snapshotDir, regionInfo.getEncodedName());
299 Map<String, List<String>> snapshotFiles =
300 SnapshotReferenceUtil.getRegionHFileReferences(fs, snapshotRegionDir);
301 Path regionDir = new Path(tableDir, regionInfo.getEncodedName());
302 String tableName = tableDesc.getNameAsString();
303
304
305 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
306 byte[] family = Bytes.toBytes(familyDir.getName());
307 Set<String> familyFiles = getTableRegionFamilyFiles(familyDir);
308 List<String> snapshotFamilyFiles = snapshotFiles.remove(familyDir.getName());
309 if (snapshotFamilyFiles != null) {
310 List<String> hfilesToAdd = new LinkedList<String>();
311 for (String hfileName: snapshotFamilyFiles) {
312 if (familyFiles.contains(hfileName)) {
313
314 familyFiles.remove(hfileName);
315 } else {
316
317 hfilesToAdd.add(hfileName);
318 }
319 }
320
321
322 for (String hfileName: hfilesToAdd) {
323 LOG.trace("Adding HFileLink " + hfileName +
324 " to region=" + regionInfo.getEncodedName() + " table=" + tableName);
325 restoreStoreFile(familyDir, regionInfo, hfileName);
326 }
327
328
329 for (String hfileName: familyFiles) {
330 Path hfile = new Path(familyDir, hfileName);
331 LOG.trace("Removing hfile=" + hfile +
332 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
333 HFileArchiver.archiveStoreFile(conf, fs, regionInfo, tableDir, family, hfile);
334 }
335 } else {
336
337 LOG.trace("Removing family=" + Bytes.toString(family) +
338 " from region=" + regionInfo.getEncodedName() + " table=" + tableName);
339 HFileArchiver.archiveFamily(fs, conf, regionInfo, tableDir, family);
340 fs.delete(familyDir, true);
341 }
342 }
343
344
345 for (Map.Entry<String, List<String>> familyEntry: snapshotFiles.entrySet()) {
346 Path familyDir = new Path(regionDir, familyEntry.getKey());
347 if (!fs.mkdirs(familyDir)) {
348 throw new IOException("Unable to create familyDir=" + familyDir);
349 }
350
351 for (String hfileName: familyEntry.getValue()) {
352 LOG.trace("Adding HFileLink " + hfileName + " to table=" + tableName);
353 restoreStoreFile(familyDir, regionInfo, hfileName);
354 }
355 }
356 }
357
358
359
360
361 private Set<String> getTableRegionFamilyFiles(final Path familyDir) throws IOException {
362 Set<String> familyFiles = new HashSet<String>();
363
364 FileStatus[] hfiles = FSUtils.listStatus(fs, familyDir);
365 if (hfiles == null) return familyFiles;
366
367 for (FileStatus hfileRef: hfiles) {
368 String hfileName = hfileRef.getPath().getName();
369 familyFiles.add(hfileName);
370 }
371
372 return familyFiles;
373 }
374
375
376
377
378
379 private HRegionInfo[] cloneHdfsRegions(final List<HRegionInfo> regions) throws IOException {
380 if (regions == null || regions.size() == 0) return null;
381
382 final Map<String, HRegionInfo> snapshotRegions =
383 new HashMap<String, HRegionInfo>(regions.size());
384
385
386 HRegionInfo[] clonedRegionsInfo = new HRegionInfo[regions.size()];
387 for (int i = 0; i < clonedRegionsInfo.length; ++i) {
388
389 HRegionInfo snapshotRegionInfo = regions.get(i);
390 clonedRegionsInfo[i] = cloneRegionInfo(snapshotRegionInfo);
391
392
393 String snapshotRegionName = snapshotRegionInfo.getEncodedName();
394 String clonedRegionName = clonedRegionsInfo[i].getEncodedName();
395 regionsMap.put(Bytes.toBytes(snapshotRegionName), Bytes.toBytes(clonedRegionName));
396 LOG.info("clone region=" + snapshotRegionName + " as " + clonedRegionName);
397
398
399 snapshotRegions.put(clonedRegionName, snapshotRegionInfo);
400 }
401
402
403 ModifyRegionUtils.createRegions(conf, tableDir.getParent(),
404 tableDesc, clonedRegionsInfo, new ModifyRegionUtils.RegionFillTask() {
405 public void fillRegion(final HRegion region) throws IOException {
406 cloneRegion(region, snapshotRegions.get(region.getRegionInfo().getEncodedName()));
407 }
408 });
409
410 return clonedRegionsInfo;
411 }
412
413
414
415
416
417
418
419
420
421
422
423
424 private void cloneRegion(final HRegion region, final HRegionInfo snapshotRegionInfo)
425 throws IOException {
426 final Path snapshotRegionDir = new Path(snapshotDir, snapshotRegionInfo.getEncodedName());
427 final Path regionDir = new Path(tableDir, region.getRegionInfo().getEncodedName());
428 final String tableName = tableDesc.getNameAsString();
429 SnapshotReferenceUtil.visitRegionStoreFiles(fs, snapshotRegionDir,
430 new FSVisitor.StoreFileVisitor() {
431 public void storeFile (final String region, final String family, final String hfile)
432 throws IOException {
433 LOG.info("Adding HFileLink " + hfile + " to table=" + tableName);
434 Path familyDir = new Path(regionDir, family);
435 restoreStoreFile(familyDir, snapshotRegionInfo, hfile);
436 }
437 });
438 }
439
440
441
442
443
444
445
446
447
448
449
450
451
452 private void restoreStoreFile(final Path familyDir, final HRegionInfo regionInfo,
453 final String hfileName) throws IOException {
454 if (HFileLink.isHFileLink(hfileName)) {
455 HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName);
456 } else if (StoreFileInfo.isReference(hfileName)) {
457 restoreReferenceFile(familyDir, regionInfo, hfileName);
458 } else {
459 HFileLink.create(conf, fs, familyDir, regionInfo, hfileName);
460 }
461 }
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481 private void restoreReferenceFile(final Path familyDir, final HRegionInfo regionInfo,
482 final String hfileName) throws IOException {
483
484 String tableName = snapshotDesc.getTable();
485 Path refPath = StoreFileInfo.getReferredToFile(new Path(new Path(new Path(tableName,
486 regionInfo.getEncodedName()), familyDir.getName()), hfileName));
487 String snapshotRegionName = refPath.getParent().getParent().getName();
488 String fileName = refPath.getName();
489
490
491 String clonedRegionName = Bytes.toString(regionsMap.get(Bytes.toBytes(snapshotRegionName)));
492 if (clonedRegionName == null) clonedRegionName = snapshotRegionName;
493
494
495 String refLink = fileName;
496 if (!HFileLink.isHFileLink(fileName)) {
497 refLink = HFileLink.createHFileLinkName(tableName, snapshotRegionName, fileName);
498 }
499 Path outPath = new Path(familyDir, refLink + '.' + clonedRegionName);
500
501
502 Path linkPath = new Path(familyDir,
503 HFileLink.createHFileLinkName(tableName, regionInfo.getEncodedName(), hfileName));
504 InputStream in = new HFileLink(conf, linkPath).open(fs);
505 OutputStream out = fs.create(outPath);
506 IOUtils.copyBytes(in, out, conf);
507 }
508
509
510
511
512
513
514
515
516
517 public HRegionInfo cloneRegionInfo(final HRegionInfo snapshotRegionInfo) {
518 return new HRegionInfo(tableDesc.getName(),
519 snapshotRegionInfo.getStartKey(), snapshotRegionInfo.getEndKey(),
520 snapshotRegionInfo.isSplit(), snapshotRegionInfo.getRegionId());
521 }
522
523
524
525
526
527
528
529
530
531
532 private void restoreWALs() throws IOException {
533 final SnapshotLogSplitter logSplitter = new SnapshotLogSplitter(conf, fs, tableDir,
534 Bytes.toBytes(snapshotDesc.getTable()), regionsMap);
535 try {
536
537 SnapshotReferenceUtil.visitRecoveredEdits(fs, snapshotDir,
538 new FSVisitor.RecoveredEditsVisitor() {
539 public void recoveredEdits (final String region, final String logfile) throws IOException {
540 Path path = SnapshotReferenceUtil.getRecoveredEdits(snapshotDir, region, logfile);
541 logSplitter.splitRecoveredEdit(path);
542 }
543 });
544
545
546 SnapshotReferenceUtil.visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
547 public void logFile (final String server, final String logfile) throws IOException {
548 logSplitter.splitLog(server, logfile);
549 }
550 });
551 } finally {
552 logSplitter.close();
553 }
554 }
555
556
557
558
559 private List<HRegionInfo> getTableRegions() throws IOException {
560 LOG.debug("get table regions: " + tableDir);
561 FileStatus[] regionDirs = FSUtils.listStatus(fs, tableDir, new FSUtils.RegionDirFilter(fs));
562 if (regionDirs == null) return null;
563
564 List<HRegionInfo> regions = new LinkedList<HRegionInfo>();
565 for (FileStatus regionDir: regionDirs) {
566 HRegionInfo hri = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir.getPath());
567 regions.add(hri);
568 }
569 LOG.debug("found " + regions.size() + " regions for table=" + tableDesc.getNameAsString());
570 return regions;
571 }
572
573
574
575
576
577
578
579
580
581 public static HTableDescriptor cloneTableSchema(final HTableDescriptor snapshotTableDescriptor,
582 final byte[] tableName) throws IOException {
583 HTableDescriptor htd = new HTableDescriptor(tableName);
584 for (HColumnDescriptor hcd: snapshotTableDescriptor.getColumnFamilies()) {
585 htd.addFamily(hcd);
586 }
587 return htd;
588 }
589 }