1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Collections;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileStatus;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.FileUtil;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HColumnDescriptor;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.HRegionInfo;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.SmallTests;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.catalog.CatalogTracker;
43 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
44 import org.apache.hadoop.hbase.io.HFileLink;
45 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
46 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
47 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
48 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
49 import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
50 import org.apache.hadoop.hbase.util.FSTableDescriptors;
51 import org.apache.hadoop.hbase.util.FSUtils;
52 import org.junit.After;
53 import org.junit.Before;
54 import org.junit.Test;
55 import org.junit.experimental.categories.Category;
56 import org.mockito.Mockito;
57
58
59
60
61 @Category(SmallTests.class)
62 public class TestRestoreSnapshotHelper {
63 final Log LOG = LogFactory.getLog(getClass());
64
65 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66 private final static String TEST_FAMILY = "cf";
67 private final static String TEST_HFILE = "abc";
68
69 private Configuration conf;
70 private Path archiveDir;
71 private FileSystem fs;
72 private Path rootDir;
73
74 @Before
75 public void setup() throws Exception {
76 rootDir = TEST_UTIL.getDataTestDir("testRestore");
77 archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
78 fs = TEST_UTIL.getTestFileSystem();
79 conf = TEST_UTIL.getConfiguration();
80 FSUtils.setRootDir(conf, rootDir);
81 }
82
83 @After
84 public void tearDown() throws Exception {
85 fs.delete(TEST_UTIL.getDataTestDir(), true);
86 }
87
88 @Test
89 public void testRestore() throws IOException {
90 HTableDescriptor htd = createTableDescriptor("testtb");
91
92 Path snapshotDir = new Path(rootDir, "snapshot");
93 createSnapshot(rootDir, snapshotDir, htd);
94
95
96 HTableDescriptor htdClone = createTableDescriptor("testtb-clone");
97 testRestore(snapshotDir, htd.getTableName().getNameAsString(), htdClone);
98 verifyRestore(rootDir, htd, htdClone);
99
100
101 Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName());
102 HTableDescriptor htdClone2 = createTableDescriptor("testtb-clone2");
103 testRestore(cloneDir, htdClone.getTableName().getNameAsString(), htdClone2);
104 verifyRestore(rootDir, htd, htdClone2);
105 }
106
107 private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd,
108 final HTableDescriptor htdClone) throws IOException {
109 String[] files = getHFiles(FSUtils.getTableDir(rootDir, htdClone.getTableName()));
110 assertEquals(2, files.length);
111 assertTrue(files[0] + " should be a HFileLink", HFileLink.isHFileLink(files[0]));
112 assertTrue(files[1] + " should be a Referene", StoreFileInfo.isReference(files[1]));
113 assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(files[0]));
114 assertEquals(TEST_HFILE, HFileLink.getReferencedHFileName(files[0]));
115 Path refPath = getReferredToFile(files[1]);
116 assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
117 assertEquals(files[0], refPath.getName());
118 }
119
120
121
122
123
124
125
126 public void testRestore(final Path snapshotDir, final String sourceTableName,
127 final HTableDescriptor htdClone) throws IOException {
128 LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
129 FSUtils.logFileSystemState(fs, rootDir, LOG);
130
131 new FSTableDescriptors(conf).createTableDescriptor(htdClone);
132 RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sourceTableName, htdClone);
133 helper.restoreHdfsRegions();
134
135 LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
136 FSUtils.logFileSystemState(fs, rootDir, LOG);
137 }
138
139
140
141
142 private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
143 final String sourceTableName, final HTableDescriptor htdClone) throws IOException {
144 CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
145 HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class);
146 ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
147 MonitoredTask status = Mockito.mock(MonitoredTask.class);
148
149 SnapshotDescription sd = SnapshotDescription.newBuilder()
150 .setName("snapshot")
151 .setTable(sourceTableName)
152 .build();
153
154 return new RestoreSnapshotHelper(conf, fs, sd, snapshotDir,
155 htdClone, rootDir, monitor, status);
156 }
157
158 private void createSnapshot(final Path rootDir, final Path snapshotDir, final HTableDescriptor htd)
159 throws IOException {
160
161 HRegionInfo hri = new HRegionInfo(htd.getTableName());
162 HRegionFileSystem r0fs = HRegionFileSystem.createRegionOnFileSystem(conf,
163 fs, FSUtils.getTableDir(archiveDir, hri.getTableName()), hri);
164 Path storeFile = new Path(rootDir, TEST_HFILE);
165 fs.createNewFile(storeFile);
166 r0fs.commitStoreFile(TEST_FAMILY, storeFile);
167
168
169
170 hri = new HRegionInfo(htd.getTableName());
171 HRegionFileSystem r1fs = HRegionFileSystem.createRegionOnFileSystem(conf,
172 fs, FSUtils.getTableDir(archiveDir, hri.getTableName()), hri);
173 storeFile = new Path(rootDir, TEST_HFILE + '.' + r0fs.getRegionInfo().getEncodedName());
174 fs.createNewFile(storeFile);
175 r1fs.commitStoreFile(TEST_FAMILY, storeFile);
176
177 Path tableDir = FSUtils.getTableDir(archiveDir, htd.getTableName());
178 FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
179 }
180
181 private HTableDescriptor createTableDescriptor(final String tableName) {
182 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
183 htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
184 return htd;
185 }
186
187 private Path getReferredToFile(final String referenceName) {
188 Path fakeBasePath = new Path(new Path("table", "region"), "cf");
189 return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
190 }
191
192 private String[] getHFiles(final Path tableDir) throws IOException {
193 List<String> files = new ArrayList<String>();
194 for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
195 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
196 for (FileStatus file: FSUtils.listStatus(fs, familyDir)) {
197 files.add(file.getPath().getName());
198 }
199 }
200 }
201 Collections.sort(files);
202 return files.toArray(new String[files.size()]);
203 }
204 }