1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.Arrays;
26 import java.util.ArrayList;
27 import java.util.Collections;
28 import java.util.List;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.fs.FileStatus;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.FileUtil;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.HTableDescriptor;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.SmallTests;
43 import org.apache.hadoop.hbase.catalog.CatalogTracker;
44 import org.apache.hadoop.hbase.client.HConnection;
45 import org.apache.hadoop.hbase.client.HConnectionTestingUtility;
46 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
47 import org.apache.hadoop.hbase.io.HFileLink;
48 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
49 import org.apache.hadoop.hbase.regionserver.HRegion;
50 import org.apache.hadoop.hbase.regionserver.StoreFile;
51 import org.apache.hadoop.hbase.util.Bytes;
52 import org.apache.hadoop.hbase.util.FSTableDescriptors;
53 import org.apache.hadoop.hbase.util.FSUtils;
54 import org.apache.hadoop.hbase.util.MD5Hash;
55 import org.junit.*;
56 import org.junit.experimental.categories.Category;
57 import org.mockito.Mockito;
58
59
60
61
62 @Category(SmallTests.class)
63 public class TestRestoreSnapshotHelper {
64 final Log LOG = LogFactory.getLog(getClass());
65
66 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
67 private final static String TEST_FAMILY = "cf";
68 private final static String TEST_HFILE = "abc";
69
70 private Configuration conf;
71 private Path archiveDir;
72 private FileSystem fs;
73 private Path rootDir;
74
75 @Before
76 public void setup() throws Exception {
77 rootDir = TEST_UTIL.getDataTestDir("testRestore");
78 archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
79 fs = TEST_UTIL.getTestFileSystem();
80 conf = TEST_UTIL.getConfiguration();
81 FSUtils.setRootDir(conf, rootDir);
82 }
83
84 @After
85 public void tearDown() throws Exception {
86 fs.delete(TEST_UTIL.getDataTestDir(), true);
87 }
88
89 @Test
90 public void testRestore() throws IOException {
91 HTableDescriptor htd = createTableDescriptor("testtb");
92
93 Path snapshotDir = new Path(rootDir, "snapshot");
94 createSnapshot(rootDir, snapshotDir, htd);
95
96
97 HTableDescriptor htdClone = createTableDescriptor("testtb-clone");
98 testRestore(snapshotDir, htd.getNameAsString(), htdClone);
99 verifyRestore(rootDir, htd, htdClone);
100
101
102 Path cloneDir = HTableDescriptor.getTableDir(rootDir, htdClone.getName());
103 HTableDescriptor htdClone2 = createTableDescriptor("testtb-clone2");
104 testRestore(cloneDir, htdClone.getNameAsString(), htdClone2);
105 verifyRestore(rootDir, htd, htdClone2);
106 }
107
108 private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd,
109 final HTableDescriptor htdClone) throws IOException {
110 String[] files = getHFiles(HTableDescriptor.getTableDir(rootDir, htdClone.getName()));
111 assertEquals(2, files.length);
112 assertTrue(files[0] + " should be a HFileLink", HFileLink.isHFileLink(files[0]));
113 assertTrue(files[1] + " should be a Referene", StoreFile.isReference(files[1]));
114 assertEquals(sourceHtd.getNameAsString(), HFileLink.getReferencedTableName(files[0]));
115 assertEquals(TEST_HFILE, HFileLink.getReferencedHFileName(files[0]));
116 Path refPath = getReferredToFile(files[1]);
117 assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
118 assertEquals(files[0], refPath.getName());
119 }
120
121
122
123
124
125
126
127 public void testRestore(final Path snapshotDir, final String sourceTableName,
128 final HTableDescriptor htdClone) throws IOException {
129 LOG.debug("pre-restore table=" + htdClone.getNameAsString() + " snapshot=" + snapshotDir);
130 FSUtils.logFileSystemState(fs, rootDir, LOG);
131
132 FSTableDescriptors.createTableDescriptor(htdClone, conf);
133 RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sourceTableName, htdClone);
134 helper.restoreHdfsRegions();
135
136 LOG.debug("post-restore table=" + htdClone.getNameAsString() + " snapshot=" + snapshotDir);
137 FSUtils.logFileSystemState(fs, rootDir, LOG);
138 }
139
140
141
142
143 private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
144 final String sourceTableName, final HTableDescriptor htdClone) throws IOException {
145 CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
146 HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class);
147 ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
148
149 SnapshotDescription sd = SnapshotDescription.newBuilder()
150 .setName("snapshot").setTable(sourceTableName).build();
151
152 return new RestoreSnapshotHelper(conf, fs, sd, snapshotDir,
153 htdClone, HTableDescriptor.getTableDir(rootDir, htdClone.getName()), monitor);
154 }
155
156 private void createSnapshot(final Path rootDir, final Path snapshotDir, final HTableDescriptor htd)
157 throws IOException {
158
159 HRegion r0 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
160 conf, htd, null, true, true);
161 Path storeFile = new Path(new Path(r0.getRegionDir(), TEST_FAMILY), TEST_HFILE);
162 fs.createNewFile(storeFile);
163 r0.close();
164
165
166
167 HRegion r1 = HRegion.createHRegion(new HRegionInfo(htd.getName()), archiveDir,
168 conf, htd, null, true, true);
169 fs.createNewFile(new Path(new Path(r1.getRegionDir(), TEST_FAMILY),
170 storeFile.getName() + '.' + r0.getRegionInfo().getEncodedName()));
171 r1.close();
172
173 Path tableDir = HTableDescriptor.getTableDir(archiveDir, htd.getName());
174 FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
175 }
176
177 private HTableDescriptor createTableDescriptor(final String tableName) {
178 HTableDescriptor htd = new HTableDescriptor(tableName);
179 htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
180 return htd;
181 }
182
183 private Path getReferredToFile(final String referenceName) {
184 Path fakeBasePath = new Path(new Path("table", "region"), "cf");
185 return StoreFile.getReferredToFile(new Path(fakeBasePath, referenceName));
186 }
187
188 private String[] getHFiles(final Path tableDir) throws IOException {
189 List<String> files = new ArrayList<String>();
190 for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
191 for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
192 for (FileStatus file: FSUtils.listStatus(fs, familyDir)) {
193 files.add(file.getPath().getName());
194 }
195 }
196 }
197 Collections.sort(files);
198 return files.toArray(new String[files.size()]);
199 }
200 }