View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HTableDescriptor;
33  import org.apache.hadoop.hbase.catalog.CatalogTracker;
34  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
35  import org.apache.hadoop.hbase.io.HFileLink;
36  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
37  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
38  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
39  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
40  import org.apache.hadoop.hbase.testclassification.SmallTests;
41  import org.apache.hadoop.hbase.util.FSTableDescriptors;
42  import org.apache.hadoop.hbase.util.FSUtils;
43  import org.junit.After;
44  import org.junit.Before;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  import org.mockito.Mockito;
48  
49  /**
50   * Test the restore/clone operation from a file-system point of view.
51   */
52  @Category(SmallTests.class)
53  public class TestRestoreSnapshotHelper {
54    final Log LOG = LogFactory.getLog(getClass());
55  
56    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57    private final static String TEST_HFILE = "abc";
58  
59    private Configuration conf;
60    private Path archiveDir;
61    private FileSystem fs;
62    private Path rootDir;
63  
64    @Before
65    public void setup() throws Exception {
66      rootDir = TEST_UTIL.getDataTestDir("testRestore");
67      archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
68      fs = TEST_UTIL.getTestFileSystem();
69      conf = TEST_UTIL.getConfiguration();
70      FSUtils.setRootDir(conf, rootDir);
71    }
72  
73    @After
74    public void tearDown() throws Exception {
75      fs.delete(TEST_UTIL.getDataTestDir(), true);
76    }
77  
78    @Test
79    public void testRestore() throws IOException {
80      restoreAndVerify("snapshot", "testRestore");
81    }
82  
83    @Test
84    public void testRestoreWithNamespace() throws IOException {
85      restoreAndVerify("snapshot", "namespace1:testRestoreWithNamespace");
86    }
87  
88    private void restoreAndVerify(final String snapshotName, final String tableName) throws IOException {
89      // Test Rolling-Upgrade like Snapshot.
90      // half machines writing using v1 and the others using v2 format.
91      SnapshotMock snapshotMock = new SnapshotMock(TEST_UTIL.getConfiguration(), fs, rootDir);
92      SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(snapshotName, tableName);
93      builder.addRegionV1();
94      builder.addRegionV2();
95      builder.addRegionV2();
96      builder.addRegionV1();
97      Path snapshotDir = builder.commit();
98      HTableDescriptor htd = builder.getTableDescriptor();
99      SnapshotDescription desc = builder.getSnapshotDescription();
100 
101     // Test clone a snapshot
102     HTableDescriptor htdClone = snapshotMock.createHtd("testtb-clone");
103     testRestore(snapshotDir, desc, htdClone);
104     verifyRestore(rootDir, htd, htdClone);
105 
106     // Test clone a clone ("link to link")
107     SnapshotDescription cloneDesc = SnapshotDescription.newBuilder()
108         .setName("cloneSnapshot")
109         .setTable("testtb-clone")
110         .build();
111     Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName());
112     HTableDescriptor htdClone2 = snapshotMock.createHtd("testtb-clone2");
113     testRestore(cloneDir, cloneDesc, htdClone2);
114     verifyRestore(rootDir, htd, htdClone2);
115   }
116 
117   private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd,
118       final HTableDescriptor htdClone) throws IOException {
119     String[] files = SnapshotTestingUtils.listHFileNames(fs,
120       FSUtils.getTableDir(rootDir, htdClone.getTableName()));
121     assertEquals(12, files.length);
122     for (int i = 0; i < files.length; i += 2) {
123       String linkFile = files[i];
124       String refFile = files[i+1];
125       assertTrue(linkFile + " should be a HFileLink", HFileLink.isHFileLink(linkFile));
126       assertTrue(refFile + " should be a Referene", StoreFileInfo.isReference(refFile));
127       assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(linkFile));
128       Path refPath = getReferredToFile(refFile);
129       LOG.debug("get reference name for file " + refFile + " = " + refPath);
130       assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
131       assertEquals(linkFile, refPath.getName());
132     }
133   }
134 
135   /**
136    * Execute the restore operation
137    * @param snapshotDir The snapshot directory to use as "restore source"
138    * @param sd The snapshot descriptor
139    * @param htdClone The HTableDescriptor of the table to restore/clone.
140    */
141   public void testRestore(final Path snapshotDir, final SnapshotDescription sd,
142       final HTableDescriptor htdClone) throws IOException {
143     LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
144     FSUtils.logFileSystemState(fs, rootDir, LOG);
145 
146     new FSTableDescriptors(conf).createTableDescriptor(htdClone);
147     RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sd, htdClone);
148     helper.restoreHdfsRegions();
149 
150     LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
151     FSUtils.logFileSystemState(fs, rootDir, LOG);
152   }
153 
154   /**
155    * Initialize the restore helper, based on the snapshot and table information provided.
156    */
157   private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
158       final SnapshotDescription sd, final HTableDescriptor htdClone) throws IOException {
159     CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
160     HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class);
161     ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
162     MonitoredTask status = Mockito.mock(MonitoredTask.class);
163 
164     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
165     return new RestoreSnapshotHelper(conf, fs, manifest,
166       htdClone, rootDir, monitor, status);
167   }
168 
169   private Path getReferredToFile(final String referenceName) {
170     Path fakeBasePath = new Path(new Path("table", "region"), "cf");
171     return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
172   }
173 }