View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.Collections;
26  import java.util.List;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileStatus;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.FileUtil;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HColumnDescriptor;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.SmallTests;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.catalog.CatalogTracker;
43  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
44  import org.apache.hadoop.hbase.io.HFileLink;
45  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
46  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
47  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
48  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
49  import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
50  import org.apache.hadoop.hbase.util.FSTableDescriptors;
51  import org.apache.hadoop.hbase.util.FSUtils;
52  import org.junit.After;
53  import org.junit.Before;
54  import org.junit.Test;
55  import org.junit.experimental.categories.Category;
56  import org.mockito.Mockito;
57  
58  /**
59   * Test the restore/clone operation from a file-system point of view.
60   */
61  @Category(SmallTests.class)
62  public class TestRestoreSnapshotHelper {
63    final Log LOG = LogFactory.getLog(getClass());
64  
65    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
66    private final static String TEST_FAMILY = "cf";
67    private final static String TEST_HFILE = "abc";
68  
69    private Configuration conf;
70    private Path archiveDir;
71    private FileSystem fs;
72    private Path rootDir;
73  
74    @Before
75    public void setup() throws Exception {
76      rootDir = TEST_UTIL.getDataTestDir("testRestore");
77      archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
78      fs = TEST_UTIL.getTestFileSystem();
79      conf = TEST_UTIL.getConfiguration();
80      FSUtils.setRootDir(conf, rootDir);
81    }
82  
83    @After
84    public void tearDown() throws Exception {
85      fs.delete(TEST_UTIL.getDataTestDir(), true);
86    }
87  
88    @Test
89    public void testRestore() throws IOException {
90      HTableDescriptor htd = createTableDescriptor("testtb");
91  
92      Path snapshotDir = new Path(rootDir, "snapshot");
93      createSnapshot(rootDir, snapshotDir, htd);
94  
95      // Test clone a snapshot
96      HTableDescriptor htdClone = createTableDescriptor("testtb-clone");
97      testRestore(snapshotDir, htd.getTableName().getNameAsString(), htdClone);
98      verifyRestore(rootDir, htd, htdClone);
99  
100     // Test clone a clone ("link to link")
101     Path cloneDir = FSUtils.getTableDir(rootDir, htdClone.getTableName());
102     HTableDescriptor htdClone2 = createTableDescriptor("testtb-clone2");
103     testRestore(cloneDir, htdClone.getTableName().getNameAsString(), htdClone2);
104     verifyRestore(rootDir, htd, htdClone2);
105   }
106 
107   private void verifyRestore(final Path rootDir, final HTableDescriptor sourceHtd,
108       final HTableDescriptor htdClone) throws IOException {
109     String[] files = getHFiles(FSUtils.getTableDir(rootDir, htdClone.getTableName()));
110     assertEquals(2, files.length);
111     assertTrue(files[0] + " should be a HFileLink", HFileLink.isHFileLink(files[0]));
112     assertTrue(files[1] + " should be a Referene", StoreFileInfo.isReference(files[1]));
113     assertEquals(sourceHtd.getTableName(), HFileLink.getReferencedTableName(files[0]));
114     assertEquals(TEST_HFILE, HFileLink.getReferencedHFileName(files[0]));
115     Path refPath = getReferredToFile(files[1]);
116     assertTrue(refPath.getName() + " should be a HFileLink", HFileLink.isHFileLink(refPath.getName()));
117     assertEquals(files[0], refPath.getName());
118   }
119 
120   /**
121    * Execute the restore operation
122    * @param snapshotDir The snapshot directory to use as "restore source"
123    * @param sourceTableName The name of the snapshotted table
124    * @param htdClone The HTableDescriptor of the table to restore/clone.
125    */
126   public void testRestore(final Path snapshotDir, final String sourceTableName,
127       final HTableDescriptor htdClone) throws IOException {
128     LOG.debug("pre-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
129     FSUtils.logFileSystemState(fs, rootDir, LOG);
130 
131     new FSTableDescriptors(conf).createTableDescriptor(htdClone);
132     RestoreSnapshotHelper helper = getRestoreHelper(rootDir, snapshotDir, sourceTableName, htdClone);
133     helper.restoreHdfsRegions();
134 
135     LOG.debug("post-restore table=" + htdClone.getTableName() + " snapshot=" + snapshotDir);
136     FSUtils.logFileSystemState(fs, rootDir, LOG);
137   }
138 
139   /**
140    * Initialize the restore helper, based on the snapshot and table information provided.
141    */
142   private RestoreSnapshotHelper getRestoreHelper(final Path rootDir, final Path snapshotDir,
143       final String sourceTableName, final HTableDescriptor htdClone) throws IOException {
144     CatalogTracker catalogTracker = Mockito.mock(CatalogTracker.class);
145     HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class);
146     ForeignExceptionDispatcher monitor = Mockito.mock(ForeignExceptionDispatcher.class);
147     MonitoredTask status = Mockito.mock(MonitoredTask.class);
148 
149     SnapshotDescription sd = SnapshotDescription.newBuilder()
150       .setName("snapshot")
151       .setTable(sourceTableName)
152       .build();
153 
154     return new RestoreSnapshotHelper(conf, fs, sd, snapshotDir,
155       htdClone, rootDir, monitor, status);
156   }
157 
158   private void createSnapshot(final Path rootDir, final Path snapshotDir, final HTableDescriptor htd)
159       throws IOException {
160     // First region, simple with one plain hfile.
161     HRegionInfo hri = new HRegionInfo(htd.getTableName());
162     HRegionFileSystem r0fs = HRegionFileSystem.createRegionOnFileSystem(conf,
163       fs, FSUtils.getTableDir(archiveDir, hri.getTableName()), hri);
164     Path storeFile = new Path(rootDir, TEST_HFILE);
165     fs.createNewFile(storeFile);
166     r0fs.commitStoreFile(TEST_FAMILY, storeFile);
167 
168     // Second region, used to test the split case.
169     // This region contains a reference to the hfile in the first region.
170     hri = new HRegionInfo(htd.getTableName());
171     HRegionFileSystem r1fs = HRegionFileSystem.createRegionOnFileSystem(conf,
172       fs, FSUtils.getTableDir(archiveDir, hri.getTableName()), hri);
173     storeFile = new Path(rootDir, TEST_HFILE + '.' + r0fs.getRegionInfo().getEncodedName());
174     fs.createNewFile(storeFile);
175     r1fs.commitStoreFile(TEST_FAMILY, storeFile);
176 
177     Path tableDir = FSUtils.getTableDir(archiveDir, htd.getTableName());
178     FileUtil.copy(fs, tableDir, fs, snapshotDir, false, conf);
179   }
180 
181   private HTableDescriptor createTableDescriptor(final String tableName) {
182     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
183     htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
184     return htd;
185   }
186 
187   private Path getReferredToFile(final String referenceName) {
188     Path fakeBasePath = new Path(new Path("table", "region"), "cf");
189     return StoreFileInfo.getReferredToFile(new Path(fakeBasePath, referenceName));
190   }
191 
192   private String[] getHFiles(final Path tableDir) throws IOException {
193     List<String> files = new ArrayList<String>();
194     for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
195       for (Path familyDir: FSUtils.getFamilyDirs(fs, regionDir)) {
196         for (FileStatus file: FSUtils.listStatus(fs, familyDir)) {
197           files.add(file.getPath().getName());
198         }
199       }
200     }
201     Collections.sort(files);
202     return files.toArray(new String[files.size()]);
203   }
204 }