View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.HashSet;
26  import java.util.Set;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.TableName;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HColumnDescriptor;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.testclassification.LargeTests;
37  import org.apache.hadoop.hbase.master.MasterFileSystem;
38  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
39  import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
40  import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
41  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  import org.junit.After;
45  import org.junit.AfterClass;
46  import org.junit.Before;
47  import org.junit.BeforeClass;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  /**
52   * Test restore snapshots from the client
53   */
54  @Category(LargeTests.class)
55  public class TestRestoreSnapshotFromClient {
56    final Log LOG = LogFactory.getLog(getClass());
57  
58    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59  
60    private final byte[] FAMILY = Bytes.toBytes("cf");
61  
62    private byte[] emptySnapshot;
63    private byte[] snapshotName0;
64    private byte[] snapshotName1;
65    private byte[] snapshotName2;
66    private int snapshot0Rows;
67    private int snapshot1Rows;
68    private TableName tableName;
69    private Admin admin;
70  
71    @BeforeClass
72    public static void setUpBeforeClass() throws Exception {
73      TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
74      TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
75      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
76      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
77      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
78      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
79      TEST_UTIL.getConfiguration().setBoolean(
80          "hbase.master.enabletable.roundrobin", true);
81      TEST_UTIL.startMiniCluster(3);
82    }
83  
84    @AfterClass
85    public static void tearDownAfterClass() throws Exception {
86      TEST_UTIL.shutdownMiniCluster();
87    }
88  
89    /**
90     * Initialize the tests with a table filled with some data
91     * and two snapshots (snapshotName0, snapshotName1) of different states.
92     * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
93     */
94    @Before
95    public void setup() throws Exception {
96      this.admin = TEST_UTIL.getHBaseAdmin();
97  
98      long tid = System.currentTimeMillis();
99      tableName =
100         TableName.valueOf("testtb-" + tid);
101     emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
102     snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
103     snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
104     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
105 
106     // create Table and disable it
107     SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
108     admin.disableTable(tableName);
109 
110     // take an empty snapshot
111     admin.snapshot(emptySnapshot, tableName);
112 
113     // enable table and insert data
114     admin.enableTable(tableName);
115     SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
116     try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
117       snapshot0Rows = TEST_UTIL.countRows(table);
118     }
119     admin.disableTable(tableName);
120 
121     // take a snapshot
122     admin.snapshot(snapshotName0, tableName);
123 
124     // enable table and insert more data
125     admin.enableTable(tableName);
126     SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
127     try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
128       snapshot1Rows = TEST_UTIL.countRows(table);
129     }
130   }
131 
132   @After
133   public void tearDown() throws Exception {
134     TEST_UTIL.deleteTable(tableName);
135     SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
136     SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
137   }
138 
139   @Test
140   public void testRestoreSnapshot() throws IOException {
141     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
142     admin.disableTable(tableName);
143     admin.snapshot(snapshotName1, tableName);
144     // Restore from snapshot-0
145     admin.restoreSnapshot(snapshotName0);
146     admin.enableTable(tableName);
147     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
148     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
149 
150     // Restore from emptySnapshot
151     admin.disableTable(tableName);
152     admin.restoreSnapshot(emptySnapshot);
153     admin.enableTable(tableName);
154     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, 0);
155     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
156 
157     // Restore from snapshot-1
158     admin.disableTable(tableName);
159     admin.restoreSnapshot(snapshotName1);
160     admin.enableTable(tableName);
161     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
162     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
163 
164     // Restore from snapshot-1
165     TEST_UTIL.deleteTable(tableName);
166     admin.restoreSnapshot(snapshotName1);
167     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
168     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
169   }
170 
171   protected int getNumReplicas() {
172     return 1;
173   }
174 
175   @Test
176   public void testRestoreSchemaChange() throws Exception {
177     byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
178 
179     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
180 
181     // Add one column family and put some data in it
182     admin.disableTable(tableName);
183     admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2));
184     admin.enableTable(tableName);
185     assertEquals(2, table.getTableDescriptor().getFamilies().size());
186     HTableDescriptor htd = admin.getTableDescriptor(tableName);
187     assertEquals(2, htd.getFamilies().size());
188     SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, TEST_FAMILY2);
189     long snapshot2Rows = snapshot1Rows + 500;
190     assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
191     assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
192     Set<String> fsFamilies = getFamiliesFromFS(tableName);
193     assertEquals(2, fsFamilies.size());
194 
195     // Take a snapshot
196     admin.disableTable(tableName);
197     admin.snapshot(snapshotName2, tableName);
198 
199     // Restore the snapshot (without the cf)
200     admin.restoreSnapshot(snapshotName0);
201     admin.enableTable(tableName);
202     assertEquals(1, table.getTableDescriptor().getFamilies().size());
203     try {
204       TEST_UTIL.countRows(table, TEST_FAMILY2);
205       fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
206     } catch (NoSuchColumnFamilyException e) {
207       // expected
208     }
209     assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
210     htd = admin.getTableDescriptor(tableName);
211     assertEquals(1, htd.getFamilies().size());
212     fsFamilies = getFamiliesFromFS(tableName);
213     assertEquals(1, fsFamilies.size());
214 
215     // Restore back the snapshot (with the cf)
216     admin.disableTable(tableName);
217     admin.restoreSnapshot(snapshotName2);
218     admin.enableTable(tableName);
219     htd = admin.getTableDescriptor(tableName);
220     assertEquals(2, htd.getFamilies().size());
221     assertEquals(2, table.getTableDescriptor().getFamilies().size());
222     assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
223     assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
224     fsFamilies = getFamiliesFromFS(tableName);
225     assertEquals(2, fsFamilies.size());
226     table.close();
227   }
228 
229   @Test
230   public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
231     TableName clonedTableName =
232         TableName.valueOf("clonedtb-" + System.currentTimeMillis());
233     admin.cloneSnapshot(snapshotName0, clonedTableName);
234     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
235     SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
236     admin.disableTable(clonedTableName);
237     admin.snapshot(snapshotName2, clonedTableName);
238     TEST_UTIL.deleteTable(clonedTableName);
239     waitCleanerRun();
240 
241     admin.cloneSnapshot(snapshotName2, clonedTableName);
242     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
243     SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
244     TEST_UTIL.deleteTable(clonedTableName);
245   }
246 
247   @Test
248   public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
249     TEST_UTIL.deleteTable(tableName);
250     waitCleanerRun();
251 
252     admin.cloneSnapshot(snapshotName0, tableName);
253     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
254     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
255     waitCleanerRun();
256 
257     admin.disableTable(tableName);
258     admin.restoreSnapshot(snapshotName0);
259     admin.enableTable(tableName);
260     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
261     SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
262   }
263 
264   @Test
265   public void testCorruptedSnapshot() throws IOException, InterruptedException {
266     SnapshotTestingUtils.corruptSnapshot(TEST_UTIL, Bytes.toString(snapshotName0));
267     TableName cloneName = TableName.valueOf("corruptedClone-" + System.currentTimeMillis());
268     try {
269       admin.cloneSnapshot(snapshotName0, cloneName);
270       fail("Expected CorruptedSnapshotException, got succeeded cloneSnapshot()");
271     } catch (CorruptedSnapshotException e) {
272       // Got the expected corruption exception.
273       // check for no references of the cloned table.
274       assertFalse(admin.tableExists(cloneName));
275     } catch (Exception e) {
276       fail("Expected CorruptedSnapshotException got: " + e);
277     }
278   }
279 
280   // ==========================================================================
281   //  Helpers
282   // ==========================================================================
283   private void waitCleanerRun() throws InterruptedException {
284     TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
285   }
286 
287   private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
288     MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
289     Set<String> families = new HashSet<String>();
290     Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
291     for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
292       for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
293         families.add(familyDir.getName());
294       }
295     }
296     return families;
297   }
298 }