1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.HashSet;
26  import java.util.Set;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HColumnDescriptor;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.LargeTests;
35  import org.apache.hadoop.hbase.master.MasterFileSystem;
36  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
37  import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
38  import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
39  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.apache.hadoop.hbase.util.FSUtils;
42  import org.junit.After;
43  import org.junit.AfterClass;
44  import org.junit.Before;
45  import org.junit.BeforeClass;
46  import org.junit.Test;
47  import org.junit.experimental.categories.Category;
48  
49  /**
50   * Test clone/restore snapshots from the client
51   */
52  @Category(LargeTests.class)
53  public class TestRestoreSnapshotFromClient {
54    final Log LOG = LogFactory.getLog(getClass());
55  
56    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57  
58    private final byte[] FAMILY = Bytes.toBytes("cf");
59  
60    private byte[] emptySnapshot;
61    private byte[] snapshotName0;
62    private byte[] snapshotName1;
63    private byte[] snapshotName2;
64    private int snapshot0Rows;
65    private int snapshot1Rows;
66    private byte[] tableName;
67    private HBaseAdmin admin;
68  
69    @BeforeClass
70    public static void setUpBeforeClass() throws Exception {
71      TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
72      TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
73      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
74      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
75      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
76      TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
77      TEST_UTIL.getConfiguration().setBoolean(
78          "hbase.master.enabletable.roundrobin", true);
79      TEST_UTIL.startMiniCluster(3);
80    }
81  
82    @AfterClass
83    public static void tearDownAfterClass() throws Exception {
84      TEST_UTIL.shutdownMiniCluster();
85    }
86  
87    /**
88     * Initialize the tests with a table filled with some data
89     * and two snapshots (snapshotName0, snapshotName1) of different states.
90     * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
91     */
92    @Before
93    public void setup() throws Exception {
94      this.admin = TEST_UTIL.getHBaseAdmin();
95  
96      long tid = System.currentTimeMillis();
97      tableName = Bytes.toBytes("testtb-" + tid);
98      emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
99      snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
100     snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
101     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
102 
103     // create Table and disable it
104     SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
105     admin.disableTable(tableName);
106 
107     // take an empty snapshot
108     admin.snapshot(emptySnapshot, tableName);
109 
110     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
111     // enable table and insert data
112     admin.enableTable(tableName);
113     SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
114     snapshot0Rows = TEST_UTIL.countRows(table);
115     admin.disableTable(tableName);
116 
117     // take a snapshot
118     admin.snapshot(snapshotName0, tableName);
119 
120     // enable table and insert more data
121     admin.enableTable(tableName);
122     SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
123     snapshot1Rows = TEST_UTIL.countRows(table);
124     admin.disableTable(tableName);
125 
126     // take a snapshot of the updated table
127     admin.snapshot(snapshotName1, tableName);
128 
129     // re-enable table
130     admin.enableTable(tableName);
131     table.close();
132   }
133 
134   @After
135   public void tearDown() throws Exception {
136     TEST_UTIL.deleteTable(tableName);
137     SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
138     SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
139   }
140 
141   @Test
142   public void testRestoreSnapshot() throws IOException {
143     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
144 
145     // Restore from snapshot-0
146     admin.disableTable(tableName);
147     admin.restoreSnapshot(snapshotName0);
148     admin.enableTable(tableName);
149     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
150 
151     // Restore from emptySnapshot
152     admin.disableTable(tableName);
153     admin.restoreSnapshot(emptySnapshot);
154     admin.enableTable(tableName);
155     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, 0);
156 
157     // Restore from snapshot-1
158     admin.disableTable(tableName);
159     admin.restoreSnapshot(snapshotName1);
160     admin.enableTable(tableName);
161     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
162   }
163 
164   @Test
165   public void testRestoreSchemaChange() throws Exception {
166     byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
167 
168     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
169 
170     // Add one column family and put some data in it
171     admin.disableTable(tableName);
172     admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2));
173     admin.enableTable(tableName);
174     assertEquals(2, table.getTableDescriptor().getFamilies().size());
175     HTableDescriptor htd = admin.getTableDescriptor(tableName);
176     assertEquals(2, htd.getFamilies().size());
177     SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, TEST_FAMILY2);
178     long snapshot2Rows = snapshot1Rows + 500;
179     assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
180     assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
181     Set<String> fsFamilies = getFamiliesFromFS(tableName);
182     assertEquals(2, fsFamilies.size());
183     table.close();
184 
185     // Take a snapshot
186     admin.disableTable(tableName);
187     admin.snapshot(snapshotName2, tableName);
188 
189     // Restore the snapshot (without the cf)
190     admin.restoreSnapshot(snapshotName0);
191     assertEquals(1, table.getTableDescriptor().getFamilies().size());
192     admin.enableTable(tableName);
193     try {
194       TEST_UTIL.countRows(table, TEST_FAMILY2);
195       fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
196     } catch (NoSuchColumnFamilyException e) {
197       // expected
198     }
199     assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
200     htd = admin.getTableDescriptor(tableName);
201     assertEquals(1, htd.getFamilies().size());
202     fsFamilies = getFamiliesFromFS(tableName);
203     assertEquals(1, fsFamilies.size());
204     table.close();
205 
206     // Restore back the snapshot (with the cf)
207     admin.disableTable(tableName);
208     admin.restoreSnapshot(snapshotName2);
209     admin.enableTable(tableName);
210     htd = admin.getTableDescriptor(tableName);
211     assertEquals(2, htd.getFamilies().size());
212     assertEquals(2, table.getTableDescriptor().getFamilies().size());
213     assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
214     assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
215     fsFamilies = getFamiliesFromFS(tableName);
216     assertEquals(2, fsFamilies.size());
217     table.close();
218   }
219 
220   @Test
221   public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
222     byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
223     admin.cloneSnapshot(snapshotName0, clonedTableName);
224     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
225     admin.disableTable(clonedTableName);
226     admin.snapshot(snapshotName2, clonedTableName);
227     admin.deleteTable(clonedTableName);
228     waitCleanerRun();
229 
230     admin.cloneSnapshot(snapshotName2, clonedTableName);
231     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
232     TEST_UTIL.deleteTable(clonedTableName);
233   }
234 
235   @Test
236   public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
237     TEST_UTIL.deleteTable(tableName);
238     waitCleanerRun();
239 
240     admin.cloneSnapshot(snapshotName0, tableName);
241     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
242     waitCleanerRun();
243 
244     admin.disableTable(tableName);
245     admin.restoreSnapshot(snapshotName0);
246     admin.enableTable(tableName);
247     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
248   }
249 
250   @Test
251   public void testCorruptedSnapshot() throws IOException, InterruptedException {
252     SnapshotTestingUtils.corruptSnapshot(TEST_UTIL, Bytes.toString(snapshotName0));
253     byte[] cloneName = Bytes.toBytes("corruptedClone-" + System.currentTimeMillis());
254     try {
255       admin.cloneSnapshot(snapshotName0, cloneName);
256       fail("Expected CorruptedSnapshotException, got succeeded cloneSnapshot()");
257     } catch (CorruptedSnapshotException e) {
258       // Got the expected corruption exception.
259       // check for no references of the cloned table.
260       assertFalse(admin.tableExists(cloneName));
261     } catch (Exception e) {
262       fail("Expected CorruptedSnapshotException got: " + e);
263     }
264   }
265 
266   // ==========================================================================
267   //  Helpers
268   // ==========================================================================
269   private void waitCleanerRun() throws InterruptedException {
270     TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
271   }
272 
273   private Set<String> getFamiliesFromFS(final byte[] tableName) throws IOException {
274     MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
275     Set<String> families = new HashSet<String>();
276     Path tableDir = HTableDescriptor.getTableDir(mfs.getRootDir(), tableName);
277     for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
278       for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
279         families.add(familyDir.getName());
280       }
281     }
282     return families;
283   }
284 }