1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.fail;
22  
23  import java.io.IOException;
24  import java.util.HashSet;
25  import java.util.Set;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.LargeTests;
35  import org.apache.hadoop.hbase.master.MasterFileSystem;
36  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
37  import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
38  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.apache.hadoop.hbase.util.FSUtils;
41  import org.junit.After;
42  import org.junit.AfterClass;
43  import org.junit.Before;
44  import org.junit.BeforeClass;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  
48  /**
49   * Test clone/restore snapshots from the client
50   */
51  @Category(LargeTests.class)
52  public class TestRestoreSnapshotFromClient {
53    final Log LOG = LogFactory.getLog(getClass());
54  
55    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56  
57    private final byte[] FAMILY = Bytes.toBytes("cf");
58  
59    private byte[] emptySnapshot;
60    private byte[] snapshotName0;
61    private byte[] snapshotName1;
62    private byte[] snapshotName2;
63    private int snapshot0Rows;
64    private int snapshot1Rows;
65    private byte[] tableName;
66    private HBaseAdmin admin;
67  
68    @BeforeClass
69    public static void setUpBeforeClass() throws Exception {
70      TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
71      TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
72      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
73      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
74      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
75      TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
76      TEST_UTIL.getConfiguration().setBoolean(
77          "hbase.master.enabletable.roundrobin", true);
78      TEST_UTIL.startMiniCluster(3);
79    }
80  
81    @AfterClass
82    public static void tearDownAfterClass() throws Exception {
83      TEST_UTIL.shutdownMiniCluster();
84    }
85  
86    /**
87     * Initialize the tests with a table filled with some data
88     * and two snapshots (snapshotName0, snapshotName1) of different states.
89     * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
90     */
91    @Before
92    public void setup() throws Exception {
93      this.admin = TEST_UTIL.getHBaseAdmin();
94  
95      long tid = System.currentTimeMillis();
96      tableName = Bytes.toBytes("testtb-" + tid);
97      emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
98      snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
99      snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
100     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
101 
102     // create Table and disable it
103     SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
104     admin.disableTable(tableName);
105 
106     // take an empty snapshot
107     admin.snapshot(emptySnapshot, tableName);
108 
109     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
110     try {
111       // enable table and insert data
112       admin.enableTable(tableName);
113       SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
114       snapshot0Rows = TEST_UTIL.countRows(table);
115       admin.disableTable(tableName);
116 
117       // take a snapshot
118       admin.snapshot(snapshotName0, tableName);
119 
120       // enable table and insert more data
121       admin.enableTable(tableName);
122       SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
123       snapshot1Rows = TEST_UTIL.countRows(table);
124       admin.disableTable(tableName);
125 
126       // take a snapshot of the updated table
127       admin.snapshot(snapshotName1, tableName);
128 
129       // re-enable table
130       admin.enableTable(tableName);
131     } finally {
132       table.close();
133     }
134   }
135 
136   @After
137   public void tearDown() throws Exception {
138     TEST_UTIL.deleteTable(tableName);
139     SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
140     SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
141   }
142 
143   @Test
144   public void testRestoreSnapshot() throws IOException {
145     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
146 
147     // Restore from snapshot-0
148     admin.disableTable(tableName);
149     admin.restoreSnapshot(snapshotName0);
150     admin.enableTable(tableName);
151     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
152 
153     // Restore from emptySnapshot
154     admin.disableTable(tableName);
155     admin.restoreSnapshot(emptySnapshot);
156     admin.enableTable(tableName);
157     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, 0);
158 
159     // Restore from snapshot-1
160     admin.disableTable(tableName);
161     admin.restoreSnapshot(snapshotName1);
162     admin.enableTable(tableName);
163     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
164   }
165 
166   @Test
167   public void testRestoreSchemaChange() throws Exception {
168     byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
169 
170     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
171 
172     // Add one column family and put some data in it
173     admin.disableTable(tableName);
174     admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2));
175     admin.enableTable(tableName);
176     assertEquals(2, table.getTableDescriptor().getFamilies().size());
177     HTableDescriptor htd = admin.getTableDescriptor(tableName);
178     assertEquals(2, htd.getFamilies().size());
179     SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, TEST_FAMILY2);
180     long snapshot2Rows = snapshot1Rows + 500;
181     assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
182     assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
183     Set<String> fsFamilies = getFamiliesFromFS(tableName);
184     assertEquals(2, fsFamilies.size());
185     table.close();
186 
187     // Take a snapshot
188     admin.disableTable(tableName);
189     admin.snapshot(snapshotName2, tableName);
190 
191     // Restore the snapshot (without the cf)
192     admin.restoreSnapshot(snapshotName0);
193     assertEquals(1, table.getTableDescriptor().getFamilies().size());
194     admin.enableTable(tableName);
195     try {
196       TEST_UTIL.countRows(table, TEST_FAMILY2);
197       fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
198     } catch (NoSuchColumnFamilyException e) {
199       // expected
200     }
201     assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
202     htd = admin.getTableDescriptor(tableName);
203     assertEquals(1, htd.getFamilies().size());
204     fsFamilies = getFamiliesFromFS(tableName);
205     assertEquals(1, fsFamilies.size());
206     table.close();
207 
208     // Restore back the snapshot (with the cf)
209     admin.disableTable(tableName);
210     admin.restoreSnapshot(snapshotName2);
211     admin.enableTable(tableName);
212     htd = admin.getTableDescriptor(tableName);
213     assertEquals(2, htd.getFamilies().size());
214     assertEquals(2, table.getTableDescriptor().getFamilies().size());
215     assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
216     assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
217     fsFamilies = getFamiliesFromFS(tableName);
218     assertEquals(2, fsFamilies.size());
219     table.close();
220   }
221 
222   @Test
223   public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
224     byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
225     admin.cloneSnapshot(snapshotName0, clonedTableName);
226     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
227     admin.disableTable(clonedTableName);
228     admin.snapshot(snapshotName2, clonedTableName);
229     admin.deleteTable(clonedTableName);
230     waitCleanerRun();
231 
232     admin.cloneSnapshot(snapshotName2, clonedTableName);
233     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
234     TEST_UTIL.deleteTable(clonedTableName);
235   }
236 
237   @Test
238   public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
239     TEST_UTIL.deleteTable(tableName);
240     waitCleanerRun();
241 
242     admin.cloneSnapshot(snapshotName0, tableName);
243     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
244     waitCleanerRun();
245 
246     admin.disableTable(tableName);
247     admin.restoreSnapshot(snapshotName0);
248     admin.enableTable(tableName);
249     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
250   }
251 
252   // ==========================================================================
253   //  Helpers
254   // ==========================================================================
255   private void waitCleanerRun() throws InterruptedException {
256     TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
257   }
258 
259   private Set<String> getFamiliesFromFS(final byte[] tableName) throws IOException {
260     MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
261     Set<String> families = new HashSet<String>();
262     Path tableDir = HTableDescriptor.getTableDir(mfs.getRootDir(), tableName);
263     for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
264       for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
265         families.add(familyDir.getName());
266       }
267     }
268     return families;
269   }
270 }