View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.fail;
22  
23  import java.io.IOException;
24  import java.util.HashSet;
25  import java.util.Set;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HColumnDescriptor;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.LargeTests;
36  import org.apache.hadoop.hbase.master.MasterFileSystem;
37  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
38  import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
39  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.apache.hadoop.hbase.util.FSUtils;
42  import org.junit.After;
43  import org.junit.AfterClass;
44  import org.junit.Before;
45  import org.junit.BeforeClass;
46  import org.junit.Test;
47  import org.junit.experimental.categories.Category;
48  
49  /**
50   * Test restore snapshots from the client
51   */
52  @Category(LargeTests.class)
53  public class TestRestoreSnapshotFromClient {
54    final Log LOG = LogFactory.getLog(getClass());
55  
56    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57  
58    private final byte[] FAMILY = Bytes.toBytes("cf");
59  
60    private byte[] emptySnapshot;
61    private byte[] snapshotName0;
62    private byte[] snapshotName1;
63    private byte[] snapshotName2;
64    private int snapshot0Rows;
65    private int snapshot1Rows;
66    private TableName tableName;
67    private HBaseAdmin admin;
68  
69    @BeforeClass
70    public static void setUpBeforeClass() throws Exception {
71      TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
72      TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
73      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
74      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
75      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
76      TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
77      TEST_UTIL.getConfiguration().setBoolean(
78          "hbase.master.enabletable.roundrobin", true);
79      TEST_UTIL.startMiniCluster(3);
80    }
81  
82    @AfterClass
83    public static void tearDownAfterClass() throws Exception {
84      TEST_UTIL.shutdownMiniCluster();
85    }
86  
87    /**
88     * Initialize the tests with a table filled with some data
89     * and two snapshots (snapshotName0, snapshotName1) of different states.
90     * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
91     */
92    @Before
93    public void setup() throws Exception {
94      this.admin = TEST_UTIL.getHBaseAdmin();
95  
96      long tid = System.currentTimeMillis();
97      tableName =
98          TableName.valueOf("testtb-" + tid);
99      emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
100     snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
101     snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
102     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
103 
104     // create Table and disable it
105     SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
106     admin.disableTable(tableName);
107 
108     // take an empty snapshot
109     admin.snapshot(emptySnapshot, tableName);
110 
111     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
112     // enable table and insert data
113     admin.enableTable(tableName);
114     SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
115     snapshot0Rows = TEST_UTIL.countRows(table);
116     admin.disableTable(tableName);
117 
118     // take a snapshot
119     admin.snapshot(snapshotName0, tableName);
120 
121     // enable table and insert more data
122     admin.enableTable(tableName);
123     SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
124     snapshot1Rows = TEST_UTIL.countRows(table);
125     admin.disableTable(tableName);
126 
127     // take a snapshot of the updated table
128     admin.snapshot(snapshotName1, tableName);
129 
130     // re-enable table
131     admin.enableTable(tableName);
132     table.close();
133   }
134 
135   @After
136   public void tearDown() throws Exception {
137     TEST_UTIL.deleteTable(tableName);
138     SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
139     SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
140   }
141 
142   @Test
143   public void testRestoreSnapshot() throws IOException {
144     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
145 
146     // Restore from snapshot-0
147     admin.disableTable(tableName);
148     admin.restoreSnapshot(snapshotName0);
149     admin.enableTable(tableName);
150     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
151 
152     // Restore from emptySnapshot
153     admin.disableTable(tableName);
154     admin.restoreSnapshot(emptySnapshot);
155     admin.enableTable(tableName);
156     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, 0);
157 
158     // Restore from snapshot-1
159     admin.disableTable(tableName);
160     admin.restoreSnapshot(snapshotName1);
161     admin.enableTable(tableName);
162     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
163 
164     // Restore from snapshot-1
165     TEST_UTIL.deleteTable(tableName);
166     admin.restoreSnapshot(snapshotName1);
167     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
168   }
169 
170   @Test
171   public void testRestoreSchemaChange() throws Exception {
172     byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
173 
174     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
175 
176     // Add one column family and put some data in it
177     admin.disableTable(tableName);
178     admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2));
179     admin.enableTable(tableName);
180     assertEquals(2, table.getTableDescriptor().getFamilies().size());
181     HTableDescriptor htd = admin.getTableDescriptor(tableName);
182     assertEquals(2, htd.getFamilies().size());
183     SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, TEST_FAMILY2);
184     long snapshot2Rows = snapshot1Rows + 500;
185     assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
186     assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
187     Set<String> fsFamilies = getFamiliesFromFS(tableName);
188     assertEquals(2, fsFamilies.size());
189     table.close();
190 
191     // Take a snapshot
192     admin.disableTable(tableName);
193     admin.snapshot(snapshotName2, tableName);
194 
195     // Restore the snapshot (without the cf)
196     admin.restoreSnapshot(snapshotName0);
197     admin.enableTable(tableName);
198     assertEquals(1, table.getTableDescriptor().getFamilies().size());
199     try {
200       TEST_UTIL.countRows(table, TEST_FAMILY2);
201       fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
202     } catch (NoSuchColumnFamilyException e) {
203       // expected
204     }
205     assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
206     htd = admin.getTableDescriptor(tableName);
207     assertEquals(1, htd.getFamilies().size());
208     fsFamilies = getFamiliesFromFS(tableName);
209     assertEquals(1, fsFamilies.size());
210     table.close();
211 
212     // Restore back the snapshot (with the cf)
213     admin.disableTable(tableName);
214     admin.restoreSnapshot(snapshotName2);
215     admin.enableTable(tableName);
216     htd = admin.getTableDescriptor(tableName);
217     assertEquals(2, htd.getFamilies().size());
218     assertEquals(2, table.getTableDescriptor().getFamilies().size());
219     assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
220     assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
221     fsFamilies = getFamiliesFromFS(tableName);
222     assertEquals(2, fsFamilies.size());
223     table.close();
224   }
225 
226   @Test
227   public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
228     TableName clonedTableName =
229         TableName.valueOf("clonedtb-" + System.currentTimeMillis());
230     admin.cloneSnapshot(snapshotName0, clonedTableName);
231     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
232     admin.disableTable(clonedTableName);
233     admin.snapshot(snapshotName2, clonedTableName);
234     TEST_UTIL.deleteTable(clonedTableName);
235     waitCleanerRun();
236 
237     admin.cloneSnapshot(snapshotName2, clonedTableName);
238     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
239     TEST_UTIL.deleteTable(clonedTableName);
240   }
241 
242   @Test
243   public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
244     TEST_UTIL.deleteTable(tableName);
245     waitCleanerRun();
246 
247     admin.cloneSnapshot(snapshotName0, tableName);
248     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
249     waitCleanerRun();
250 
251     admin.disableTable(tableName);
252     admin.restoreSnapshot(snapshotName0);
253     admin.enableTable(tableName);
254     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
255   }
256 
257   // ==========================================================================
258   //  Helpers
259   // ==========================================================================
260   private void waitCleanerRun() throws InterruptedException {
261     TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
262   }
263 
264   private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
265     MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
266     Set<String> families = new HashSet<String>();
267     Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
268     for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
269       for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
270         families.add(familyDir.getName());
271       }
272     }
273     return families;
274   }
275 }