View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  import static org.junit.Assert.fail;
24  
25  import java.util.ArrayList;
26  import java.util.List;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.testclassification.LargeTests;
38  import org.apache.hadoop.hbase.TableNotFoundException;
39  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
40  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
41  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
42  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
43  import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
44  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
45  import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.FSUtils;
48  import org.junit.After;
49  import org.junit.AfterClass;
50  import org.junit.Before;
51  import org.junit.BeforeClass;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  import com.google.common.collect.Lists;
56  
57  /**
58   * Test create/using/deleting snapshots from the client
59   * <p>
60   * This is an end-to-end test for the snapshot utility
61   */
62  @Category(LargeTests.class)
63  public class TestSnapshotFromClient {
64    private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
65    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
66    private static final int NUM_RS = 2;
67    private static final String STRING_TABLE_NAME = "test";
68    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
69    private static final TableName TABLE_NAME =
70        TableName.valueOf(STRING_TABLE_NAME);
71  
72    /**
73     * Setup the config for the cluster
74     * @throws Exception on failure
75     */
76    @BeforeClass
77    public static void setupCluster() throws Exception {
78      setupConf(UTIL.getConfiguration());
79      UTIL.startMiniCluster(NUM_RS);
80    }
81  
82    private static void setupConf(Configuration conf) {
83      // disable the ui
84      conf.setInt("hbase.regionsever.info.port", -1);
85      // change the flush size to a small amount, regulating number of store files
86      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
87      // so make sure we get a compaction when doing a load, but keep around some
88      // files in the store
89      conf.setInt("hbase.hstore.compaction.min", 10);
90      conf.setInt("hbase.hstore.compactionThreshold", 10);
91      // block writes if we get to 12 store files
92      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
93      // Enable snapshot
94      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
95      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
96        ConstantSizeRegionSplitPolicy.class.getName());
97    }
98  
99    @Before
100   public void setup() throws Exception {
101     UTIL.createTable(TABLE_NAME, TEST_FAM);
102   }
103 
104   @After
105   public void tearDown() throws Exception {
106     UTIL.deleteTable(TABLE_NAME);
107     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
108     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
109   }
110 
111   @AfterClass
112   public static void cleanupTest() throws Exception {
113     try {
114       UTIL.shutdownMiniCluster();
115     } catch (Exception e) {
116       LOG.warn("failure shutting down cluster", e);
117     }
118   }
119 
120   /**
121    * Test snapshotting not allowed hbase:meta and -ROOT-
122    * @throws Exception
123    */
124   @Test (timeout=300000)
125   public void testMetaTablesSnapshot() throws Exception {
126     HBaseAdmin admin = UTIL.getHBaseAdmin();
127     byte[] snapshotName = Bytes.toBytes("metaSnapshot");
128 
129     try {
130       admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
131       fail("taking a snapshot of hbase:meta should not be allowed");
132     } catch (IllegalArgumentException e) {
133       // expected
134     }
135   }
136 
137   /**
138    * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
139    *
140    * @throws Exception
141    */
142   @Test (timeout=300000)
143   public void testSnapshotDeletionWithRegex() throws Exception {
144     HBaseAdmin admin = UTIL.getHBaseAdmin();
145     // make sure we don't fail on listing snapshots
146     SnapshotTestingUtils.assertNoSnapshots(admin);
147 
148     // put some stuff in the table
149     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
150     UTIL.loadTable(table, TEST_FAM);
151     table.close();
152 
153     byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
154     admin.snapshot(snapshot1, TABLE_NAME);
155     LOG.debug("Snapshot1 completed.");
156 
157     byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
158     admin.snapshot(snapshot2, TABLE_NAME);
159     LOG.debug("Snapshot2 completed.");
160 
161     String snapshot3 = "3rdTableSnapshot";
162     admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
163     LOG.debug(snapshot3 + " completed.");
164 
165     // delete the first two snapshots
166     admin.deleteSnapshots("TableSnapshot.*");
167     List<SnapshotDescription> snapshots = admin.listSnapshots();
168     assertEquals(1, snapshots.size());
169     assertEquals(snapshots.get(0).getName(), snapshot3);
170 
171     admin.deleteSnapshot(snapshot3);
172     admin.close();
173   }
174   /**
175    * Test snapshotting a table that is offline
176    * @throws Exception
177    */
178   @Test (timeout=300000)
179   public void testOfflineTableSnapshot() throws Exception {
180     HBaseAdmin admin = UTIL.getHBaseAdmin();
181     // make sure we don't fail on listing snapshots
182     SnapshotTestingUtils.assertNoSnapshots(admin);
183 
184     // put some stuff in the table
185     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
186     UTIL.loadTable(table, TEST_FAM, false);
187 
188     LOG.debug("FS state before disable:");
189     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
190       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
191     // XXX if this is flakey, might want to consider using the async version and looping as
192     // disableTable can succeed and still timeout.
193     admin.disableTable(TABLE_NAME);
194 
195     LOG.debug("FS state before snapshot:");
196     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
197       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
198 
199     // take a snapshot of the disabled table
200     final String SNAPSHOT_NAME = "offlineTableSnapshot";
201     byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
202 
203     SnapshotDescription desc = SnapshotDescription.newBuilder()
204       .setType(SnapshotDescription.Type.DISABLED)
205       .setTable(STRING_TABLE_NAME)
206       .setName(SNAPSHOT_NAME)
207       .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
208       .build();
209     admin.snapshot(desc);
210     LOG.debug("Snapshot completed.");
211 
212     // make sure we have the snapshot
213     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
214       snapshot, TABLE_NAME);
215 
216     // make sure its a valid snapshot
217     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
218     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
219     LOG.debug("FS state after snapshot:");
220     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
221       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
222 
223     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
224       admin, fs);
225 
226     admin.deleteSnapshot(snapshot);
227     snapshots = admin.listSnapshots();
228     SnapshotTestingUtils.assertNoSnapshots(admin);
229   }
230 
231   @Test (timeout=300000)
232   public void testSnapshotFailsOnNonExistantTable() throws Exception {
233     HBaseAdmin admin = UTIL.getHBaseAdmin();
234     // make sure we don't fail on listing snapshots
235     SnapshotTestingUtils.assertNoSnapshots(admin);
236     String tableName = "_not_a_table";
237 
238     // make sure the table doesn't exist
239     boolean fail = false;
240     do {
241     try {
242       admin.getTableDescriptor(Bytes.toBytes(tableName));
243       fail = true;
244           LOG.error("Table:" + tableName + " already exists, checking a new name");
245       tableName = tableName+"!";
246     } catch (TableNotFoundException e) {
247       fail = false;
248       }
249     } while (fail);
250 
251     // snapshot the non-existant table
252     try {
253       admin.snapshot("fail", tableName);
254       fail("Snapshot succeeded even though there is not table.");
255     } catch (SnapshotCreationException e) {
256       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
257     }
258   }
259 
260   @Test (timeout=300000)
261   public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
262     // test with an empty table with one region
263 
264     HBaseAdmin admin = UTIL.getHBaseAdmin();
265     // make sure we don't fail on listing snapshots
266     SnapshotTestingUtils.assertNoSnapshots(admin);
267 
268     LOG.debug("FS state before disable:");
269     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
270       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
271     admin.disableTable(TABLE_NAME);
272 
273     LOG.debug("FS state before snapshot:");
274     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
275       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
276 
277     // take a snapshot of the disabled table
278     byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
279     admin.snapshot(snapshot, TABLE_NAME);
280     LOG.debug("Snapshot completed.");
281 
282     // make sure we have the snapshot
283     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
284       snapshot, TABLE_NAME);
285 
286     // make sure its a valid snapshot
287     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
288     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
289     LOG.debug("FS state after snapshot:");
290     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
291       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
292 
293     List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
294     List<byte[]> nonEmptyCfs = Lists.newArrayList();
295     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs,
296       rootDir, admin, fs);
297 
298     admin.deleteSnapshot(snapshot);
299     snapshots = admin.listSnapshots();
300     SnapshotTestingUtils.assertNoSnapshots(admin);
301   }
302 
303   @Test(timeout = 300000)
304   public void testListTableSnapshots() throws Exception {
305     HBaseAdmin admin = null;
306     TableName tableName2 = TableName.valueOf("testListTableSnapshots");
307     try {
308       admin = UTIL.getHBaseAdmin();
309 
310       HTableDescriptor htd = new HTableDescriptor(tableName2);
311       UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
312 
313       String table1Snapshot1 = "Table1Snapshot1";
314       admin.snapshot(table1Snapshot1, TABLE_NAME);
315       LOG.debug("Snapshot1 completed.");
316 
317       String table1Snapshot2 = "Table1Snapshot2";
318       admin.snapshot(table1Snapshot2, TABLE_NAME);
319       LOG.debug("Snapshot2 completed.");
320 
321       String table2Snapshot1 = "Table2Snapshot1";
322       admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName2);
323       LOG.debug(table2Snapshot1 + " completed.");
324 
325       List<SnapshotDescription> listTableSnapshots = admin.listTableSnapshots("test.*", ".*");
326       List<String> listTableSnapshotNames = new ArrayList<String>();
327       assertEquals(3, listTableSnapshots.size());
328       for (SnapshotDescription s : listTableSnapshots) {
329         listTableSnapshotNames.add(s.getName());
330       }
331       assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
332       assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
333       assertTrue(listTableSnapshotNames.contains(table2Snapshot1));
334     } finally {
335       if (admin != null) {
336         try {
337           admin.deleteSnapshots("Table.*");
338         } catch (SnapshotDoesNotExistException ignore) {
339         }
340         if (admin.tableExists(tableName2)) {
341           UTIL.deleteTable(tableName2);
342         }
343         admin.close();
344       }
345     }
346   }
347 
348   @Test(timeout = 300000)
349   public void testListTableSnapshotsWithRegex() throws Exception {
350     HBaseAdmin admin = null;
351     try {
352       admin = UTIL.getHBaseAdmin();
353 
354       String table1Snapshot1 = "Table1Snapshot1";
355       admin.snapshot(table1Snapshot1, TABLE_NAME);
356       LOG.debug("Snapshot1 completed.");
357 
358       String table1Snapshot2 = "Table1Snapshot2";
359       admin.snapshot(table1Snapshot2, TABLE_NAME);
360       LOG.debug("Snapshot2 completed.");
361 
362       String table2Snapshot1 = "Table2Snapshot1";
363       admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
364       LOG.debug(table2Snapshot1 + " completed.");
365 
366       List<SnapshotDescription> listTableSnapshots = admin.listTableSnapshots("test.*", "Table1.*");
367       List<String> listTableSnapshotNames = new ArrayList<String>();
368       assertEquals(2, listTableSnapshots.size());
369       for (SnapshotDescription s : listTableSnapshots) {
370         listTableSnapshotNames.add(s.getName());
371       }
372       assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
373       assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
374       assertFalse(listTableSnapshotNames.contains(table2Snapshot1));
375     } finally {
376       if (admin != null) {
377         try {
378           admin.deleteSnapshots("Table.*");
379         } catch (SnapshotDoesNotExistException ignore) {
380         }
381         admin.close();
382       }
383     }
384   }
385 
386   @Test(timeout = 300000)
387   public void testDeleteTableSnapshots() throws Exception {
388     HBaseAdmin admin = null;
389     TableName tableName2 = TableName.valueOf("testListTableSnapshots");
390     try {
391       admin = UTIL.getHBaseAdmin();
392 
393       HTableDescriptor htd = new HTableDescriptor(tableName2);
394       UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
395 
396       String table1Snapshot1 = "Table1Snapshot1";
397       admin.snapshot(table1Snapshot1, TABLE_NAME);
398       LOG.debug("Snapshot1 completed.");
399 
400       String table1Snapshot2 = "Table1Snapshot2";
401       admin.snapshot(table1Snapshot2, TABLE_NAME);
402       LOG.debug("Snapshot2 completed.");
403 
404       String table2Snapshot1 = "Table2Snapshot1";
405       admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName2);
406       LOG.debug(table2Snapshot1 + " completed.");
407 
408       admin.deleteTableSnapshots("test.*", ".*");
409       assertEquals(0, admin.listTableSnapshots("test.*", ".*").size());
410     } finally {
411       if (admin != null) {
412         if (admin.tableExists(tableName2)) {
413           UTIL.deleteTable(tableName2);
414         }
415         admin.close();
416       }
417     }
418   }
419 
420   @Test(timeout = 300000)
421   public void testDeleteTableSnapshotsWithRegex() throws Exception {
422     HBaseAdmin admin = null;
423     try {
424       admin = UTIL.getHBaseAdmin();
425 
426       String table1Snapshot1 = "Table1Snapshot1";
427       admin.snapshot(table1Snapshot1, TABLE_NAME);
428       LOG.debug("Snapshot1 completed.");
429 
430       String table1Snapshot2 = "Table1Snapshot2";
431       admin.snapshot(table1Snapshot2, TABLE_NAME);
432       LOG.debug("Snapshot2 completed.");
433 
434       String table2Snapshot1 = "Table2Snapshot1";
435       admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
436       LOG.debug(table2Snapshot1 + " completed.");
437 
438       admin.deleteTableSnapshots("test.*", "Table1.*");
439       assertEquals(1, admin.listTableSnapshots("test.*", ".*").size());
440     } finally {
441       if (admin != null) {
442         try {
443           admin.deleteTableSnapshots("test.*", ".*");
444         } catch (SnapshotDoesNotExistException ignore) {
445         }
446         admin.close();
447       }
448     }
449   }
450 }