View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.fail;
22  
23  import java.util.HashSet;
24  import java.util.List;
25  import java.util.Set;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.TableName;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.LargeTests;
36  import org.apache.hadoop.hbase.TableNotFoundException;
37  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
38  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
39  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
40  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
41  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
45  import org.junit.After;
46  import org.junit.AfterClass;
47  import org.junit.Before;
48  import org.junit.BeforeClass;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  /**
53   * Test create/using/deleting snapshots from the client
54   * <p>
55   * This is an end-to-end test for the snapshot utility
56   */
57  @Category(LargeTests.class)
58  public class TestSnapshotFromClient {
59    private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
60    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
61    private static final int NUM_RS = 2;
62    private static final String STRING_TABLE_NAME = "test";
63    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
64    private static final TableName TABLE_NAME =
65        TableName.valueOf(STRING_TABLE_NAME);
66  
67    /**
68     * Setup the config for the cluster
69     * @throws Exception on failure
70     */
71    @BeforeClass
72    public static void setupCluster() throws Exception {
73      setupConf(UTIL.getConfiguration());
74      UTIL.startMiniCluster(NUM_RS);
75    }
76  
77    private static void setupConf(Configuration conf) {
78      // disable the ui
79      conf.setInt("hbase.regionsever.info.port", -1);
80      // change the flush size to a small amount, regulating number of store files
81      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
82      // so make sure we get a compaction when doing a load, but keep around some
83      // files in the store
84      conf.setInt("hbase.hstore.compaction.min", 10);
85      conf.setInt("hbase.hstore.compactionThreshold", 10);
86      // block writes if we get to 12 store files
87      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
88      // Enable snapshot
89      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
90      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
91        ConstantSizeRegionSplitPolicy.class.getName());
92    }
93  
94    @Before
95    public void setup() throws Exception {
96      UTIL.createTable(TABLE_NAME, TEST_FAM);
97    }
98  
99    @After
100   public void tearDown() throws Exception {
101     UTIL.deleteTable(TABLE_NAME);
102     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
103     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
104   }
105 
106   @AfterClass
107   public static void cleanupTest() throws Exception {
108     try {
109       UTIL.shutdownMiniCluster();
110     } catch (Exception e) {
111       LOG.warn("failure shutting down cluster", e);
112     }
113   }
114 
115   /**
116    * Test snapshotting not allowed .META. and -ROOT-
117    * @throws Exception
118    */
119   @Test (timeout=300000)
120   public void testMetaTablesSnapshot() throws Exception {
121     HBaseAdmin admin = UTIL.getHBaseAdmin();
122     byte[] snapshotName = Bytes.toBytes("metaSnapshot");
123 
124     try {
125       admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
126       fail("taking a snapshot of .META. should not be allowed");
127     } catch (IllegalArgumentException e) {
128       // expected
129     }
130 
131     try {
132       admin.snapshot(snapshotName, TableName.ROOT_TABLE_NAME);
133       fail("taking a snapshot of -ROOT- should not be allowed");
134     } catch (IllegalArgumentException e) {
135       // expected
136     }
137   }
138 
139   /**
140    * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
141    *
142    * @throws Exception
143    */
144   @Test (timeout=300000)
145   public void testSnapshotDeletionWithRegex() throws Exception {
146     HBaseAdmin admin = UTIL.getHBaseAdmin();
147     // make sure we don't fail on listing snapshots
148     SnapshotTestingUtils.assertNoSnapshots(admin);
149 
150     // put some stuff in the table
151     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
152     UTIL.loadTable(table, TEST_FAM);
153     table.close();
154 
155     byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
156     admin.snapshot(snapshot1, TABLE_NAME);
157     LOG.debug("Snapshot1 completed.");
158 
159     byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
160     admin.snapshot(snapshot2, TABLE_NAME);
161     LOG.debug("Snapshot2 completed.");
162 
163     String snapshot3 = "3rdTableSnapshot";
164     admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
165     LOG.debug(snapshot3 + " completed.");
166 
167     // delete the first two snapshots
168     admin.deleteSnapshots("TableSnapshot.*");
169     List<SnapshotDescription> snapshots = admin.listSnapshots();
170     assertEquals(1, snapshots.size());
171     assertEquals(snapshots.get(0).getName(), snapshot3);
172 
173     admin.deleteSnapshot(snapshot3);
174     admin.close();
175   }
176   /**
177    * Test snapshotting a table that is offline
178    * @throws Exception
179    */
180   @Test (timeout=300000)
181   public void testOfflineTableSnapshot() throws Exception {
182     HBaseAdmin admin = UTIL.getHBaseAdmin();
183     // make sure we don't fail on listing snapshots
184     SnapshotTestingUtils.assertNoSnapshots(admin);
185 
186     // put some stuff in the table
187     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
188     UTIL.loadTable(table, TEST_FAM);
189 
190     // get the name of all the regionservers hosting the snapshotted table
191     Set<String> snapshotServers = new HashSet<String>();
192     List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
193     for (RegionServerThread server : servers) {
194       if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
195         snapshotServers.add(server.getRegionServer().getServerName().toString());
196       }
197     }
198 
199     LOG.debug("FS state before disable:");
200     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
201       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
202     // XXX if this is flakey, might want to consider using the async version and looping as
203     // disableTable can succeed and still timeout.
204     admin.disableTable(TABLE_NAME);
205 
206     LOG.debug("FS state before snapshot:");
207     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
208       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
209 
210     // take a snapshot of the disabled table
211     byte[] snapshot = Bytes.toBytes("offlineTableSnapshot");
212     admin.snapshot(snapshot, TABLE_NAME);
213     LOG.debug("Snapshot completed.");
214 
215     // make sure we have the snapshot
216     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
217       snapshot, TABLE_NAME);
218 
219     // make sure its a valid snapshot
220     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
221     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
222     LOG.debug("FS state after snapshot:");
223     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
224       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
225 
226     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
227       admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
228 
229     admin.deleteSnapshot(snapshot);
230     snapshots = admin.listSnapshots();
231     SnapshotTestingUtils.assertNoSnapshots(admin);
232   }
233 
234   @Test (timeout=300000)
235   public void testSnapshotFailsOnNonExistantTable() throws Exception {
236     HBaseAdmin admin = UTIL.getHBaseAdmin();
237     // make sure we don't fail on listing snapshots
238     SnapshotTestingUtils.assertNoSnapshots(admin);
239     String tableName = "_not_a_table";
240 
241     // make sure the table doesn't exist
242     boolean fail = false;
243     do {
244     try {
245       admin.getTableDescriptor(Bytes.toBytes(tableName));
246       fail = true;
247           LOG.error("Table:" + tableName + " already exists, checking a new name");
248       tableName = tableName+"!";
249     } catch (TableNotFoundException e) {
250       fail = false;
251       }
252     } while (fail);
253 
254     // snapshot the non-existant table
255     try {
256       admin.snapshot("fail", tableName);
257       fail("Snapshot succeeded even though there is not table.");
258     } catch (SnapshotCreationException e) {
259       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
260     }
261   }
262 }