View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.fail;
22  
23  import java.util.HashSet;
24  import java.util.List;
25  import java.util.Set;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.TableName;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.LargeTests;
36  import org.apache.hadoop.hbase.TableNotFoundException;
37  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
38  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
39  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
40  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
41  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
45  import org.junit.After;
46  import org.junit.AfterClass;
47  import org.junit.Before;
48  import org.junit.BeforeClass;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  /**
53   * Test create/using/deleting snapshots from the client
54   * <p>
55   * This is an end-to-end test for the snapshot utility
56   */
57  @Category(LargeTests.class)
58  public class TestSnapshotFromClient {
59    private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
60    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
61    private static final int NUM_RS = 2;
62    private static final String STRING_TABLE_NAME = "test";
63    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
64    private static final TableName TABLE_NAME =
65        TableName.valueOf(STRING_TABLE_NAME);
66  
67    /**
68     * Setup the config for the cluster
69     * @throws Exception on failure
70     */
71    @BeforeClass
72    public static void setupCluster() throws Exception {
73      setupConf(UTIL.getConfiguration());
74      UTIL.startMiniCluster(NUM_RS);
75    }
76  
77    private static void setupConf(Configuration conf) {
78      // disable the ui
79      conf.setInt("hbase.regionsever.info.port", -1);
80      // change the flush size to a small amount, regulating number of store files
81      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
82      // so make sure we get a compaction when doing a load, but keep around some
83      // files in the store
84      conf.setInt("hbase.hstore.compaction.min", 10);
85      conf.setInt("hbase.hstore.compactionThreshold", 10);
86      // block writes if we get to 12 store files
87      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
88      // Enable snapshot
89      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
90      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
91        ConstantSizeRegionSplitPolicy.class.getName());
92    }
93  
94    @Before
95    public void setup() throws Exception {
96      UTIL.createTable(TABLE_NAME, TEST_FAM);
97    }
98  
99    @After
100   public void tearDown() throws Exception {
101     UTIL.deleteTable(TABLE_NAME);
102     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
103     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
104   }
105 
106   @AfterClass
107   public static void cleanupTest() throws Exception {
108     try {
109       UTIL.shutdownMiniCluster();
110     } catch (Exception e) {
111       LOG.warn("failure shutting down cluster", e);
112     }
113   }
114 
115   /**
116    * Test snapshotting not allowed hbase:meta and -ROOT-
117    * @throws Exception
118    */
119   @Test (timeout=300000)
120   public void testMetaTablesSnapshot() throws Exception {
121     HBaseAdmin admin = UTIL.getHBaseAdmin();
122     byte[] snapshotName = Bytes.toBytes("metaSnapshot");
123 
124     try {
125       admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
126       fail("taking a snapshot of hbase:meta should not be allowed");
127     } catch (IllegalArgumentException e) {
128       // expected
129     }
130   }
131 
132   /**
133    * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
134    *
135    * @throws Exception
136    */
137   @Test (timeout=300000)
138   public void testSnapshotDeletionWithRegex() throws Exception {
139     HBaseAdmin admin = UTIL.getHBaseAdmin();
140     // make sure we don't fail on listing snapshots
141     SnapshotTestingUtils.assertNoSnapshots(admin);
142 
143     // put some stuff in the table
144     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
145     UTIL.loadTable(table, TEST_FAM);
146     table.close();
147 
148     byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
149     admin.snapshot(snapshot1, TABLE_NAME);
150     LOG.debug("Snapshot1 completed.");
151 
152     byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
153     admin.snapshot(snapshot2, TABLE_NAME);
154     LOG.debug("Snapshot2 completed.");
155 
156     String snapshot3 = "3rdTableSnapshot";
157     admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
158     LOG.debug(snapshot3 + " completed.");
159 
160     // delete the first two snapshots
161     admin.deleteSnapshots("TableSnapshot.*");
162     List<SnapshotDescription> snapshots = admin.listSnapshots();
163     assertEquals(1, snapshots.size());
164     assertEquals(snapshots.get(0).getName(), snapshot3);
165 
166     admin.deleteSnapshot(snapshot3);
167     admin.close();
168   }
169   /**
170    * Test snapshotting a table that is offline
171    * @throws Exception
172    */
173   @Test (timeout=300000)
174   public void testOfflineTableSnapshot() throws Exception {
175     HBaseAdmin admin = UTIL.getHBaseAdmin();
176     // make sure we don't fail on listing snapshots
177     SnapshotTestingUtils.assertNoSnapshots(admin);
178 
179     // put some stuff in the table
180     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
181     UTIL.loadTable(table, TEST_FAM, false);
182 
183     // get the name of all the regionservers hosting the snapshotted table
184     Set<String> snapshotServers = new HashSet<String>();
185     List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
186     for (RegionServerThread server : servers) {
187       if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
188         snapshotServers.add(server.getRegionServer().getServerName().toString());
189       }
190     }
191 
192     LOG.debug("FS state before disable:");
193     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
194       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
195     // XXX if this is flakey, might want to consider using the async version and looping as
196     // disableTable can succeed and still timeout.
197     admin.disableTable(TABLE_NAME);
198 
199     LOG.debug("FS state before snapshot:");
200     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
201       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
202 
203     // take a snapshot of the disabled table
204     byte[] snapshot = Bytes.toBytes("offlineTableSnapshot");
205     admin.snapshot(snapshot, TABLE_NAME);
206     LOG.debug("Snapshot completed.");
207 
208     // make sure we have the snapshot
209     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
210       snapshot, TABLE_NAME);
211 
212     // make sure its a valid snapshot
213     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
214     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
215     LOG.debug("FS state after snapshot:");
216     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
217       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
218 
219     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
220       admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
221 
222     admin.deleteSnapshot(snapshot);
223     snapshots = admin.listSnapshots();
224     SnapshotTestingUtils.assertNoSnapshots(admin);
225   }
226 
227   @Test (timeout=300000)
228   public void testSnapshotFailsOnNonExistantTable() throws Exception {
229     HBaseAdmin admin = UTIL.getHBaseAdmin();
230     // make sure we don't fail on listing snapshots
231     SnapshotTestingUtils.assertNoSnapshots(admin);
232     String tableName = "_not_a_table";
233 
234     // make sure the table doesn't exist
235     boolean fail = false;
236     do {
237     try {
238       admin.getTableDescriptor(Bytes.toBytes(tableName));
239       fail = true;
240           LOG.error("Table:" + tableName + " already exists, checking a new name");
241       tableName = tableName+"!";
242     } catch (TableNotFoundException e) {
243       fail = false;
244       }
245     } while (fail);
246 
247     // snapshot the non-existant table
248     try {
249       admin.snapshot("fail", tableName);
250       fail("Snapshot succeeded even though there is not table.");
251     } catch (SnapshotCreationException e) {
252       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
253     }
254   }
255 }