1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.fail;
21  
22  import java.io.IOException;
23  import java.util.HashSet;
24  import java.util.List;
25  import java.util.Set;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.LargeTests;
35  import org.apache.hadoop.hbase.TableNotFoundException;
36  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
37  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
38  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
39  import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
40  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
41  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
45  import org.junit.After;
46  import org.junit.AfterClass;
47  import org.junit.Before;
48  import org.junit.BeforeClass;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  /**
53   * Test create/using/deleting snapshots from the client
54   * <p>
55   * This is an end-to-end test for the snapshot utility
56   */
57  @Category(LargeTests.class)
58  public class TestSnapshotFromClient {
59    private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
60    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
61    private static final int NUM_RS = 2;
62    private static final String STRING_TABLE_NAME = "test";
63    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
64    private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
65  
66    /**
67     * Setup the config for the cluster
68     * @throws Exception on failure
69     */
70    @BeforeClass
71    public static void setupCluster() throws Exception {
72      setupConf(UTIL.getConfiguration());
73      UTIL.startMiniCluster(NUM_RS);
74    }
75  
76    private static void setupConf(Configuration conf) {
77      // disable the ui
78      conf.setInt("hbase.regionsever.info.port", -1);
79      // change the flush size to a small amount, regulating number of store files
80      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
81      // so make sure we get a compaction when doing a load, but keep around some
82      // files in the store
83      conf.setInt("hbase.hstore.compaction.min", 10);
84      conf.setInt("hbase.hstore.compactionThreshold", 10);
85      // block writes if we get to 12 store files
86      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
87      // drop the number of attempts for the hbase admin
88      conf.setInt("hbase.client.retries.number", 1);
89      // Enable snapshot
90      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
91      // prevent aggressive region split
92      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
93        ConstantSizeRegionSplitPolicy.class.getName());
94    }
95  
96    @Before
97    public void setup() throws Exception {
98      UTIL.createTable(TABLE_NAME, TEST_FAM);
99    }
100 
101   @After
102   public void tearDown() throws Exception {
103     UTIL.deleteTable(TABLE_NAME);
104     // and cleanup the archive directory
105     try {
106       UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true);
107     } catch (IOException e) {
108       LOG.warn("Failure to delete archive directory", e);
109     }
110   }
111 
112   @AfterClass
113   public static void cleanupTest() throws Exception {
114     try {
115       UTIL.shutdownMiniCluster();
116     } catch (Exception e) {
117       LOG.warn("failure shutting down cluster", e);
118     }
119   }
120 
121   /**
122    * Test snapshotting not allowed .META. and -ROOT-
123    * @throws Exception
124    */
125   @Test
126   public void testMetaTablesSnapshot() throws Exception {
127     HBaseAdmin admin = UTIL.getHBaseAdmin();
128     byte[] snapshotName = Bytes.toBytes("metaSnapshot");
129 
130     try {
131       admin.snapshot(snapshotName, HConstants.META_TABLE_NAME);
132       fail("taking a snapshot of .META. should not be allowed");
133     } catch (IllegalArgumentException e) {
134       // expected
135     }
136 
137     try {
138       admin.snapshot(snapshotName, HConstants.ROOT_TABLE_NAME);
139       fail("taking a snapshot of -ROOT- should not be allowed");
140     } catch (IllegalArgumentException e) {
141       // expected
142     }
143   }
144 
145   /**
146    * Test snapshotting a table that is offline
147    * @throws Exception
148    */
149   @Test
150   public void testOfflineTableSnapshot() throws Exception {
151     HBaseAdmin admin = UTIL.getHBaseAdmin();
152     // make sure we don't fail on listing snapshots
153     SnapshotTestingUtils.assertNoSnapshots(admin);
154 
155     // put some stuff in the table
156     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
157     UTIL.loadTable(table, TEST_FAM);
158 
159     // get the name of all the regionservers hosting the snapshotted table
160     Set<String> snapshotServers = new HashSet<String>();
161     List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
162     for (RegionServerThread server : servers) {
163       if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
164         snapshotServers.add(server.getRegionServer().getServerName().toString());
165       }
166     }
167 
168     LOG.debug("FS state before disable:");
169     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
170       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
171     // XXX if this is flakey, might want to consider using the async version and looping as
172     // disableTable can succeed and still timeout.
173     admin.disableTable(TABLE_NAME);
174 
175     LOG.debug("FS state before snapshot:");
176     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
177       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
178 
179     // take a snapshot of the disabled table
180     byte[] snapshot = Bytes.toBytes("offlineTableSnapshot");
181     admin.snapshot(snapshot, TABLE_NAME);
182     LOG.debug("Snapshot completed.");
183 
184     // make sure we have the snapshot
185     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
186       snapshot, TABLE_NAME);
187 
188     // make sure its a valid snapshot
189     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
190     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
191     LOG.debug("FS state after snapshot:");
192     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
193       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
194 
195     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
196       admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
197 
198     admin.deleteSnapshot(snapshot);
199     snapshots = admin.listSnapshots();
200     SnapshotTestingUtils.assertNoSnapshots(admin);
201   }
202 
203   @Test
204   public void testSnapshotFailsOnNonExistantTable() throws Exception {
205     HBaseAdmin admin = UTIL.getHBaseAdmin();
206     // make sure we don't fail on listing snapshots
207     SnapshotTestingUtils.assertNoSnapshots(admin);
208     String tableName = "_not_a_table";
209 
210     // make sure the table doesn't exist
211     boolean fail = false;
212     do {
213     try {
214       admin.getTableDescriptor(Bytes.toBytes(tableName));
215       fail = true;
216           LOG.error("Table:" + tableName + " already exists, checking a new name");
217       tableName = tableName+"!";
218     } catch (TableNotFoundException e) {
219       fail = false;
220       }
221     } while (fail);
222 
223     // snapshot the non-existant table
224     try {
225       admin.snapshot("fail", tableName);
226       fail("Snapshot succeeded even though there is not table.");
227     } catch (SnapshotCreationException e) {
228       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
229     }
230   }
231 }