1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.fail;
21  
22  import java.io.IOException;
23  import java.util.HashSet;
24  import java.util.List;
25  import java.util.Set;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.LargeTests;
35  import org.apache.hadoop.hbase.TableNotFoundException;
36  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
37  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
38  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
39  import org.apache.hadoop.hbase.snapshot.HSnapshotDescription;
40  import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
41  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.FSUtils;
44  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
45  import org.junit.After;
46  import org.junit.AfterClass;
47  import org.junit.Before;
48  import org.junit.BeforeClass;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  /**
53   * Test create/using/deleting snapshots from the client
54   * <p>
55   * This is an end-to-end test for the snapshot utility
56   */
57  @Category(LargeTests.class)
58  public class TestSnapshotFromClient {
59    private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
60    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
61    private static final int NUM_RS = 2;
62    private static final String STRING_TABLE_NAME = "test";
63    private static final byte[] TEST_FAM = Bytes.toBytes("fam");
64    private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
65  
66    /**
67     * Setup the config for the cluster
68     * @throws Exception on failure
69     */
70    @BeforeClass
71    public static void setupCluster() throws Exception {
72      setupConf(UTIL.getConfiguration());
73      UTIL.startMiniCluster(NUM_RS);
74    }
75  
76    private static void setupConf(Configuration conf) {
77      // disable the ui
78      conf.setInt("hbase.regionsever.info.port", -1);
79      // change the flush size to a small amount, regulating number of store files
80      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
81      // so make sure we get a compaction when doing a load, but keep around some
82      // files in the store
83      conf.setInt("hbase.hstore.compaction.min", 10);
84      conf.setInt("hbase.hstore.compactionThreshold", 10);
85      // block writes if we get to 12 store files
86      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
87      // drop the number of attempts for the hbase admin
88      conf.setInt("hbase.client.retries.number", 1);
89      // Enable snapshot
90      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
91      // prevent aggressive region split
92      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
93        ConstantSizeRegionSplitPolicy.class.getName());
94    }
95  
96    @Before
97    public void setup() throws Exception {
98      UTIL.createTable(TABLE_NAME, TEST_FAM);
99    }
100 
101   @After
102   public void tearDown() throws Exception {
103     UTIL.deleteTable(TABLE_NAME);
104     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
105     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
106   }
107 
108   @AfterClass
109   public static void cleanupTest() throws Exception {
110     try {
111       UTIL.shutdownMiniCluster();
112     } catch (Exception e) {
113       LOG.warn("failure shutting down cluster", e);
114     }
115   }
116 
117   /**
118    * Test snapshotting not allowed .META. and -ROOT-
119    * @throws Exception
120    */
121   @Test
122   public void testMetaTablesSnapshot() throws Exception {
123     HBaseAdmin admin = UTIL.getHBaseAdmin();
124     byte[] snapshotName = Bytes.toBytes("metaSnapshot");
125 
126     try {
127       admin.snapshot(snapshotName, HConstants.META_TABLE_NAME);
128       fail("taking a snapshot of .META. should not be allowed");
129     } catch (IllegalArgumentException e) {
130       // expected
131     }
132 
133     try {
134       admin.snapshot(snapshotName, HConstants.ROOT_TABLE_NAME);
135       fail("taking a snapshot of -ROOT- should not be allowed");
136     } catch (IllegalArgumentException e) {
137       // expected
138     }
139   }
140 
141   /**
142    * Test snapshotting a table that is offline
143    * @throws Exception
144    */
145   @Test
146   public void testOfflineTableSnapshot() throws Exception {
147     HBaseAdmin admin = UTIL.getHBaseAdmin();
148     // make sure we don't fail on listing snapshots
149     SnapshotTestingUtils.assertNoSnapshots(admin);
150 
151     // put some stuff in the table
152     HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
153     UTIL.loadTable(table, TEST_FAM);
154 
155     // get the name of all the regionservers hosting the snapshotted table
156     Set<String> snapshotServers = new HashSet<String>();
157     List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
158     for (RegionServerThread server : servers) {
159       if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
160         snapshotServers.add(server.getRegionServer().getServerName().toString());
161       }
162     }
163 
164     LOG.debug("FS state before disable:");
165     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
166       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
167     // XXX if this is flakey, might want to consider using the async version and looping as
168     // disableTable can succeed and still timeout.
169     admin.disableTable(TABLE_NAME);
170 
171     LOG.debug("FS state before snapshot:");
172     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
173       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
174 
175     // take a snapshot of the disabled table
176     byte[] snapshot = Bytes.toBytes("offlineTableSnapshot");
177     admin.snapshot(snapshot, TABLE_NAME);
178     LOG.debug("Snapshot completed.");
179 
180     // make sure we have the snapshot
181     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
182       snapshot, TABLE_NAME);
183 
184     // make sure its a valid snapshot
185     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
186     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
187     LOG.debug("FS state after snapshot:");
188     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
189       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
190 
191     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
192       admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
193 
194     admin.deleteSnapshot(snapshot);
195     snapshots = admin.listSnapshots();
196     SnapshotTestingUtils.assertNoSnapshots(admin);
197   }
198 
199   @Test
200   public void testSnapshotFailsOnNonExistantTable() throws Exception {
201     HBaseAdmin admin = UTIL.getHBaseAdmin();
202     // make sure we don't fail on listing snapshots
203     SnapshotTestingUtils.assertNoSnapshots(admin);
204     String tableName = "_not_a_table";
205 
206     // make sure the table doesn't exist
207     boolean fail = false;
208     do {
209     try {
210       admin.getTableDescriptor(Bytes.toBytes(tableName));
211       fail = true;
212           LOG.error("Table:" + tableName + " already exists, checking a new name");
213       tableName = tableName+"!";
214     } catch (TableNotFoundException e) {
215       fail = false;
216       }
217     } while (fail);
218 
219     // snapshot the non-existant table
220     try {
221       admin.snapshot("fail", tableName);
222       fail("Snapshot succeeded even though there is not table.");
223     } catch (SnapshotCreationException e) {
224       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
225     }
226   }
227 }