1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.HBaseTestingUtility;
28  import org.apache.hadoop.hbase.HColumnDescriptor;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.HTableDescriptor;
31  import org.apache.hadoop.hbase.LargeTests;
32  import org.apache.hadoop.hbase.master.MasterFileSystem;
33  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
34  import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
35  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.junit.After;
38  import org.junit.AfterClass;
39  import org.junit.Before;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  /**
45   * Test clone snapshots from the client
46   */
47  @Category(LargeTests.class)
48  public class TestCloneSnapshotFromClient {
49    final Log LOG = LogFactory.getLog(getClass());
50  
51    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
52  
53    private final byte[] FAMILY = Bytes.toBytes("cf");
54  
55    private byte[] emptySnapshot;
56    private byte[] snapshotName0;
57    private byte[] snapshotName1;
58    private byte[] snapshotName2;
59    private int snapshot0Rows;
60    private int snapshot1Rows;
61    private byte[] tableName;
62    private HBaseAdmin admin;
63  
64    @BeforeClass
65    public static void setUpBeforeClass() throws Exception {
66      TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
67      TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
68      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
69      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
70      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
71      TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
72      TEST_UTIL.getConfiguration().setBoolean(
73          "hbase.master.enabletable.roundrobin", true);
74      TEST_UTIL.startMiniCluster(3);
75    }
76  
77    @AfterClass
78    public static void tearDownAfterClass() throws Exception {
79      TEST_UTIL.shutdownMiniCluster();
80    }
81  
82    /**
83     * Initialize the tests with a table filled with some data
84     * and two snapshots (snapshotName0, snapshotName1) of different states.
85     * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
86     */
87    @Before
88    public void setup() throws Exception {
89      this.admin = TEST_UTIL.getHBaseAdmin();
90  
91      long tid = System.currentTimeMillis();
92      tableName = Bytes.toBytes("testtb-" + tid);
93      emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
94      snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
95      snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
96      snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
97  
98      // create Table and disable it
99      SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
100     admin.disableTable(tableName);
101 
102     // take an empty snapshot
103     admin.snapshot(emptySnapshot, tableName);
104 
105     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
106     try {
107       // enable table and insert data
108       admin.enableTable(tableName);
109       SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
110       snapshot0Rows = TEST_UTIL.countRows(table);
111       admin.disableTable(tableName);
112 
113       // take a snapshot
114       admin.snapshot(snapshotName0, tableName);
115 
116       // enable table and insert more data
117       admin.enableTable(tableName);
118       SnapshotTestingUtils.loadData(TEST_UTIL, table, 500, FAMILY);
119       snapshot1Rows = TEST_UTIL.countRows(table);
120       admin.disableTable(tableName);
121 
122       // take a snapshot of the updated table
123       admin.snapshot(snapshotName1, tableName);
124 
125       // re-enable table
126       admin.enableTable(tableName);
127     } finally {
128       table.close();
129     }
130   }
131 
132   @After
133   public void tearDown() throws Exception {
134     if (admin.tableExists(tableName)) {
135       TEST_UTIL.deleteTable(tableName);
136     }
137     SnapshotTestingUtils.deleteAllSnapshots(admin);
138     SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
139   }
140 
141   @Test(expected=SnapshotDoesNotExistException.class)
142   public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
143     String snapshotName = "random-snapshot-" + System.currentTimeMillis();
144     String tableName = "random-table-" + System.currentTimeMillis();
145     admin.cloneSnapshot(snapshotName, tableName);
146   }
147 
148   @Test
149   public void testCloneSnapshot() throws IOException, InterruptedException {
150     byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
151     testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
152     testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
153     testCloneSnapshot(clonedTableName, emptySnapshot, 0);
154   }
155 
156   private void testCloneSnapshot(final byte[] tableName, final byte[] snapshotName,
157       int snapshotRows) throws IOException, InterruptedException {
158     // create a new table from snapshot
159     admin.cloneSnapshot(snapshotName, tableName);
160     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshotRows);
161 
162     TEST_UTIL.deleteTable(tableName);
163   }
164 
165   /**
166    * Verify that tables created from the snapshot are still alive after source table deletion.
167    */
168   @Test
169   public void testCloneLinksAfterDelete() throws IOException, InterruptedException {
170     // Clone a table from the first snapshot
171     byte[] clonedTableName = Bytes.toBytes("clonedtb1-" + System.currentTimeMillis());
172     admin.cloneSnapshot(snapshotName0, clonedTableName);
173     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
174 
175     // Take a snapshot of this cloned table.
176     admin.disableTable(clonedTableName);
177     admin.snapshot(snapshotName2, clonedTableName);
178 
179     // Clone the snapshot of the cloned table
180     byte[] clonedTableName2 = Bytes.toBytes("clonedtb2-" + System.currentTimeMillis());
181     admin.cloneSnapshot(snapshotName2, clonedTableName2);
182     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
183     admin.disableTable(clonedTableName2);
184 
185     // Remove the original table
186     TEST_UTIL.deleteTable(tableName);
187     waitCleanerRun();
188 
189     // Verify the first cloned table
190     admin.enableTable(clonedTableName);
191     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
192 
193     // Verify the second cloned table
194     admin.enableTable(clonedTableName2);
195     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
196     admin.disableTable(clonedTableName2);
197 
198     // Delete the first cloned table
199     TEST_UTIL.deleteTable(clonedTableName);
200     waitCleanerRun();
201 
202     // Verify the second cloned table
203     admin.enableTable(clonedTableName2);
204     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName2, snapshot0Rows);
205 
206     // Clone a new table from cloned
207     byte[] clonedTableName3 = Bytes.toBytes("clonedtb3-" + System.currentTimeMillis());
208     admin.cloneSnapshot(snapshotName2, clonedTableName3);
209     SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName3, snapshot0Rows);
210 
211     // Delete the cloned tables
212     TEST_UTIL.deleteTable(clonedTableName2);
213     TEST_UTIL.deleteTable(clonedTableName3);
214     admin.deleteSnapshot(snapshotName2);
215   }
216 
217   // ==========================================================================
218   //  Helpers
219   // ==========================================================================
220   private void waitCleanerRun() throws InterruptedException {
221     TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
222   }
223 }