1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.HBaseTestingUtility;
28  import org.apache.hadoop.hbase.HColumnDescriptor;
29  import org.apache.hadoop.hbase.HConstants;
30  import org.apache.hadoop.hbase.HTableDescriptor;
31  import org.apache.hadoop.hbase.LargeTests;
32  import org.apache.hadoop.hbase.master.MasterFileSystem;
33  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
34  import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
35  import org.apache.hadoop.hbase.util.Bytes;
36  import org.apache.hadoop.hbase.util.MD5Hash;
37  import org.junit.After;
38  import org.junit.AfterClass;
39  import org.junit.Before;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  /**
45   * Test clone snapshots from the client
46   */
47  @Category(LargeTests.class)
48  public class TestCloneSnapshotFromClient {
49    final Log LOG = LogFactory.getLog(getClass());
50  
51    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
52  
53    private final byte[] FAMILY = Bytes.toBytes("cf");
54  
55    private byte[] emptySnapshot;
56    private byte[] snapshotName0;
57    private byte[] snapshotName1;
58    private byte[] snapshotName2;
59    private int snapshot0Rows;
60    private int snapshot1Rows;
61    private byte[] tableName;
62    private HBaseAdmin admin;
63  
64    @BeforeClass
65    public static void setUpBeforeClass() throws Exception {
66      TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
67      TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
68      TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
69      TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
70      TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
71      TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
72      TEST_UTIL.getConfiguration().setBoolean(
73          "hbase.master.enabletable.roundrobin", true);
74      TEST_UTIL.startMiniCluster(3);
75    }
76  
77    @AfterClass
78    public static void tearDownAfterClass() throws Exception {
79      TEST_UTIL.shutdownMiniCluster();
80    }
81  
82    /**
83     * Initialize the tests with a table filled with some data
84     * and two snapshots (snapshotName0, snapshotName1) of different states.
85     * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
86     */
87    @Before
88    public void setup() throws Exception {
89      this.admin = TEST_UTIL.getHBaseAdmin();
90  
91      long tid = System.currentTimeMillis();
92      tableName = Bytes.toBytes("testtb-" + tid);
93      emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
94      snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
95      snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
96      snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
97  
98      // create Table and disable it
99      createTable(tableName, FAMILY);
100     admin.disableTable(tableName);
101 
102     // take an empty snapshot
103     admin.snapshot(emptySnapshot, tableName);
104 
105     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
106     try {
107       // enable table and insert data
108       admin.enableTable(tableName);
109       loadData(table, 500, FAMILY);
110       snapshot0Rows = TEST_UTIL.countRows(table);
111       admin.disableTable(tableName);
112 
113       // take a snapshot
114       admin.snapshot(snapshotName0, tableName);
115 
116       // enable table and insert more data
117       admin.enableTable(tableName);
118       loadData(table, 500, FAMILY);
119       snapshot1Rows = TEST_UTIL.countRows(table);
120       admin.disableTable(tableName);
121 
122       // take a snapshot of the updated table
123       admin.snapshot(snapshotName1, tableName);
124 
125       // re-enable table
126       admin.enableTable(tableName);
127     } finally {
128       table.close();
129     }
130   }
131 
132   @After
133   public void tearDown() throws Exception {
134     if (admin.tableExists(tableName)) {
135       TEST_UTIL.deleteTable(tableName);
136     }
137     admin.deleteSnapshot(snapshotName0);
138     admin.deleteSnapshot(snapshotName1);
139 
140     // Ensure the archiver to be empty
141     MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
142     mfs.getFileSystem().delete(
143       new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
144   }
145 
146   @Test(expected=SnapshotDoesNotExistException.class)
147   public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
148     String snapshotName = "random-snapshot-" + System.currentTimeMillis();
149     String tableName = "random-table-" + System.currentTimeMillis();
150     admin.cloneSnapshot(snapshotName, tableName);
151   }
152 
153   @Test
154   public void testCloneSnapshot() throws IOException, InterruptedException {
155     byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
156     testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
157     testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
158     testCloneSnapshot(clonedTableName, emptySnapshot, 0);
159   }
160 
161   private void testCloneSnapshot(final byte[] tableName, final byte[] snapshotName,
162       int snapshotRows) throws IOException, InterruptedException {
163     // create a new table from snapshot
164     admin.cloneSnapshot(snapshotName, tableName);
165     verifyRowCount(tableName, snapshotRows);
166 
167     admin.disableTable(tableName);
168     admin.deleteTable(tableName);
169   }
170 
171   /**
172    * Verify that tables created from the snapshot are still alive after source table deletion.
173    */
174   @Test
175   public void testCloneLinksAfterDelete() throws IOException, InterruptedException {
176     // Clone a table from the first snapshot
177     byte[] clonedTableName = Bytes.toBytes("clonedtb1-" + System.currentTimeMillis());
178     admin.cloneSnapshot(snapshotName0, clonedTableName);
179     verifyRowCount(clonedTableName, snapshot0Rows);
180 
181     // Take a snapshot of this cloned table.
182     admin.disableTable(clonedTableName);
183     admin.snapshot(snapshotName2, clonedTableName);
184 
185     // Clone the snapshot of the cloned table
186     byte[] clonedTableName2 = Bytes.toBytes("clonedtb2-" + System.currentTimeMillis());
187     admin.cloneSnapshot(snapshotName2, clonedTableName2);
188     verifyRowCount(clonedTableName2, snapshot0Rows);
189     admin.disableTable(clonedTableName2);
190 
191     // Remove the original table
192     admin.disableTable(tableName);
193     admin.deleteTable(tableName);
194     waitCleanerRun();
195 
196     // Verify the first cloned table
197     admin.enableTable(clonedTableName);
198     verifyRowCount(clonedTableName, snapshot0Rows);
199 
200     // Verify the second cloned table
201     admin.enableTable(clonedTableName2);
202     verifyRowCount(clonedTableName2, snapshot0Rows);
203     admin.disableTable(clonedTableName2);
204 
205     // Delete the first cloned table
206     admin.disableTable(clonedTableName);
207     admin.deleteTable(clonedTableName);
208     waitCleanerRun();
209 
210     // Verify the second cloned table
211     admin.enableTable(clonedTableName2);
212     verifyRowCount(clonedTableName2, snapshot0Rows);
213 
214     // Clone a new table from cloned
215     byte[] clonedTableName3 = Bytes.toBytes("clonedtb3-" + System.currentTimeMillis());
216     admin.cloneSnapshot(snapshotName2, clonedTableName3);
217     verifyRowCount(clonedTableName3, snapshot0Rows);
218 
219     // Delete the cloned tables
220     admin.disableTable(clonedTableName2);
221     admin.deleteTable(clonedTableName2);
222     admin.disableTable(clonedTableName3);
223     admin.deleteTable(clonedTableName3);
224     admin.deleteSnapshot(snapshotName2);
225   }
226 
227   // ==========================================================================
228   //  Helpers
229   // ==========================================================================
230   private void createTable(final byte[] tableName, final byte[]... families) throws IOException {
231     HTableDescriptor htd = new HTableDescriptor(tableName);
232     for (byte[] family: families) {
233       HColumnDescriptor hcd = new HColumnDescriptor(family);
234       htd.addFamily(hcd);
235     }
236     byte[][] splitKeys = new byte[16][];
237     byte[] hex = Bytes.toBytes("0123456789abcdef");
238     for (int i = 0; i < 16; ++i) {
239       splitKeys[i] = new byte[] { hex[i] };
240     }
241     admin.createTable(htd, splitKeys);
242   }
243 
244   public void loadData(final HTable table, int rows, byte[]... families) throws IOException {
245     byte[] qualifier = Bytes.toBytes("q");
246     table.setAutoFlush(false);
247     while (rows-- > 0) {
248       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
249       byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
250       Put put = new Put(key);
251       put.setWriteToWAL(false);
252       for (byte[] family: families) {
253         put.add(family, qualifier, value);
254       }
255       table.put(put);
256     }
257     table.flushCommits();
258   }
259 
260   private void waitCleanerRun() throws InterruptedException {
261     TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
262   }
263 
264   private void verifyRowCount(final byte[] tableName, long expectedRows) throws IOException {
265     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
266     assertEquals(expectedRows, TEST_UTIL.countRows(table));
267     table.close();
268   }
269 }