1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.fs.FileStatus;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HColumnDescriptor;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.LargeTests;
35  import org.apache.hadoop.hbase.client.HBaseAdmin;
36  import org.apache.hadoop.hbase.client.HTable;
37  import org.apache.hadoop.hbase.client.Put;
38  import org.apache.hadoop.hbase.master.MasterFileSystem;
39  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
40  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.FSUtils;
43  import org.apache.hadoop.hbase.util.MD5Hash;
44  import org.junit.After;
45  import org.junit.AfterClass;
46  import org.junit.Before;
47  import org.junit.BeforeClass;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  /**
52   * Test clone/restore snapshots from the client
53   *
54   * TODO This is essentially a clone of TestRestoreSnapshotFromClient.  This is worth refactoring
55   * this because there will be a few more flavors of snapshots that need to run these tests.
56   */
57  @Category(LargeTests.class)
58  public class TestRestoreFlushSnapshotFromClient {
59    final Log LOG = LogFactory.getLog(getClass());
60  
61    private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
62  
63    private final byte[] FAMILY = Bytes.toBytes("cf");
64    private static final byte[] TEST_QUAL = Bytes.toBytes("q");
65  
66    private byte[] snapshotName0;
67    private byte[] snapshotName1;
68    private byte[] snapshotName2;
69    private int snapshot0Rows;
70    private int snapshot1Rows;
71    private byte[] tableName;
72    private HBaseAdmin admin;
73  
74    @BeforeClass
75    public static void setUpBeforeClass() throws Exception {
76      UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
77      UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
78      UTIL.getConfiguration().setInt("hbase.client.pause", 250);
79      UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
80      UTIL.getConfiguration().setBoolean(
81          "hbase.master.enabletable.roundrobin", true);
82  
83      // Enable snapshot
84      UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
85  
86      UTIL.startMiniCluster(3);
87    }
88  
89    @AfterClass
90    public static void tearDownAfterClass() throws Exception {
91      UTIL.shutdownMiniCluster();
92    }
93  
94    /**
95     * Initialize the tests with a table filled with some data
96     * and two snapshots (snapshotName0, snapshotName1) of different states.
97     * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
98     */
99    @Before
100   public void setup() throws Exception {
101     this.admin = UTIL.getHBaseAdmin();
102 
103     long tid = System.currentTimeMillis();
104     tableName = Bytes.toBytes("testtb-" + tid);
105     snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
106     snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
107     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
108 
109     // create Table and disable it
110     SnapshotTestingUtils.createTable(UTIL, tableName, FAMILY);
111     HTable table = new HTable(UTIL.getConfiguration(), tableName);
112     try {
113       SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
114       snapshot0Rows = UTIL.countRows(table);
115       LOG.info("=== before snapshot with 500 rows");
116       logFSTree();
117 
118       // take a snapshot
119       admin.snapshot(Bytes.toString(snapshotName0), Bytes.toString(tableName),
120           SnapshotDescription.Type.FLUSH);
121 
122       LOG.info("=== after snapshot with 500 rows");
123       logFSTree();
124 
125       // insert more data
126       SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
127       snapshot1Rows = UTIL.countRows(table);
128       LOG.info("=== before snapshot with 1000 rows");
129       logFSTree();
130 
131       // take a snapshot of the updated table
132       admin.snapshot(Bytes.toString(snapshotName1), Bytes.toString(tableName),
133           SnapshotDescription.Type.FLUSH);
134       LOG.info("=== after snapshot with 1000 rows");
135       logFSTree();
136     } finally {
137       table.close();
138     }
139   }
140 
141   @After
142   public void tearDown() throws Exception {
143     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
144     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
145   }
146 
147   @Test
148   public void testTakeFlushSnapshot() throws IOException {
149     // taking happens in setup.
150   }
151 
152   @Test
153   public void testRestoreSnapshot() throws IOException {
154     SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);
155 
156     // Restore from snapshot-0
157     admin.disableTable(tableName);
158     admin.restoreSnapshot(snapshotName0);
159     logFSTree();
160     admin.enableTable(tableName);
161     LOG.info("=== after restore with 500 row snapshot");
162     logFSTree();
163     SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot0Rows);
164 
165     // Restore from snapshot-1
166     admin.disableTable(tableName);
167     admin.restoreSnapshot(snapshotName1);
168     admin.enableTable(tableName);
169     SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);
170   }
171 
172   @Test(expected=SnapshotDoesNotExistException.class)
173   public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
174     String snapshotName = "random-snapshot-" + System.currentTimeMillis();
175     String tableName = "random-table-" + System.currentTimeMillis();
176     admin.cloneSnapshot(snapshotName, tableName);
177   }
178 
179   @Test
180   public void testCloneSnapshot() throws IOException, InterruptedException {
181     byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
182     testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
183     testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
184   }
185 
186   private void testCloneSnapshot(final byte[] tableName, final byte[] snapshotName,
187       int snapshotRows) throws IOException, InterruptedException {
188     // create a new table from snapshot
189     admin.cloneSnapshot(snapshotName, tableName);
190     SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshotRows);
191 
192     UTIL.deleteTable(tableName);
193   }
194 
195   @Test
196   public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException {
197     byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
198     admin.cloneSnapshot(snapshotName0, clonedTableName);
199     SnapshotTestingUtils.verifyRowCount(UTIL, clonedTableName, snapshot0Rows);
200     admin.snapshot(Bytes.toString(snapshotName2), Bytes.toString(clonedTableName), SnapshotDescription.Type.FLUSH);
201     UTIL.deleteTable(clonedTableName);
202 
203     admin.cloneSnapshot(snapshotName2, clonedTableName);
204     SnapshotTestingUtils.verifyRowCount(UTIL, clonedTableName, snapshot0Rows);
205     UTIL.deleteTable(clonedTableName);
206   }
207 
208   // ==========================================================================
209   //  Helpers
210   // ==========================================================================
211   private void logFSTree() throws IOException {
212     MasterFileSystem mfs = UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
213     FSUtils.logFileSystemState(mfs.getFileSystem(), mfs.getRootDir(), LOG);
214   }
215 }