View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.HBaseTestingUtility;
30  import org.apache.hadoop.hbase.HColumnDescriptor;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HTableDescriptor;
33  import org.apache.hadoop.hbase.LargeTests;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.client.HBaseAdmin;
36  import org.apache.hadoop.hbase.client.HTable;
37  import org.apache.hadoop.hbase.client.Put;
38  import org.apache.hadoop.hbase.client.Durability;
39  import org.apache.hadoop.hbase.master.MasterFileSystem;
40  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
41  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
42  import org.apache.hadoop.hbase.regionserver.HRegion;
43  import org.apache.hadoop.hbase.regionserver.HRegionServer;
44  import org.apache.hadoop.hbase.regionserver.snapshot.RegionServerSnapshotManager;
45  import org.apache.hadoop.hbase.util.Bytes;
46  import org.apache.hadoop.hbase.util.FSUtils;
47  import org.apache.hadoop.hbase.util.MD5Hash;
48  import org.junit.After;
49  import org.junit.AfterClass;
50  import org.junit.Before;
51  import org.junit.BeforeClass;
52  import org.junit.Test;
53  import org.junit.experimental.categories.Category;
54  
55  /**
56   * Test clone/restore snapshots from the client
57   *
58   * TODO This is essentially a clone of TestRestoreSnapshotFromClient.  This is worth refactoring
59   * this because there will be a few more flavors of snapshots that need to run these tests.
60   */
61  @Category(LargeTests.class)
62  public class TestRestoreFlushSnapshotFromClient {
63    final Log LOG = LogFactory.getLog(getClass());
64  
65    private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
66  
67    private final byte[] FAMILY = Bytes.toBytes("cf");
68    private static final byte[] TEST_QUAL = Bytes.toBytes("q");
69  
70    private byte[] snapshotName0;
71    private byte[] snapshotName1;
72    private byte[] snapshotName2;
73    private int snapshot0Rows;
74    private int snapshot1Rows;
75    private TableName tableName;
76    private HBaseAdmin admin;
77  
78    @BeforeClass
79    public static void setUpBeforeClass() throws Exception {
80      UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
81      UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
82      UTIL.getConfiguration().setInt("hbase.client.pause", 250);
83      UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
84      UTIL.getConfiguration().setBoolean(
85          "hbase.master.enabletable.roundrobin", true);
86  
87      // Enable snapshot
88      UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
89      UTIL.getConfiguration().setLong(RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_KEY,
90        RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT * 2);
91  
92      UTIL.startMiniCluster(3);
93    }
94  
95    @AfterClass
96    public static void tearDownAfterClass() throws Exception {
97      UTIL.shutdownMiniCluster();
98    }
99  
100   /**
101    * Initialize the tests with a table filled with some data
102    * and two snapshots (snapshotName0, snapshotName1) of different states.
103    * The tableName, snapshotNames and the number of rows in the snapshot are initialized.
104    */
105   @Before
106   public void setup() throws Exception {
107     this.admin = UTIL.getHBaseAdmin();
108 
109     long tid = System.currentTimeMillis();
110     tableName = TableName.valueOf("testtb-" + tid);
111     snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
112     snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
113     snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
114 
115     // create Table and disable it
116     SnapshotTestingUtils.createTable(UTIL, tableName, FAMILY);
117     HTable table = new HTable(UTIL.getConfiguration(), tableName);
118     try {
119       SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
120       snapshot0Rows = UTIL.countRows(table);
121       LOG.info("=== before snapshot with 500 rows");
122       logFSTree();
123 
124       // take a snapshot
125       admin.snapshot(Bytes.toString(snapshotName0), tableName,
126           SnapshotDescription.Type.FLUSH);
127 
128       LOG.info("=== after snapshot with 500 rows");
129       logFSTree();
130 
131       // insert more data
132       SnapshotTestingUtils.loadData(UTIL, table, 500, FAMILY);
133       snapshot1Rows = UTIL.countRows(table);
134       LOG.info("=== before snapshot with 1000 rows");
135       logFSTree();
136 
137       // take a snapshot of the updated table
138       admin.snapshot(Bytes.toString(snapshotName1), tableName,
139           SnapshotDescription.Type.FLUSH);
140       LOG.info("=== after snapshot with 1000 rows");
141       logFSTree();
142     } finally {
143       table.close();
144     }
145   }
146 
147   @After
148   public void tearDown() throws Exception {
149     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
150     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
151   }
152 
153   @Test
154   public void testTakeFlushSnapshot() throws IOException {
155     // taking happens in setup.
156   }
157 
158   @Test
159   public void testRestoreSnapshot() throws IOException {
160     SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);
161 
162     // Restore from snapshot-0
163     admin.disableTable(tableName);
164     admin.restoreSnapshot(snapshotName0);
165     logFSTree();
166     admin.enableTable(tableName);
167     LOG.info("=== after restore with 500 row snapshot");
168     logFSTree();
169     SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot0Rows);
170 
171     // Restore from snapshot-1
172     admin.disableTable(tableName);
173     admin.restoreSnapshot(snapshotName1);
174     admin.enableTable(tableName);
175     SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshot1Rows);
176   }
177 
178   @Test(expected=SnapshotDoesNotExistException.class)
179   public void testCloneNonExistentSnapshot() throws IOException, InterruptedException {
180     String snapshotName = "random-snapshot-" + System.currentTimeMillis();
181     String tableName = "random-table-" + System.currentTimeMillis();
182     admin.cloneSnapshot(snapshotName, tableName);
183   }
184 
185   @Test
186   public void testCloneSnapshot() throws IOException, InterruptedException {
187     TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis());
188     testCloneSnapshot(clonedTableName, snapshotName0, snapshot0Rows);
189     testCloneSnapshot(clonedTableName, snapshotName1, snapshot1Rows);
190   }
191 
192   private void testCloneSnapshot(final TableName tableName, final byte[] snapshotName,
193       int snapshotRows) throws IOException, InterruptedException {
194     // create a new table from snapshot
195     admin.cloneSnapshot(snapshotName, tableName);
196     SnapshotTestingUtils.verifyRowCount(UTIL, tableName, snapshotRows);
197 
198     UTIL.deleteTable(tableName);
199   }
200 
201   @Test
202   public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException {
203     TableName clonedTableName = TableName.valueOf("clonedtb-" + System.currentTimeMillis());
204     admin.cloneSnapshot(snapshotName0, clonedTableName);
205     SnapshotTestingUtils.verifyRowCount(UTIL, clonedTableName, snapshot0Rows);
206     admin.snapshot(Bytes.toString(snapshotName2), clonedTableName, SnapshotDescription.Type.FLUSH);
207     UTIL.deleteTable(clonedTableName);
208 
209     admin.cloneSnapshot(snapshotName2, clonedTableName);
210     SnapshotTestingUtils.verifyRowCount(UTIL, clonedTableName, snapshot0Rows);
211     UTIL.deleteTable(clonedTableName);
212   }
213 
214   // ==========================================================================
215   //  Helpers
216   // ==========================================================================
217   private void logFSTree() throws IOException {
218     MasterFileSystem mfs = UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
219     FSUtils.logFileSystemState(mfs.getFileSystem(), mfs.getRootDir(), LOG);
220   }
221 }