View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.util.List;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.fs.FileSystem;
27  import org.apache.hadoop.fs.Path;
28  import org.apache.hadoop.hbase.TableName;
29  import org.apache.hadoop.hbase.HBaseTestingUtility;
30  import org.apache.hadoop.hbase.HColumnDescriptor;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.testclassification.LargeTests;
35  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
36  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.junit.After;
40  import org.junit.AfterClass;
41  import org.junit.Assert;
42  import org.junit.Before;
43  import org.junit.BeforeClass;
44  import org.junit.Test;
45  import org.junit.experimental.categories.Category;
46  
47  /**
48   * Test to verify that the cloned table is independent of the table from which it was cloned
49   */
50  @Category(LargeTests.class)
51  public class TestSnapshotCloneIndependence {
52    private static final Log LOG = LogFactory.getLog(TestSnapshotCloneIndependence.class);
53  
54    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
55  
56    private static final int NUM_RS = 2;
57    private static final String STRING_TABLE_NAME = "test";
58    private static final String TEST_FAM_STR = "fam";
59    private static final byte[] TEST_FAM = Bytes.toBytes(TEST_FAM_STR);
60    private static final TableName TABLE_NAME = TableName.valueOf(STRING_TABLE_NAME);
61  
62    /**
63     * Setup the config for the cluster and start it
64     * @throws Exception on failure
65     */
66    @BeforeClass
67    public static void setupCluster() throws Exception {
68      setupConf(UTIL.getConfiguration());
69      UTIL.startMiniCluster(NUM_RS);
70    }
71  
72    private static void setupConf(Configuration conf) {
73      // enable snapshot support
74      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
75      // disable the ui
76      conf.setInt("hbase.regionsever.info.port", -1);
77      // change the flush size to a small amount, regulating number of store files
78      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
79      // so make sure we get a compaction when doing a load, but keep around
80      // some files in the store
81      conf.setInt("hbase.hstore.compaction.min", 10);
82      conf.setInt("hbase.hstore.compactionThreshold", 10);
83      // block writes if we get to 12 store files
84      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
85      conf.setInt("hbase.regionserver.msginterval", 100);
86      conf.setBoolean("hbase.master.enabletable.roundrobin", true);
87      // Avoid potentially aggressive splitting which would cause snapshot to fail
88      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
89        ConstantSizeRegionSplitPolicy.class.getName());
90    }
91  
92    @Before
93    public void setup() throws Exception {
94      UTIL.createTable(TABLE_NAME, TEST_FAM);
95    }
96  
97    @After
98    public void tearDown() throws Exception {
99      UTIL.deleteTable(TABLE_NAME);
100     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
101     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
102   }
103 
104   @AfterClass
105   public static void cleanupTest() throws Exception {
106     try {
107       UTIL.shutdownMiniCluster();
108     } catch (Exception e) {
109       LOG.warn("failure shutting down cluster", e);
110     }
111   }
112 
113   /**
114    * Verify that adding data to the cloned table will not affect the original, and vice-versa when
115    * it is taken as an online snapshot.
116    */
117   @Test (timeout=300000)
118   public void testOnlineSnapshotAppendIndependent() throws Exception {
119     runTestSnapshotAppendIndependent(true);
120   }
121 
122   /**
123    * Verify that adding data to the cloned table will not affect the original, and vice-versa when
124    * it is taken as an offline snapshot.
125    */
126   @Test (timeout=300000)
127   public void testOfflineSnapshotAppendIndependent() throws Exception {
128     runTestSnapshotAppendIndependent(false);
129   }
130 
131   /**
132    * Verify that adding metadata to the cloned table will not affect the original, and vice-versa
133    * when it is taken as an online snapshot.
134    */
135   @Test (timeout=300000)
136   public void testOnlineSnapshotMetadataChangesIndependent() throws Exception {
137     runTestSnapshotMetadataChangesIndependent(true);
138   }
139 
140   /**
141    * Verify that adding netadata to the cloned table will not affect the original, and vice-versa
142    * when is taken as an online snapshot.
143    */
144   @Test (timeout=300000)
145   public void testOfflineSnapshotMetadataChangesIndependent() throws Exception {
146     runTestSnapshotMetadataChangesIndependent(false);
147   }
148 
149   /**
150    * Verify that region operations, in this case splitting a region, are independent between the
151    * cloned table and the original.
152    */
153   @Test (timeout=300000)
154   public void testOfflineSnapshotRegionOperationsIndependent() throws Exception {
155     runTestRegionOperationsIndependent(false);
156   }
157 
158   /**
159    * Verify that region operations, in this case splitting a region, are independent between the
160    * cloned table and the original.
161    */
162   @Test (timeout=300000)
163   public void testOnlineSnapshotRegionOperationsIndependent() throws Exception {
164     runTestRegionOperationsIndependent(true);
165   }
166 
167   private static void waitOnSplit(final HTable t, int originalCount) throws Exception {
168     for (int i = 0; i < 200; i++) {
169       try {
170         Thread.sleep(50);
171       } catch (InterruptedException e) {
172         // Restore the interrupted status
173         Thread.currentThread().interrupt();
174       }
175       if (t.getAllRegionLocations().size() > originalCount) {
176         return;
177       }
178     }
179     throw new Exception("Split did not increase the number of regions");
180   }
181 
182   /*
183    * Take a snapshot of a table, add data, and verify that this only
184    * affects one table
185    * @param online - Whether the table is online or not during the snapshot
186    */
187   private void runTestSnapshotAppendIndependent(boolean online) throws Exception {
188     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
189     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
190 
191     Admin admin = UTIL.getHBaseAdmin();
192     final long startTime = System.currentTimeMillis();
193     final TableName localTableName =
194         TableName.valueOf(STRING_TABLE_NAME + startTime);
195 
196     try (Table original = UTIL.createTable(localTableName, TEST_FAM)) {
197       UTIL.loadTable(original, TEST_FAM);
198       final int origTableRowCount = UTIL.countRows(original);
199 
200       // Take a snapshot
201       final String snapshotNameAsString = "snapshot_" + localTableName;
202       byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
203 
204       SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
205         snapshotNameAsString, rootDir, fs, online);
206 
207       if (!online) {
208         admin.enableTable(localTableName);
209       }
210       TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
211       admin.cloneSnapshot(snapshotName, cloneTableName);
212 
213       try (Table clonedTable = new HTable(UTIL.getConfiguration(), cloneTableName)){
214         final int clonedTableRowCount = UTIL.countRows(clonedTable);
215 
216         Assert.assertEquals(
217           "The line counts of original and cloned tables do not match after clone. ",
218           origTableRowCount, clonedTableRowCount);
219 
220         // Attempt to add data to the test
221         final String rowKey = "new-row-" + System.currentTimeMillis();
222 
223         Put p = new Put(Bytes.toBytes(rowKey));
224         p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
225         original.put(p);
226 
227         // Verify that it is not present in the original table
228         Assert.assertEquals("The row count of the original table was not modified by the put",
229           origTableRowCount + 1, UTIL.countRows(original));
230         Assert.assertEquals(
231           "The row count of the cloned table changed as a result of addition to the original",
232           clonedTableRowCount, UTIL.countRows(clonedTable));
233 
234         p = new Put(Bytes.toBytes(rowKey));
235         p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
236         clonedTable.put(p);
237 
238         // Verify that the new family is not in the restored table's description
239         Assert.assertEquals(
240           "The row count of the original table was modified by the put to the clone",
241           origTableRowCount + 1, UTIL.countRows(original));
242         Assert.assertEquals("The row count of the cloned table was not modified by the put",
243           clonedTableRowCount + 1, UTIL.countRows(clonedTable));
244       }
245     }
246   }
247 
248   /*
249    * Take a snapshot of a table, do a split, and verify that this only affects one table
250    * @param online - Whether the table is online or not during the snapshot
251    */
252   private void runTestRegionOperationsIndependent(boolean online) throws Exception {
253     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
254     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
255 
256     // Create a table
257     Admin admin = UTIL.getHBaseAdmin();
258     final long startTime = System.currentTimeMillis();
259     final TableName localTableName =
260         TableName.valueOf(STRING_TABLE_NAME + startTime);
261     HTable original = UTIL.createTable(localTableName, TEST_FAM);
262     UTIL.loadTable(original, TEST_FAM);
263     final int loadedTableCount = UTIL.countRows(original);
264     System.out.println("Original table has: " + loadedTableCount + " rows");
265 
266     final String snapshotNameAsString = "snapshot_" + localTableName;
267 
268     // Create a snapshot
269     SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
270       snapshotNameAsString, rootDir, fs, online);
271 
272     if (!online) {
273       admin.enableTable(localTableName);
274     }
275 
276     TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
277 
278     // Clone the snapshot
279     byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
280     admin.cloneSnapshot(snapshotName, cloneTableName);
281 
282     // Verify that region information is the same pre-split
283     original.clearRegionCache();
284     List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
285 
286     final int originalRegionCount = originalTableHRegions.size();
287     final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
288     Assert.assertEquals(
289       "The number of regions in the cloned table is different than in the original table.",
290       originalRegionCount, cloneTableRegionCount);
291 
292     // Split a region on the parent table
293     admin.splitRegion(originalTableHRegions.get(0).getRegionName());
294     waitOnSplit(original, originalRegionCount);
295 
296     // Verify that the cloned table region is not split
297     final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
298     Assert.assertEquals(
299       "The number of regions in the cloned table changed though none of its regions were split.",
300       cloneTableRegionCount, cloneTableRegionCount2);
301   }
302 
303   /*
304    * Take a snapshot of a table, add metadata, and verify that this only
305    * affects one table
306    * @param online - Whether the table is online or not during the snapshot
307    */
308   private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
309     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
310     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
311 
312     // Create a table
313     Admin admin = UTIL.getHBaseAdmin();
314     final long startTime = System.currentTimeMillis();
315     final TableName localTableName =
316         TableName.valueOf(STRING_TABLE_NAME + startTime);
317     HTable original = UTIL.createTable(localTableName, TEST_FAM);
318     UTIL.loadTable(original, TEST_FAM);
319 
320     final String snapshotNameAsString = "snapshot_" + localTableName;
321 
322     // Create a snapshot
323     SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
324       snapshotNameAsString, rootDir, fs, online);
325 
326     if (!online) {
327       admin.enableTable(localTableName);
328     }
329     TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
330 
331     // Clone the snapshot
332     byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
333     admin.cloneSnapshot(snapshotName, cloneTableName);
334 
335     // Add a new column family to the original table
336     byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
337     HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
338 
339     admin.disableTable(localTableName);
340     admin.addColumn(localTableName, hcd);
341 
342     // Verify that it is not in the snapshot
343     admin.enableTable(localTableName);
344 
345     // get a description of the cloned table
346     // get a list of its families
347     // assert that the family is there
348     HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
349     HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
350 
351     Assert.assertTrue("The original family was not found. There is something wrong. ",
352       originalTableDescriptor.hasFamily(TEST_FAM));
353     Assert.assertTrue("The original family was not found in the clone. There is something wrong. ",
354       clonedTableDescriptor.hasFamily(TEST_FAM));
355 
356     Assert.assertTrue("The new family was not found. ",
357       originalTableDescriptor.hasFamily(TEST_FAM_2));
358     Assert.assertTrue("The new family was not found. ",
359       !clonedTableDescriptor.hasFamily(TEST_FAM_2));
360   }
361 }