View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.util.ArrayList;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HColumnDescriptor;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.MediumTests;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
40  import org.apache.hadoop.hbase.regionserver.BloomType;
41  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
42  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.junit.After;
45  import org.junit.AfterClass;
46  import org.junit.Before;
47  import org.junit.BeforeClass;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  /**
52   * Test class to verify that metadata is consistent before and after a snapshot attempt.
53   */
54  @Category(MediumTests.class)
55  public class TestSnapshotMetadata {
56    private static final Log LOG = LogFactory.getLog(TestSnapshotMetadata.class);
57  
58    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
59    private static final int NUM_RS = 2;
60    private static final String STRING_TABLE_NAME = "TestSnapshotMetadata";
61  
62    private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
63    private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
64  
65    private static final String COMPRESSED_FAM_STR = "fam_compressed";
66    private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
67  
68    private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
69    private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
70  
71    private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
72    private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
73  
74    byte[][] families = { MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM };
75  
76    private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
77    private static final BloomType BLOOM_TYPE = BloomType.ROW;
78    private static final int BLOCK_SIZE = 98;
79    private static final int MAX_VERSIONS = 8;
80  
81    HBaseAdmin admin;
82  
83    private String originalTableDescription;
84    private HTableDescriptor originalTableDescriptor;
85    TableName originalTableName;
86  
87    private static FileSystem fs;
88    private static Path rootDir;
89  
90    @BeforeClass
91    public static void setupCluster() throws Exception {
92      setupConf(UTIL.getConfiguration());
93      UTIL.startMiniCluster(NUM_RS);
94  
95      fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
96  
97      rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
98    }
99  
100   @AfterClass
101   public static void cleanupTest() throws Exception {
102     try {
103       UTIL.shutdownMiniCluster();
104     } catch (Exception e) {
105       LOG.warn("failure shutting down cluster", e);
106     }
107   }
108 
109   private static void setupConf(Configuration conf) {
110     // enable snapshot support
111     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
112     // disable the ui
113     conf.setInt("hbase.regionsever.info.port", -1);
114     // change the flush size to a small amount, regulating number of store files
115     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
116     // so make sure we get a compaction when doing a load, but keep around
117     // some files in the store
118     conf.setInt("hbase.hstore.compaction.min", 10);
119     conf.setInt("hbase.hstore.compactionThreshold", 10);
120     // block writes if we get to 12 store files
121     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
122     conf.setInt("hbase.regionserver.msginterval", 100);
123     conf.setBoolean("hbase.master.enabletable.roundrobin", true);
124     // Avoid potentially aggressive splitting which would cause snapshot to fail
125     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
126       ConstantSizeRegionSplitPolicy.class.getName());
127   }
128 
129   @Before
130   public void setup() throws Exception {
131     admin = UTIL.getHBaseAdmin();
132     createTableWithNonDefaultProperties();
133   }
134 
135   @After
136   public void tearDown() throws Exception {
137     SnapshotTestingUtils.deleteAllSnapshots(admin);
138   }
139 
140   /*
141    *  Create a table that has non-default properties so we can see if they hold
142    */
143   private void createTableWithNonDefaultProperties() throws Exception {
144     final long startTime = System.currentTimeMillis();
145     final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
146     originalTableName = TableName.valueOf(sourceTableNameAsString);
147 
148     // enable replication on a column family
149     HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM);
150     HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM);
151     HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM);
152     HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM);
153 
154     maxVersionsColumn.setMaxVersions(MAX_VERSIONS);
155     bloomFilterColumn.setBloomFilterType(BLOOM_TYPE);
156     dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE);
157     blockSizeColumn.setBlocksize(BLOCK_SIZE);
158 
159     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString));
160     htd.addFamily(maxVersionsColumn);
161     htd.addFamily(bloomFilterColumn);
162     htd.addFamily(dataBlockColumn);
163     htd.addFamily(blockSizeColumn);
164 
165     admin.createTable(htd);
166     HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
167     originalTableName = TableName.valueOf(sourceTableNameAsString);
168     originalTableDescriptor = original.getTableDescriptor();
169     originalTableDescription = originalTableDescriptor.toString();
170 
171     original.close();
172   }
173 
174 
175   /**
176    * Verify that the describe for a cloned table matches the describe from the original.
177    */
178   @Test (timeout=300000)
179   public void testDescribeMatchesAfterClone() throws Exception {
180     // Clone the original table
181     final String clonedTableNameAsString = "clone" + originalTableName;
182     final byte[] clonedTableName = Bytes.toBytes(clonedTableNameAsString);
183     final String snapshotNameAsString = "snapshot" + originalTableName
184         + System.currentTimeMillis();
185     final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
186 
187     // restore the snapshot into a cloned table and examine the output
188     List<byte[]> familiesList = new ArrayList<byte[]>();
189     for (byte[] family : families) {
190 
191       familiesList.add(family);
192     }
193 
194     // Create a snapshot in which all families are empty
195     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null,
196       familiesList, snapshotNameAsString, rootDir, fs, /* onlineSnapshot= */ false);
197 
198     admin.cloneSnapshot(snapshotName, clonedTableName);
199     HTable clonedTable = new HTable(UTIL.getConfiguration(), clonedTableName);
200     assertEquals(
201       originalTableDescription.replace(originalTableName.getNameAsString(),clonedTableNameAsString),
202       clonedTable.getTableDescriptor().toString());
203 
204     admin.enableTable(originalTableName);
205     clonedTable.close();
206   }
207 
208   /**
209    * Verify that the describe for a restored table matches the describe for one the original.
210    */
211   @Test (timeout=300000)
212   public void testDescribeMatchesAfterRestore() throws Exception {
213     runRestoreWithAdditionalMetadata(false);
214   }
215 
216   /**
217    * Verify that if metadata changed after a snapshot was taken, that the old metadata replaces the
218    * new metadata during a restore
219    */
220   @Test (timeout=300000)
221   public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
222     runRestoreWithAdditionalMetadata(true);
223   }
224 
225   /**
226    * Verify that when the table is empty, making metadata changes after the restore does not affect
227    * the restored table's original metadata
228    * @throws Exception
229    */
230   @Test (timeout=300000)
231   public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
232     runRestoreWithAdditionalMetadata(true, false);
233   }
234 
235   private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
236     runRestoreWithAdditionalMetadata(changeMetadata, true);
237   }
238 
239   private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
240       throws Exception {
241 
242     if (admin.isTableDisabled(originalTableName)) {
243       admin.enableTable(originalTableName);
244     }
245 
246     // populate it with data
247     final byte[] familyForUpdate = BLOCKSIZE_FAM;
248 
249     List<byte[]> familiesWithDataList = new ArrayList<byte[]>();
250     List<byte[]> emptyFamiliesList = new ArrayList<byte[]>();
251     if (addData) {
252       HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
253       UTIL.loadTable(original, familyForUpdate); // family arbitrarily chosen
254       original.close();
255 
256       for (byte[] family : families) {
257         if (family != familyForUpdate) {
258 
259           emptyFamiliesList.add(family);
260         }
261       }
262       familiesWithDataList.add(familyForUpdate);
263     } else {
264       for (byte[] family : families) {
265         emptyFamiliesList.add(family);
266       }
267     }
268 
269     // take a "disabled" snapshot
270     final String snapshotNameAsString = "snapshot" + originalTableName
271         + System.currentTimeMillis();
272     final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
273 
274     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName,
275       familiesWithDataList, emptyFamiliesList, snapshotNameAsString, rootDir, fs,
276       /* onlineSnapshot= */ false);
277 
278     admin.enableTable(originalTableName);
279 
280     if (changeMetadata) {
281       final String newFamilyNameAsString = "newFamily" + System.currentTimeMillis();
282       final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
283 
284       admin.disableTable(originalTableName);
285       HColumnDescriptor hcd = new HColumnDescriptor(newFamilyName);
286       admin.addColumn(originalTableName, hcd);
287       assertTrue("New column family was not added.",
288         admin.getTableDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
289     }
290     // restore it
291     if (!admin.isTableDisabled(originalTableName)) {
292       admin.disableTable(originalTableName);
293     }
294 
295     admin.restoreSnapshot(snapshotName);
296     admin.enableTable(originalTableName);
297 
298     HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
299 
300     // verify that the descrption is reverted
301     try {
302       assertTrue(originalTableDescriptor.equals(admin.getTableDescriptor(originalTableName)));
303       assertTrue(originalTableDescriptor.equals(original.getTableDescriptor()));
304     } finally {
305       original.close();
306     }
307   }
308 }