View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.util.ArrayList;
22  import java.util.List;
23  
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FileSystem;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.TableName;
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.MediumTests;
33  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
34  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
35  import org.apache.hadoop.hbase.regionserver.BloomType;
36  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37  import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
38  import org.apache.hadoop.hbase.util.Bytes;
39  import org.junit.After;
40  import org.junit.Assert;
41  import org.junit.Before;
42  import org.junit.BeforeClass;
43  import org.junit.Test;
44  import org.junit.experimental.categories.Category;
45  
46  /**
47   * Test class to verify that metadata is consistent before and after a snapshot attempt.
48   */
49  @Category(MediumTests.class)
50  public class TestSnapshotMetadata {
51  
52    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
53    private static final int NUM_RS = 2;
54    private static final String STRING_TABLE_NAME = "testtable";
55  
56    private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
57    private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
58  
59    private static final String COMPRESSED_FAM_STR = "fam_compressed";
60    private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
61  
62    private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
63    private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
64  
65    private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
66    private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
67  
68    byte[][] families = { MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM };
69  
70    private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
71    private static final BloomType BLOOM_TYPE = BloomType.ROW;
72    private static final int BLOCK_SIZE = 98;
73    private static final int MAX_VERSIONS = 8;
74  
75    HBaseAdmin admin;
76  
77    private String originalTableDescription;
78    private HTableDescriptor originalTableDescriptor;
79    TableName originalTableName;
80  
81    private static FileSystem fs;
82    private static Path rootDir;
83  
84    @BeforeClass
85    public static void setupCluster() throws Exception {
86      setupConf(UTIL.getConfiguration());
87      UTIL.startMiniCluster(NUM_RS);
88  
89      fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
90  
91      rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
92    }
93  
94    private static void setupConf(Configuration conf) {
95      // enable snapshot support
96      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
97      // disable the ui
98      conf.setInt("hbase.regionsever.info.port", -1);
99      // change the flush size to a small amount, regulating number of store files
100     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
101     // so make sure we get a compaction when doing a load, but keep around
102     // some files in the store
103     conf.setInt("hbase.hstore.compaction.min", 10);
104     conf.setInt("hbase.hstore.compactionThreshold", 10);
105     // block writes if we get to 12 store files
106     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
107     conf.setInt("hbase.regionserver.msginterval", 100);
108     conf.setBoolean("hbase.master.enabletable.roundrobin", true);
109     // Avoid potentially aggressive splitting which would cause snapshot to fail
110     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
111       ConstantSizeRegionSplitPolicy.class.getName());
112   }
113 
114   @Before
115   public void setup() throws Exception {
116     createTableWithNonDefaultProperties();
117   }
118 
119   @After
120   public void tearDown() throws Exception {
121     admin.close();
122   }
123 
124   /*
125    *  Create a table that has non-default properties so we can see if they hold
126    */
127   private void createTableWithNonDefaultProperties() throws Exception {
128     // create a table
129     admin = new HBaseAdmin(UTIL.getConfiguration());
130 
131     final long startTime = System.currentTimeMillis();
132     final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
133     originalTableName = TableName.valueOf(sourceTableNameAsString);
134 
135     // enable replication on a column family
136     HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM);
137     HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM);
138     HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM);
139     HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM);
140 
141     maxVersionsColumn.setMaxVersions(MAX_VERSIONS);
142     bloomFilterColumn.setBloomFilterType(BLOOM_TYPE);
143     dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE);
144     blockSizeColumn.setBlocksize(BLOCK_SIZE);
145 
146     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString));
147     htd.addFamily(maxVersionsColumn);
148     htd.addFamily(bloomFilterColumn);
149     htd.addFamily(dataBlockColumn);
150     htd.addFamily(blockSizeColumn);
151 
152     admin.createTable(htd);
153     HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
154     originalTableName = TableName.valueOf(sourceTableNameAsString);
155     originalTableDescriptor = original.getTableDescriptor();
156     originalTableDescription = originalTableDescriptor.toString();
157 
158     original.close();
159   }
160 
161 
162   /**
163    * Verify that the describe for a cloned table matches the describe from the original.
164    */
165   @Test (timeout=300000)
166   public void testDescribeMatchesAfterClone() throws Exception {
167     // Clone the original table
168     final String clonedTableNameAsString = "clone" + originalTableName;
169     final byte[] clonedTableName = Bytes.toBytes(clonedTableNameAsString);
170     final String snapshotNameAsString = "snapshot" + originalTableName
171         + System.currentTimeMillis();
172     final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
173 
174     // restore the snapshot into a cloned table and examine the output
175     List<byte[]> familiesList = new ArrayList<byte[]>();
176     for (byte[] family : families) {
177 
178       familiesList.add(family);
179     }
180 
181     // Create a snapshot in which all families are empty
182     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null,
183       familiesList, snapshotNameAsString, rootDir, fs);
184 
185     admin.cloneSnapshot(snapshotName, clonedTableName);
186     HTable clonedTable = new HTable(UTIL.getConfiguration(), clonedTableName);
187     Assert.assertEquals(
188       originalTableDescription.replace(originalTableName.getNameAsString(),clonedTableNameAsString),
189       clonedTable.getTableDescriptor().toString());
190 
191     admin.enableTable(originalTableName);
192     clonedTable.close();
193   }
194 
195   /**
196    * Verify that the describe for a restored table matches the describe for one the original.
197    */
198   @Test (timeout=300000)
199   public void testDescribeMatchesAfterRestore() throws Exception {
200     runRestoreWithAdditionalMetadata(false);
201   }
202 
203   /**
204    * Verify that if metadata changed after a snapshot was taken, that the old metadata replaces the
205    * new metadata during a restore
206    */
207   @Test (timeout=300000)
208   public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
209     runRestoreWithAdditionalMetadata(true);
210   }
211 
212   /**
213    * Verify that when the table is empty, making metadata changes after the restore does not affect
214    * the restored table's original metadata
215    * @throws Exception
216    */
217   @Test (timeout=300000)
218   public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
219     runRestoreWithAdditionalMetadata(true, false);
220   }
221 
222   private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
223     runRestoreWithAdditionalMetadata(changeMetadata, true);
224   }
225 
226   private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
227       throws Exception {
228 
229     if (admin.isTableDisabled(originalTableName)) {
230       admin.enableTable(originalTableName);
231     }
232 
233     // populate it with data
234     final byte[] familyForUpdate = BLOCKSIZE_FAM;
235 
236     List<byte[]> familiesWithDataList = new ArrayList<byte[]>();
237     List<byte[]> emptyFamiliesList = new ArrayList<byte[]>();
238     if (addData) {
239       HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
240       UTIL.loadTable(original, familyForUpdate); // family arbitrarily chosen
241       original.close();
242 
243       for (byte[] family : families) {
244         if (family != familyForUpdate) {
245 
246           emptyFamiliesList.add(family);
247         }
248       }
249       familiesWithDataList.add(familyForUpdate);
250     } else {
251       for (byte[] family : families) {
252         emptyFamiliesList.add(family);
253       }
254     }
255 
256     // take a snapshot
257     final String snapshotNameAsString = "snapshot" + originalTableName
258         + System.currentTimeMillis();
259     final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
260 
261     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName,
262       familiesWithDataList, emptyFamiliesList, snapshotNameAsString, rootDir, fs);
263 
264     admin.enableTable(originalTableName);
265 
266     if (changeMetadata) {
267       final String newFamilyNameAsString = "newFamily" + System.currentTimeMillis();
268       final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
269 
270       admin.disableTable(originalTableName);
271       HColumnDescriptor hcd = new HColumnDescriptor(newFamilyName);
272       admin.addColumn(originalTableName, hcd);
273       Assert.assertTrue("New column family was not added.",
274         admin.getTableDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
275     }
276     // restore it
277     if (!admin.isTableDisabled(originalTableName)) {
278       admin.disableTable(originalTableName);
279     }
280 
281     admin.restoreSnapshot(snapshotName);
282     admin.enableTable(originalTableName);
283 
284     HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
285 
286     // verify that the descrption is reverted
287     try {
288       Assert
289           .assertTrue(originalTableDescriptor.equals(admin.getTableDescriptor(originalTableName)));
290       Assert.assertTrue(originalTableDescriptor.equals(original.getTableDescriptor()));
291     } finally {
292       original.close();
293     }
294   }
295 }