1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23
24 import java.util.ArrayList;
25 import java.util.List;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HColumnDescriptor;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HTableDescriptor;
36 import org.apache.hadoop.hbase.MediumTests;
37 import org.apache.hadoop.hbase.TableName;
38 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
39 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
40 import org.apache.hadoop.hbase.regionserver.BloomType;
41 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
42 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
43 import org.apache.hadoop.hbase.util.Bytes;
44 import org.junit.After;
45 import org.junit.AfterClass;
46 import org.junit.Before;
47 import org.junit.BeforeClass;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51
52
53
54 @Category(MediumTests.class)
55 public class TestSnapshotMetadata {
56 private static final Log LOG = LogFactory.getLog(TestSnapshotMetadata.class);
57
58 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
59 private static final int NUM_RS = 2;
60 private static final String STRING_TABLE_NAME = "TestSnapshotMetadata";
61
62 private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
63 private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
64
65 private static final String COMPRESSED_FAM_STR = "fam_compressed";
66 private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
67
68 private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
69 private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
70
71 private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
72 private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
73
74 byte[][] families = { MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM };
75
76 private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
77 private static final BloomType BLOOM_TYPE = BloomType.ROW;
78 private static final int BLOCK_SIZE = 98;
79 private static final int MAX_VERSIONS = 8;
80
81 HBaseAdmin admin;
82
83 private String originalTableDescription;
84 private HTableDescriptor originalTableDescriptor;
85 TableName originalTableName;
86
87 private static FileSystem fs;
88 private static Path rootDir;
89
90 @BeforeClass
91 public static void setupCluster() throws Exception {
92 setupConf(UTIL.getConfiguration());
93 UTIL.startMiniCluster(NUM_RS);
94
95 fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
96
97 rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
98 }
99
100 @AfterClass
101 public static void cleanupTest() throws Exception {
102 try {
103 UTIL.shutdownMiniCluster();
104 } catch (Exception e) {
105 LOG.warn("failure shutting down cluster", e);
106 }
107 }
108
109 private static void setupConf(Configuration conf) {
110
111 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
112
113 conf.setInt("hbase.regionsever.info.port", -1);
114
115 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
116
117
118 conf.setInt("hbase.hstore.compaction.min", 10);
119 conf.setInt("hbase.hstore.compactionThreshold", 10);
120
121 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
122 conf.setInt("hbase.regionserver.msginterval", 100);
123 conf.setBoolean("hbase.master.enabletable.roundrobin", true);
124
125 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
126 ConstantSizeRegionSplitPolicy.class.getName());
127 }
128
129 @Before
130 public void setup() throws Exception {
131 admin = UTIL.getHBaseAdmin();
132 createTableWithNonDefaultProperties();
133 }
134
135 @After
136 public void tearDown() throws Exception {
137 SnapshotTestingUtils.deleteAllSnapshots(admin);
138 }
139
140
141
142
143 private void createTableWithNonDefaultProperties() throws Exception {
144 final long startTime = System.currentTimeMillis();
145 final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
146 originalTableName = TableName.valueOf(sourceTableNameAsString);
147
148
149 HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM);
150 HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM);
151 HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM);
152 HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM);
153
154 maxVersionsColumn.setMaxVersions(MAX_VERSIONS);
155 bloomFilterColumn.setBloomFilterType(BLOOM_TYPE);
156 dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE);
157 blockSizeColumn.setBlocksize(BLOCK_SIZE);
158
159 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString));
160 htd.addFamily(maxVersionsColumn);
161 htd.addFamily(bloomFilterColumn);
162 htd.addFamily(dataBlockColumn);
163 htd.addFamily(blockSizeColumn);
164
165 admin.createTable(htd);
166 HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
167 originalTableName = TableName.valueOf(sourceTableNameAsString);
168 originalTableDescriptor = original.getTableDescriptor();
169 originalTableDescription = originalTableDescriptor.toString();
170
171 original.close();
172 }
173
174
175
176
177
178 @Test (timeout=300000)
179 public void testDescribeMatchesAfterClone() throws Exception {
180
181 final String clonedTableNameAsString = "clone" + originalTableName;
182 final byte[] clonedTableName = Bytes.toBytes(clonedTableNameAsString);
183 final String snapshotNameAsString = "snapshot" + originalTableName
184 + System.currentTimeMillis();
185 final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
186
187
188 List<byte[]> familiesList = new ArrayList<byte[]>();
189 for (byte[] family : families) {
190
191 familiesList.add(family);
192 }
193
194
195 SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null,
196 familiesList, snapshotNameAsString, rootDir, fs,
197
198 admin.cloneSnapshot(snapshotName, clonedTableName);
199 HTable clonedTable = new HTable(UTIL.getConfiguration(), clonedTableName);
200 assertEquals(
201 originalTableDescription.replace(originalTableName.getNameAsString(),clonedTableNameAsString),
202 clonedTable.getTableDescriptor().toString());
203
204 admin.enableTable(originalTableName);
205 clonedTable.close();
206 }
207
208
209
210
211 @Test (timeout=300000)
212 public void testDescribeMatchesAfterRestore() throws Exception {
213 runRestoreWithAdditionalMetadata(false);
214 }
215
216
217
218
219
220 @Test (timeout=300000)
221 public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
222 runRestoreWithAdditionalMetadata(true);
223 }
224
225
226
227
228
229
230 @Test (timeout=300000)
231 public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
232 runRestoreWithAdditionalMetadata(true, false);
233 }
234
235 private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
236 runRestoreWithAdditionalMetadata(changeMetadata, true);
237 }
238
239 private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
240 throws Exception {
241
242 if (admin.isTableDisabled(originalTableName)) {
243 admin.enableTable(originalTableName);
244 }
245
246
247 final byte[] familyForUpdate = BLOCKSIZE_FAM;
248
249 List<byte[]> familiesWithDataList = new ArrayList<byte[]>();
250 List<byte[]> emptyFamiliesList = new ArrayList<byte[]>();
251 if (addData) {
252 HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
253 UTIL.loadTable(original, familyForUpdate);
254 original.close();
255
256 for (byte[] family : families) {
257 if (family != familyForUpdate) {
258
259 emptyFamiliesList.add(family);
260 }
261 }
262 familiesWithDataList.add(familyForUpdate);
263 } else {
264 for (byte[] family : families) {
265 emptyFamiliesList.add(family);
266 }
267 }
268
269
270 final String snapshotNameAsString = "snapshot" + originalTableName
271 + System.currentTimeMillis();
272 final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
273
274 SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName,
275 familiesWithDataList, emptyFamiliesList, snapshotNameAsString, rootDir, fs,
276
277
278 admin.enableTable(originalTableName);
279
280 if (changeMetadata) {
281 final String newFamilyNameAsString = "newFamily" + System.currentTimeMillis();
282 final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
283
284 admin.disableTable(originalTableName);
285 HColumnDescriptor hcd = new HColumnDescriptor(newFamilyName);
286 admin.addColumn(originalTableName, hcd);
287 assertTrue("New column family was not added.",
288 admin.getTableDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
289 }
290
291 if (!admin.isTableDisabled(originalTableName)) {
292 admin.disableTable(originalTableName);
293 }
294
295 admin.restoreSnapshot(snapshotName);
296 admin.enableTable(originalTableName);
297
298 HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
299
300
301 try {
302 assertTrue(originalTableDescriptor.equals(admin.getTableDescriptor(originalTableName)));
303 assertTrue(originalTableDescriptor.equals(original.getTableDescriptor()));
304 } finally {
305 original.close();
306 }
307 }
308 }