1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import java.util.ArrayList;
22 import java.util.List;
23
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.FileSystem;
26 import org.apache.hadoop.fs.Path;
27 import org.apache.hadoop.hbase.TableName;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HTableDescriptor;
32 import org.apache.hadoop.hbase.MediumTests;
33 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
34 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
35 import org.apache.hadoop.hbase.regionserver.BloomType;
36 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.After;
40 import org.junit.Assert;
41 import org.junit.Before;
42 import org.junit.BeforeClass;
43 import org.junit.Test;
44 import org.junit.experimental.categories.Category;
45
46
47
48
49 @Category(MediumTests.class)
50 public class TestSnapshotMetadata {
51
52 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
53 private static final int NUM_RS = 2;
54 private static final String STRING_TABLE_NAME = "testtable";
55
56 private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
57 private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
58
59 private static final String COMPRESSED_FAM_STR = "fam_compressed";
60 private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
61
62 private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
63 private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
64
65 private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
66 private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
67
68 byte[][] families = { MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM };
69
70 private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
71 private static final BloomType BLOOM_TYPE = BloomType.ROW;
72 private static final int BLOCK_SIZE = 98;
73 private static final int MAX_VERSIONS = 8;
74
75 HBaseAdmin admin;
76
77 private String originalTableDescription;
78 private HTableDescriptor originalTableDescriptor;
79 TableName originalTableName;
80
81 private static FileSystem fs;
82 private static Path rootDir;
83
84 @BeforeClass
85 public static void setupCluster() throws Exception {
86 setupConf(UTIL.getConfiguration());
87 UTIL.startMiniCluster(NUM_RS);
88
89 fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
90
91 rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
92 }
93
94 private static void setupConf(Configuration conf) {
95
96 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
97
98 conf.setInt("hbase.regionsever.info.port", -1);
99
100 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
101
102
103 conf.setInt("hbase.hstore.compaction.min", 10);
104 conf.setInt("hbase.hstore.compactionThreshold", 10);
105
106 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
107 conf.setInt("hbase.regionserver.msginterval", 100);
108 conf.setBoolean("hbase.master.enabletable.roundrobin", true);
109
110 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
111 ConstantSizeRegionSplitPolicy.class.getName());
112 }
113
114 @Before
115 public void setup() throws Exception {
116 createTableWithNonDefaultProperties();
117 }
118
119 @After
120 public void tearDown() throws Exception {
121 admin.close();
122 }
123
124
125
126
127 private void createTableWithNonDefaultProperties() throws Exception {
128
129 admin = new HBaseAdmin(UTIL.getConfiguration());
130
131 final long startTime = System.currentTimeMillis();
132 final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
133 originalTableName = TableName.valueOf(sourceTableNameAsString);
134
135
136 HColumnDescriptor maxVersionsColumn = new HColumnDescriptor(MAX_VERSIONS_FAM);
137 HColumnDescriptor bloomFilterColumn = new HColumnDescriptor(BLOOMFILTER_FAM);
138 HColumnDescriptor dataBlockColumn = new HColumnDescriptor(COMPRESSED_FAM);
139 HColumnDescriptor blockSizeColumn = new HColumnDescriptor(BLOCKSIZE_FAM);
140
141 maxVersionsColumn.setMaxVersions(MAX_VERSIONS);
142 bloomFilterColumn.setBloomFilterType(BLOOM_TYPE);
143 dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE);
144 blockSizeColumn.setBlocksize(BLOCK_SIZE);
145
146 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(sourceTableNameAsString));
147 htd.addFamily(maxVersionsColumn);
148 htd.addFamily(bloomFilterColumn);
149 htd.addFamily(dataBlockColumn);
150 htd.addFamily(blockSizeColumn);
151
152 admin.createTable(htd);
153 HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
154 originalTableName = TableName.valueOf(sourceTableNameAsString);
155 originalTableDescriptor = original.getTableDescriptor();
156 originalTableDescription = originalTableDescriptor.toString();
157
158 original.close();
159 }
160
161
162
163
164
165 @Test (timeout=300000)
166 public void testDescribeMatchesAfterClone() throws Exception {
167
168 final String clonedTableNameAsString = "clone" + originalTableName;
169 final byte[] clonedTableName = Bytes.toBytes(clonedTableNameAsString);
170 final String snapshotNameAsString = "snapshot" + originalTableName
171 + System.currentTimeMillis();
172 final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
173
174
175 List<byte[]> familiesList = new ArrayList<byte[]>();
176 for (byte[] family : families) {
177
178 familiesList.add(family);
179 }
180
181
182 SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null,
183 familiesList, snapshotNameAsString, rootDir, fs);
184
185 admin.cloneSnapshot(snapshotName, clonedTableName);
186 HTable clonedTable = new HTable(UTIL.getConfiguration(), clonedTableName);
187 Assert.assertEquals(
188 originalTableDescription.replace(originalTableName.getNameAsString(),clonedTableNameAsString),
189 clonedTable.getTableDescriptor().toString());
190
191 admin.enableTable(originalTableName);
192 clonedTable.close();
193 }
194
195
196
197
198 @Test (timeout=300000)
199 public void testDescribeMatchesAfterRestore() throws Exception {
200 runRestoreWithAdditionalMetadata(false);
201 }
202
203
204
205
206
207 @Test (timeout=300000)
208 public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
209 runRestoreWithAdditionalMetadata(true);
210 }
211
212
213
214
215
216
217 @Test (timeout=300000)
218 public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
219 runRestoreWithAdditionalMetadata(true, false);
220 }
221
222 private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
223 runRestoreWithAdditionalMetadata(changeMetadata, true);
224 }
225
226 private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
227 throws Exception {
228
229 if (admin.isTableDisabled(originalTableName)) {
230 admin.enableTable(originalTableName);
231 }
232
233
234 final byte[] familyForUpdate = BLOCKSIZE_FAM;
235
236 List<byte[]> familiesWithDataList = new ArrayList<byte[]>();
237 List<byte[]> emptyFamiliesList = new ArrayList<byte[]>();
238 if (addData) {
239 HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
240 UTIL.loadTable(original, familyForUpdate);
241 original.close();
242
243 for (byte[] family : families) {
244 if (family != familyForUpdate) {
245
246 emptyFamiliesList.add(family);
247 }
248 }
249 familiesWithDataList.add(familyForUpdate);
250 } else {
251 for (byte[] family : families) {
252 emptyFamiliesList.add(family);
253 }
254 }
255
256
257 final String snapshotNameAsString = "snapshot" + originalTableName
258 + System.currentTimeMillis();
259 final byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
260
261 SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName,
262 familiesWithDataList, emptyFamiliesList, snapshotNameAsString, rootDir, fs);
263
264 admin.enableTable(originalTableName);
265
266 if (changeMetadata) {
267 final String newFamilyNameAsString = "newFamily" + System.currentTimeMillis();
268 final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
269
270 admin.disableTable(originalTableName);
271 HColumnDescriptor hcd = new HColumnDescriptor(newFamilyName);
272 admin.addColumn(originalTableName, hcd);
273 Assert.assertTrue("New column family was not added.",
274 admin.getTableDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
275 }
276
277 if (!admin.isTableDisabled(originalTableName)) {
278 admin.disableTable(originalTableName);
279 }
280
281 admin.restoreSnapshot(snapshotName);
282 admin.enableTable(originalTableName);
283
284 HTable original = new HTable(UTIL.getConfiguration(), originalTableName);
285
286
287 try {
288 Assert
289 .assertTrue(originalTableDescriptor.equals(admin.getTableDescriptor(originalTableName)));
290 Assert.assertTrue(originalTableDescriptor.equals(original.getTableDescriptor()));
291 } finally {
292 original.close();
293 }
294 }
295 }