1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import java.util.List;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.hbase.TableName;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.HColumnDescriptor;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.HTableDescriptor;
34 import org.apache.hadoop.hbase.testclassification.LargeTests;
35 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
36 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.After;
40 import org.junit.AfterClass;
41 import org.junit.Assert;
42 import org.junit.Before;
43 import org.junit.BeforeClass;
44 import org.junit.Test;
45 import org.junit.experimental.categories.Category;
46
47
48
49
50 @Category(LargeTests.class)
51 public class TestSnapshotCloneIndependence {
52 private static final Log LOG = LogFactory.getLog(TestSnapshotCloneIndependence.class);
53
54 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
55
56 private static final int NUM_RS = 2;
57 private static final String STRING_TABLE_NAME = "test";
58 private static final String TEST_FAM_STR = "fam";
59 private static final byte[] TEST_FAM = Bytes.toBytes(TEST_FAM_STR);
60 private static final TableName TABLE_NAME = TableName.valueOf(STRING_TABLE_NAME);
61
62
63
64
65
66 @BeforeClass
67 public static void setupCluster() throws Exception {
68 setupConf(UTIL.getConfiguration());
69 UTIL.startMiniCluster(NUM_RS);
70 }
71
72 private static void setupConf(Configuration conf) {
73
74 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
75
76 conf.setInt("hbase.regionsever.info.port", -1);
77
78 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
79
80
81 conf.setInt("hbase.hstore.compaction.min", 10);
82 conf.setInt("hbase.hstore.compactionThreshold", 10);
83
84 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
85 conf.setInt("hbase.regionserver.msginterval", 100);
86 conf.setBoolean("hbase.master.enabletable.roundrobin", true);
87
88 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
89 ConstantSizeRegionSplitPolicy.class.getName());
90 }
91
92 @Before
93 public void setup() throws Exception {
94 UTIL.createTable(TABLE_NAME, TEST_FAM);
95 }
96
97 @After
98 public void tearDown() throws Exception {
99 UTIL.deleteTable(TABLE_NAME);
100 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
101 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
102 }
103
104 @AfterClass
105 public static void cleanupTest() throws Exception {
106 try {
107 UTIL.shutdownMiniCluster();
108 } catch (Exception e) {
109 LOG.warn("failure shutting down cluster", e);
110 }
111 }
112
113
114
115
116
117 @Test (timeout=300000)
118 public void testOnlineSnapshotAppendIndependent() throws Exception {
119 runTestSnapshotAppendIndependent(true);
120 }
121
122
123
124
125
126 @Test (timeout=300000)
127 public void testOfflineSnapshotAppendIndependent() throws Exception {
128 runTestSnapshotAppendIndependent(false);
129 }
130
131
132
133
134
135 @Test (timeout=300000)
136 public void testOnlineSnapshotMetadataChangesIndependent() throws Exception {
137 runTestSnapshotMetadataChangesIndependent(true);
138 }
139
140
141
142
143
144 @Test (timeout=300000)
145 public void testOfflineSnapshotMetadataChangesIndependent() throws Exception {
146 runTestSnapshotMetadataChangesIndependent(false);
147 }
148
149
150
151
152
153 @Test (timeout=300000)
154 public void testOfflineSnapshotRegionOperationsIndependent() throws Exception {
155 runTestRegionOperationsIndependent(false);
156 }
157
158
159
160
161
162 @Test (timeout=300000)
163 public void testOnlineSnapshotRegionOperationsIndependent() throws Exception {
164 runTestRegionOperationsIndependent(true);
165 }
166
167 private static void waitOnSplit(final HTable t, int originalCount) throws Exception {
168 for (int i = 0; i < 200; i++) {
169 try {
170 Thread.sleep(50);
171 } catch (InterruptedException e) {
172
173 Thread.currentThread().interrupt();
174 }
175 if (t.getAllRegionLocations().size() > originalCount) {
176 return;
177 }
178 }
179 throw new Exception("Split did not increase the number of regions");
180 }
181
182
183
184
185
186
187 private void runTestSnapshotAppendIndependent(boolean online) throws Exception {
188 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
189 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
190
191 Admin admin = UTIL.getHBaseAdmin();
192 final long startTime = System.currentTimeMillis();
193 final TableName localTableName =
194 TableName.valueOf(STRING_TABLE_NAME + startTime);
195
196 try (Table original = UTIL.createTable(localTableName, TEST_FAM)) {
197 UTIL.loadTable(original, TEST_FAM);
198 final int origTableRowCount = UTIL.countRows(original);
199
200
201 final String snapshotNameAsString = "snapshot_" + localTableName;
202 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
203
204 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
205 snapshotNameAsString, rootDir, fs, online);
206
207 if (!online) {
208 admin.enableTable(localTableName);
209 }
210 TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
211 admin.cloneSnapshot(snapshotName, cloneTableName);
212
213 try (Table clonedTable = new HTable(UTIL.getConfiguration(), cloneTableName)){
214 final int clonedTableRowCount = UTIL.countRows(clonedTable);
215
216 Assert.assertEquals(
217 "The line counts of original and cloned tables do not match after clone. ",
218 origTableRowCount, clonedTableRowCount);
219
220
221 final String rowKey = "new-row-" + System.currentTimeMillis();
222
223 Put p = new Put(Bytes.toBytes(rowKey));
224 p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
225 original.put(p);
226
227
228 Assert.assertEquals("The row count of the original table was not modified by the put",
229 origTableRowCount + 1, UTIL.countRows(original));
230 Assert.assertEquals(
231 "The row count of the cloned table changed as a result of addition to the original",
232 clonedTableRowCount, UTIL.countRows(clonedTable));
233
234 p = new Put(Bytes.toBytes(rowKey));
235 p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
236 clonedTable.put(p);
237
238
239 Assert.assertEquals(
240 "The row count of the original table was modified by the put to the clone",
241 origTableRowCount + 1, UTIL.countRows(original));
242 Assert.assertEquals("The row count of the cloned table was not modified by the put",
243 clonedTableRowCount + 1, UTIL.countRows(clonedTable));
244 }
245 }
246 }
247
248
249
250
251
252 private void runTestRegionOperationsIndependent(boolean online) throws Exception {
253 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
254 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
255
256
257 Admin admin = UTIL.getHBaseAdmin();
258 final long startTime = System.currentTimeMillis();
259 final TableName localTableName =
260 TableName.valueOf(STRING_TABLE_NAME + startTime);
261 HTable original = UTIL.createTable(localTableName, TEST_FAM);
262 UTIL.loadTable(original, TEST_FAM);
263 final int loadedTableCount = UTIL.countRows(original);
264 System.out.println("Original table has: " + loadedTableCount + " rows");
265
266 final String snapshotNameAsString = "snapshot_" + localTableName;
267
268
269 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
270 snapshotNameAsString, rootDir, fs, online);
271
272 if (!online) {
273 admin.enableTable(localTableName);
274 }
275
276 TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
277
278
279 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
280 admin.cloneSnapshot(snapshotName, cloneTableName);
281
282
283 original.clearRegionCache();
284 List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
285
286 final int originalRegionCount = originalTableHRegions.size();
287 final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
288 Assert.assertEquals(
289 "The number of regions in the cloned table is different than in the original table.",
290 originalRegionCount, cloneTableRegionCount);
291
292
293 admin.splitRegion(originalTableHRegions.get(0).getRegionName());
294 waitOnSplit(original, originalRegionCount);
295
296
297 final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
298 Assert.assertEquals(
299 "The number of regions in the cloned table changed though none of its regions were split.",
300 cloneTableRegionCount, cloneTableRegionCount2);
301 }
302
303
304
305
306
307
308 private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
309 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
310 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
311
312
313 Admin admin = UTIL.getHBaseAdmin();
314 final long startTime = System.currentTimeMillis();
315 final TableName localTableName =
316 TableName.valueOf(STRING_TABLE_NAME + startTime);
317 HTable original = UTIL.createTable(localTableName, TEST_FAM);
318 UTIL.loadTable(original, TEST_FAM);
319
320 final String snapshotNameAsString = "snapshot_" + localTableName;
321
322
323 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
324 snapshotNameAsString, rootDir, fs, online);
325
326 if (!online) {
327 admin.enableTable(localTableName);
328 }
329 TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
330
331
332 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
333 admin.cloneSnapshot(snapshotName, cloneTableName);
334
335
336 byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
337 HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
338
339 admin.disableTable(localTableName);
340 admin.addColumn(localTableName, hcd);
341
342
343 admin.enableTable(localTableName);
344
345
346
347
348 HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
349 HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
350
351 Assert.assertTrue("The original family was not found. There is something wrong. ",
352 originalTableDescriptor.hasFamily(TEST_FAM));
353 Assert.assertTrue("The original family was not found in the clone. There is something wrong. ",
354 clonedTableDescriptor.hasFamily(TEST_FAM));
355
356 Assert.assertTrue("The new family was not found. ",
357 originalTableDescriptor.hasFamily(TEST_FAM_2));
358 Assert.assertTrue("The new family was not found. ",
359 !clonedTableDescriptor.hasFamily(TEST_FAM_2));
360 }
361 }