1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import java.util.List;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.hbase.TableName;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.HColumnDescriptor;
31 import org.apache.hadoop.hbase.HConstants;
32 import org.apache.hadoop.hbase.HRegionInfo;
33 import org.apache.hadoop.hbase.HTableDescriptor;
34 import org.apache.hadoop.hbase.testclassification.LargeTests;
35 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
36 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
37 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.junit.After;
40 import org.junit.AfterClass;
41 import org.junit.Assert;
42 import org.junit.Before;
43 import org.junit.BeforeClass;
44 import org.junit.Test;
45 import org.junit.experimental.categories.Category;
46
47
48
49
50 @Category(LargeTests.class)
51 public class TestSnapshotCloneIndependence {
52 private static final Log LOG = LogFactory.getLog(TestSnapshotCloneIndependence.class);
53
54 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
55
56 private static final int NUM_RS = 2;
57 private static final String STRING_TABLE_NAME = "test";
58 private static final String TEST_FAM_STR = "fam";
59 private static final byte[] TEST_FAM = Bytes.toBytes(TEST_FAM_STR);
60
61 private static final TableName TABLE_NAME = TableName.valueOf(STRING_TABLE_NAME);
62 private static final int CLEANER_INTERVAL = 10;
63
64
65
66
67
68 @BeforeClass
69 public static void setupCluster() throws Exception {
70 setupConf(UTIL.getConfiguration());
71 UTIL.startMiniCluster(NUM_RS);
72 }
73
74 private static void setupConf(Configuration conf) {
75
76 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
77
78 conf.setInt("hbase.regionsever.info.port", -1);
79
80 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
81
82
83 conf.setInt("hbase.hstore.compaction.min", 10);
84 conf.setInt("hbase.hstore.compactionThreshold", 10);
85
86 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
87 conf.setInt("hbase.regionserver.msginterval", 100);
88 conf.setBoolean("hbase.master.enabletable.roundrobin", true);
89
90 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
91 ConstantSizeRegionSplitPolicy.class.getName());
92
93 conf.setInt("hbase.master.cleaner.interval", CLEANER_INTERVAL);
94 conf.setInt("hbase.master.hfilecleaner.plugins.snapshot.period", CLEANER_INTERVAL);
95
96
97
98 conf.setInt("hbase.master.hfilecleaner.ttl", CLEANER_INTERVAL);
99 }
100
101 @Before
102 public void setup() throws Exception {
103 UTIL.createTable(TABLE_NAME, TEST_FAM);
104 }
105
106 @After
107 public void tearDown() throws Exception {
108 UTIL.deleteTable(TABLE_NAME);
109 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
110 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
111 }
112
113 @AfterClass
114 public static void cleanupTest() throws Exception {
115 try {
116 UTIL.shutdownMiniCluster();
117 } catch (Exception e) {
118 LOG.warn("failure shutting down cluster", e);
119 }
120 }
121
122
123
124
125
126 @Test (timeout=300000)
127 public void testOnlineSnapshotAppendIndependent() throws Exception {
128 runTestSnapshotAppendIndependent(true);
129 }
130
131
132
133
134
135 @Test (timeout=300000)
136 public void testOfflineSnapshotAppendIndependent() throws Exception {
137 runTestSnapshotAppendIndependent(false);
138 }
139
140
141
142
143
144 @Test (timeout=300000)
145 public void testOnlineSnapshotMetadataChangesIndependent() throws Exception {
146 runTestSnapshotMetadataChangesIndependent(true);
147 }
148
149
150
151
152
153 @Test (timeout=300000)
154 public void testOfflineSnapshotMetadataChangesIndependent() throws Exception {
155 runTestSnapshotMetadataChangesIndependent(false);
156 }
157
158
159
160
161
162 @Test (timeout=300000)
163 public void testOfflineSnapshotRegionOperationsIndependent() throws Exception {
164 runTestRegionOperationsIndependent(false);
165 }
166
167
168
169
170
171 @Test (timeout=300000)
172 public void testOnlineSnapshotRegionOperationsIndependent() throws Exception {
173 runTestRegionOperationsIndependent(true);
174 }
175
176 @Test (timeout=300000)
177 public void testOfflineSnapshotDeleteIndependent() throws Exception {
178 runTestSnapshotDeleteIndependent(false);
179 }
180
181 @Test (timeout=300000)
182 public void testOnlineSnapshotDeleteIndependent() throws Exception {
183 runTestSnapshotDeleteIndependent(true);
184 }
185
186 private static void waitOnSplit(final HTable t, int originalCount) throws Exception {
187 for (int i = 0; i < 200; i++) {
188 try {
189 Thread.sleep(50);
190 } catch (InterruptedException e) {
191
192 Thread.currentThread().interrupt();
193 }
194 if (t.getRegionLocations().size() > originalCount) {
195 return;
196 }
197 }
198 throw new Exception("Split did not increase the number of regions");
199 }
200
201
202
203
204
205
206 private void runTestSnapshotAppendIndependent(boolean online) throws Exception {
207 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
208 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
209
210 HBaseAdmin admin = UTIL.getHBaseAdmin();
211 final long startTime = System.currentTimeMillis();
212 final TableName localTableName =
213 TableName.valueOf(STRING_TABLE_NAME + startTime);
214
215 HTable original = UTIL.createTable(localTableName, TEST_FAM);
216 try {
217
218 UTIL.loadTable(original, TEST_FAM);
219 final int origTableRowCount = UTIL.countRows(original);
220
221
222 final String snapshotNameAsString = "snapshot_" + localTableName;
223 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
224
225 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
226 snapshotNameAsString, rootDir, fs, online);
227
228 if (!online) {
229 admin.enableTable(localTableName);
230 }
231 byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
232 admin.cloneSnapshot(snapshotName, cloneTableName);
233
234 HTable clonedTable = new HTable(UTIL.getConfiguration(), cloneTableName);
235
236 try {
237 final int clonedTableRowCount = UTIL.countRows(clonedTable);
238
239 Assert.assertEquals(
240 "The line counts of original and cloned tables do not match after clone. ",
241 origTableRowCount, clonedTableRowCount);
242
243
244 final String rowKey = "new-row-" + System.currentTimeMillis();
245
246 Put p = new Put(Bytes.toBytes(rowKey));
247 p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
248 original.put(p);
249 original.flushCommits();
250
251
252 Assert.assertEquals("The row count of the original table was not modified by the put",
253 origTableRowCount + 1, UTIL.countRows(original));
254 Assert.assertEquals(
255 "The row count of the cloned table changed as a result of addition to the original",
256 clonedTableRowCount, UTIL.countRows(clonedTable));
257
258 p = new Put(Bytes.toBytes(rowKey));
259 p.add(TEST_FAM, Bytes.toBytes("someQualifier"), Bytes.toBytes("someString"));
260 clonedTable.put(p);
261 clonedTable.flushCommits();
262
263
264 Assert.assertEquals(
265 "The row count of the original table was modified by the put to the clone",
266 origTableRowCount + 1, UTIL.countRows(original));
267 Assert.assertEquals("The row count of the cloned table was not modified by the put",
268 clonedTableRowCount + 1, UTIL.countRows(clonedTable));
269 } finally {
270
271 clonedTable.close();
272 }
273 } finally {
274
275 original.close();
276 }
277 }
278
279
280
281
282
283 private void runTestRegionOperationsIndependent(boolean online) throws Exception {
284 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
285 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
286
287
288 HBaseAdmin admin = UTIL.getHBaseAdmin();
289 final long startTime = System.currentTimeMillis();
290 final TableName localTableName =
291 TableName.valueOf(STRING_TABLE_NAME + startTime);
292 HTable original = UTIL.createTable(localTableName, TEST_FAM);
293 UTIL.loadTable(original, TEST_FAM);
294 final int loadedTableCount = UTIL.countRows(original);
295 System.out.println("Original table has: " + loadedTableCount + " rows");
296
297 final String snapshotNameAsString = "snapshot_" + localTableName;
298
299
300 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
301 snapshotNameAsString, rootDir, fs, online);
302
303 if (!online) {
304 admin.enableTable(localTableName);
305 }
306
307 byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
308
309
310 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
311 admin.cloneSnapshot(snapshotName, cloneTableName);
312
313
314 original.clearRegionCache();
315 List<HRegionInfo> originalTableHRegions = admin.getTableRegions(localTableName);
316
317 final int originalRegionCount = originalTableHRegions.size();
318 final int cloneTableRegionCount = admin.getTableRegions(cloneTableName).size();
319 Assert.assertEquals(
320 "The number of regions in the cloned table is different than in the original table.",
321 originalRegionCount, cloneTableRegionCount);
322
323
324 admin.split(originalTableHRegions.get(0).getRegionName());
325 waitOnSplit(original, originalRegionCount);
326
327
328 final int cloneTableRegionCount2 = admin.getTableRegions(cloneTableName).size();
329 Assert.assertEquals(
330 "The number of regions in the cloned table changed though none of its regions were split.",
331 cloneTableRegionCount, cloneTableRegionCount2);
332 }
333
334
335
336
337
338
339 private void runTestSnapshotMetadataChangesIndependent(boolean online) throws Exception {
340 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
341 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
342
343
344 HBaseAdmin admin = UTIL.getHBaseAdmin();
345 final long startTime = System.currentTimeMillis();
346 final TableName localTableName =
347 TableName.valueOf(STRING_TABLE_NAME + startTime);
348 HTable original = UTIL.createTable(localTableName, TEST_FAM);
349 UTIL.loadTable(original, TEST_FAM);
350
351 final String snapshotNameAsString = "snapshot_" + localTableName;
352
353
354 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
355 snapshotNameAsString, rootDir, fs, online);
356
357 if (!online) {
358 admin.enableTable(localTableName);
359 }
360 byte[] cloneTableName = Bytes.toBytes("test-clone-" + localTableName);
361
362
363 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
364 admin.cloneSnapshot(snapshotName, cloneTableName);
365
366
367 byte[] TEST_FAM_2 = Bytes.toBytes("fam2");
368 HColumnDescriptor hcd = new HColumnDescriptor(TEST_FAM_2);
369
370 admin.disableTable(localTableName);
371 admin.addColumn(localTableName, hcd);
372
373
374 admin.enableTable(localTableName);
375
376
377
378
379 HTableDescriptor originalTableDescriptor = original.getTableDescriptor();
380 HTableDescriptor clonedTableDescriptor = admin.getTableDescriptor(cloneTableName);
381
382 Assert.assertTrue("The original family was not found. There is something wrong. ",
383 originalTableDescriptor.hasFamily(TEST_FAM));
384 Assert.assertTrue("The original family was not found in the clone. There is something wrong. ",
385 clonedTableDescriptor.hasFamily(TEST_FAM));
386
387 Assert.assertTrue("The new family was not found. ",
388 originalTableDescriptor.hasFamily(TEST_FAM_2));
389 Assert.assertTrue("The new family was not found. ",
390 !clonedTableDescriptor.hasFamily(TEST_FAM_2));
391 }
392
393
394
395
396
397
398 private void runTestSnapshotDeleteIndependent(boolean online) throws Exception {
399 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
400 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
401
402 final HBaseAdmin admin = UTIL.getHBaseAdmin();
403 final long startTime = System.currentTimeMillis();
404 final TableName localTableName =
405 TableName.valueOf(STRING_TABLE_NAME + startTime);
406
407 HTable original = UTIL.createTable(localTableName, TEST_FAM);
408 try {
409 UTIL.loadTable(original, TEST_FAM);
410 } finally {
411 original.close();
412 }
413
414
415 final String snapshotNameAsString = "snapshot_" + localTableName;
416 byte[] snapshotName = Bytes.toBytes(snapshotNameAsString);
417
418 SnapshotTestingUtils.createSnapshotAndValidate(admin, localTableName, TEST_FAM_STR,
419 snapshotNameAsString, rootDir, fs, online);
420
421 if (!online) {
422 admin.enableTable(localTableName);
423 }
424 TableName cloneTableName = TableName.valueOf("test-clone-" + localTableName);
425 admin.cloneSnapshot(snapshotName, cloneTableName);
426
427
428 admin.majorCompact(localTableName.getNameAsString());
429
430
431 admin.deleteSnapshot(snapshotName);
432
433
434 Thread.sleep(10000);
435
436 original = new HTable(UTIL.getConfiguration(), localTableName);
437 try {
438 HTable clonedTable = new HTable(UTIL.getConfiguration(), cloneTableName);
439 try {
440
441 final int origTableRowCount = UTIL.countRows(original);
442 final int clonedTableRowCount = UTIL.countRows(clonedTable);
443 Assert.assertEquals(origTableRowCount, clonedTableRowCount);
444 } finally {
445 clonedTable.close();
446 }
447 } finally {
448 original.close();
449 }
450 }
451 }