1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.Collections;
26 import java.util.Comparator;
27 import java.util.HashMap;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.concurrent.CountDownLatch;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HConstants;
39 import org.apache.hadoop.hbase.HRegionInfo;
40 import org.apache.hadoop.hbase.client.HTable;
41 import org.apache.hadoop.hbase.testclassification.LargeTests;
42 import org.apache.hadoop.hbase.TableName;
43 import org.apache.hadoop.hbase.TableNotFoundException;
44 import org.apache.hadoop.hbase.client.Admin;
45 import org.apache.hadoop.hbase.client.Table;
46 import org.apache.hadoop.hbase.master.HMaster;
47 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
48 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
49 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.FSUtils;
52 import org.junit.After;
53 import org.junit.AfterClass;
54 import org.junit.Before;
55 import org.junit.BeforeClass;
56 import org.junit.Test;
57 import org.junit.experimental.categories.Category;
58
59
60
61
62
63
64
65
66
67 @Category(LargeTests.class)
68 public class TestFlushSnapshotFromClient {
69 private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
70 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
71 private static final int NUM_RS = 2;
72 private static final byte[] TEST_FAM = Bytes.toBytes("fam");
73 private static final TableName TABLE_NAME = TableName.valueOf("test");
74 private final int DEFAULT_NUM_ROWS = 100;
75
76
77
78
79
80 @BeforeClass
81 public static void setupCluster() throws Exception {
82
83
84
85
86
87 setupConf(UTIL.getConfiguration());
88 UTIL.startMiniCluster(NUM_RS);
89 }
90
91 private static void setupConf(Configuration conf) {
92
93 conf.setInt("hbase.regionsever.info.port", -1);
94
95 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
96
97
98 conf.setInt("hbase.hstore.compaction.min", 10);
99 conf.setInt("hbase.hstore.compactionThreshold", 10);
100
101 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
102
103 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
104 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
105 ConstantSizeRegionSplitPolicy.class.getName());
106 }
107
108 @Before
109 public void setup() throws Exception {
110 SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
111 }
112
113 @After
114 public void tearDown() throws Exception {
115 UTIL.deleteTable(TABLE_NAME);
116
117 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
118 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
119 }
120
121 @AfterClass
122 public static void cleanupTest() throws Exception {
123 try {
124 UTIL.shutdownMiniCluster();
125 } catch (Exception e) {
126 LOG.warn("failure shutting down cluster", e);
127 }
128 }
129
130
131
132
133
134 @Test (timeout=300000)
135 public void testFlushTableSnapshot() throws Exception {
136 Admin admin = UTIL.getHBaseAdmin();
137
138 SnapshotTestingUtils.assertNoSnapshots(admin);
139
140
141 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
142
143 LOG.debug("FS state before snapshot:");
144 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
145 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
146
147
148 String snapshotString = "offlineTableSnapshot";
149 byte[] snapshot = Bytes.toBytes(snapshotString);
150 admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.FLUSH);
151 LOG.debug("Snapshot completed.");
152
153
154 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
155 snapshot, TABLE_NAME);
156
157
158 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
159 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
160 LOG.debug("FS state after snapshot:");
161 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
162 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
163
164 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
165 admin, fs);
166 }
167
168
169
170
171
172 @Test(timeout=30000)
173 public void testSkipFlushTableSnapshot() throws Exception {
174 Admin admin = UTIL.getHBaseAdmin();
175
176 SnapshotTestingUtils.assertNoSnapshots(admin);
177
178
179 try (Table table = UTIL.getConnection().getTable(TABLE_NAME)) {
180 UTIL.loadTable(table, TEST_FAM);
181 }
182
183 LOG.debug("FS state before snapshot:");
184 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
185 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
186
187
188 String snapshotString = "skipFlushTableSnapshot";
189 byte[] snapshot = Bytes.toBytes(snapshotString);
190 admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
191 LOG.debug("Snapshot completed.");
192
193
194 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
195 snapshot, TABLE_NAME);
196
197
198 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
199 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
200 LOG.debug("FS state after snapshot:");
201 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
202 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
203
204 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
205 admin, fs);
206
207 admin.deleteSnapshot(snapshot);
208 snapshots = admin.listSnapshots();
209 SnapshotTestingUtils.assertNoSnapshots(admin);
210 }
211
212
213
214
215
216
217 @Test (timeout=300000)
218 public void testFlushTableSnapshotWithProcedure() throws Exception {
219 Admin admin = UTIL.getHBaseAdmin();
220
221 SnapshotTestingUtils.assertNoSnapshots(admin);
222
223
224 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
225
226 LOG.debug("FS state before snapshot:");
227 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
228 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
229
230
231 String snapshotString = "offlineTableSnapshot";
232 byte[] snapshot = Bytes.toBytes(snapshotString);
233 Map<String, String> props = new HashMap<String, String>();
234 props.put("table", TABLE_NAME.getNameAsString());
235 admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
236 snapshotString, props);
237
238
239 LOG.debug("Snapshot completed.");
240
241
242 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
243 snapshot, TABLE_NAME);
244
245
246 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
247 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
248 LOG.debug("FS state after snapshot:");
249 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
250 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
251
252 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
253 admin, fs);
254 }
255
256 @Test (timeout=300000)
257 public void testSnapshotFailsOnNonExistantTable() throws Exception {
258 Admin admin = UTIL.getHBaseAdmin();
259
260 SnapshotTestingUtils.assertNoSnapshots(admin);
261 TableName tableName = TableName.valueOf("_not_a_table");
262
263
264 boolean fail = false;
265 do {
266 try {
267 admin.getTableDescriptor(tableName);
268 fail = true;
269 LOG.error("Table:" + tableName + " already exists, checking a new name");
270 tableName = TableName.valueOf(tableName+"!");
271 } catch (TableNotFoundException e) {
272 fail = false;
273 }
274 } while (fail);
275
276
277 try {
278 admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
279 fail("Snapshot succeeded even though there is not table.");
280 } catch (SnapshotCreationException e) {
281 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
282 }
283 }
284
285 @Test(timeout = 300000)
286 public void testAsyncFlushSnapshot() throws Exception {
287 Admin admin = UTIL.getHBaseAdmin();
288 SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
289 .setTable(TABLE_NAME.getNameAsString())
290 .setType(SnapshotDescription.Type.FLUSH)
291 .build();
292
293
294 admin.takeSnapshotAsync(snapshot);
295
296
297 HMaster master = UTIL.getMiniHBaseCluster().getMaster();
298 SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
299 LOG.info(" === Async Snapshot Completed ===");
300 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
301 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
302
303 SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
304 }
305
306 @Test (timeout=300000)
307 public void testSnapshotStateAfterMerge() throws Exception {
308 int numRows = DEFAULT_NUM_ROWS;
309 Admin admin = UTIL.getHBaseAdmin();
310
311 SnapshotTestingUtils.assertNoSnapshots(admin);
312
313 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
314
315
316 String snapshotBeforeMergeName = "snapshotBeforeMerge";
317 admin.snapshot(snapshotBeforeMergeName, TABLE_NAME, SnapshotDescription.Type.FLUSH);
318
319
320 TableName cloneBeforeMergeName = TableName.valueOf("cloneBeforeMerge");
321 admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
322 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneBeforeMergeName);
323
324
325 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
326 Collections.sort(regions, new Comparator<HRegionInfo>() {
327 public int compare(HRegionInfo r1, HRegionInfo r2) {
328 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
329 }
330 });
331
332 int numRegions = admin.getTableRegions(TABLE_NAME).size();
333 int numRegionsAfterMerge = numRegions - 2;
334 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
335 regions.get(2).getEncodedNameAsBytes(), true);
336 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
337 regions.get(6).getEncodedNameAsBytes(), true);
338
339
340 waitRegionsAfterMerge(numRegionsAfterMerge);
341 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
342
343
344 TableName cloneAfterMergeName = TableName.valueOf("cloneAfterMerge");
345 admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
346 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneAfterMergeName);
347
348 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
349 SnapshotTestingUtils.verifyRowCount(UTIL, cloneBeforeMergeName, numRows);
350 SnapshotTestingUtils.verifyRowCount(UTIL, cloneAfterMergeName, numRows);
351
352
353 UTIL.deleteTable(cloneAfterMergeName);
354 UTIL.deleteTable(cloneBeforeMergeName);
355 }
356
357 @Test (timeout=300000)
358 public void testTakeSnapshotAfterMerge() throws Exception {
359 int numRows = DEFAULT_NUM_ROWS;
360 Admin admin = UTIL.getHBaseAdmin();
361
362 SnapshotTestingUtils.assertNoSnapshots(admin);
363
364 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
365
366
367 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
368 Collections.sort(regions, new Comparator<HRegionInfo>() {
369 public int compare(HRegionInfo r1, HRegionInfo r2) {
370 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
371 }
372 });
373
374 int numRegions = admin.getTableRegions(TABLE_NAME).size();
375 int numRegionsAfterMerge = numRegions - 2;
376 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
377 regions.get(2).getEncodedNameAsBytes(), true);
378 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
379 regions.get(6).getEncodedNameAsBytes(), true);
380
381 waitRegionsAfterMerge(numRegionsAfterMerge);
382 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
383
384
385 String snapshotName = "snapshotAfterMerge";
386 SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME.getNameAsString(),
387 SnapshotDescription.Type.FLUSH, 3);
388
389
390 TableName cloneName = TableName.valueOf("cloneMerge");
391 admin.cloneSnapshot(snapshotName, cloneName);
392 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName);
393
394 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
395 SnapshotTestingUtils.verifyRowCount(UTIL, cloneName, numRows);
396
397
398 UTIL.deleteTable(cloneName);
399 }
400
401
402
403
404 @Test (timeout=300000)
405 public void testFlushCreateListDestroy() throws Exception {
406 LOG.debug("------- Starting Snapshot test -------------");
407 Admin admin = UTIL.getHBaseAdmin();
408
409 SnapshotTestingUtils.assertNoSnapshots(admin);
410
411 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
412
413 String snapshotName = "flushSnapshotCreateListDestroy";
414 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
415 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
416 SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM),
417 snapshotName, rootDir, fs, true);
418 }
419
420
421
422
423
424
425 @Test(timeout=300000)
426 public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
427 final TableName TABLE2_NAME = TableName.valueOf(TABLE_NAME + "2");
428
429 int ssNum = 20;
430 Admin admin = UTIL.getHBaseAdmin();
431
432 SnapshotTestingUtils.assertNoSnapshots(admin);
433
434 SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
435
436 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
437 SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
438
439 final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
440
441 class SSRunnable implements Runnable {
442 SnapshotDescription ss;
443 SSRunnable(SnapshotDescription ss) {
444 this.ss = ss;
445 }
446
447 @Override
448 public void run() {
449 try {
450 Admin admin = UTIL.getHBaseAdmin();
451 LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
452 admin.takeSnapshotAsync(ss);
453 } catch (Exception e) {
454 LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
455 ss)
456 + ". This is ok, we expect some", e);
457 }
458 LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
459 toBeSubmitted.countDown();
460 }
461 };
462
463
464 SnapshotDescription[] descs = new SnapshotDescription[ssNum];
465 for (int i = 0; i < ssNum; i++) {
466 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
467 builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
468 builder.setName("ss"+i);
469 builder.setType(SnapshotDescription.Type.FLUSH);
470 descs[i] = builder.build();
471 }
472
473
474 for (int i=0 ; i < ssNum; i++) {
475 new Thread(new SSRunnable(descs[i])).start();
476 }
477
478
479 toBeSubmitted.await();
480
481
482 while (true) {
483 int doneCount = 0;
484 for (SnapshotDescription ss : descs) {
485 try {
486 if (admin.isSnapshotFinished(ss)) {
487 doneCount++;
488 }
489 } catch (Exception e) {
490 LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
491 doneCount++;
492 }
493 }
494 if (doneCount == descs.length) {
495 break;
496 }
497 Thread.sleep(100);
498 }
499
500
501 logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
502
503 List<SnapshotDescription> taken = admin.listSnapshots();
504 int takenSize = taken.size();
505 LOG.info("Taken " + takenSize + " snapshots: " + taken);
506 assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
507 " issued many requests", takenSize < ssNum && takenSize > 0);
508
509
510 int t1SnapshotsCount = 0;
511 int t2SnapshotsCount = 0;
512 for (SnapshotDescription ss : taken) {
513 if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
514 t1SnapshotsCount++;
515 } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
516 t2SnapshotsCount++;
517 }
518 }
519 assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
520 assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
521
522 UTIL.deleteTable(TABLE2_NAME);
523 }
524
525 private void logFSTree(Path root) throws IOException {
526 FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
527 }
528
529 private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
530 throws IOException, InterruptedException {
531 Admin admin = UTIL.getHBaseAdmin();
532
533 long startTime = System.currentTimeMillis();
534 while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
535
536
537 if ((System.currentTimeMillis() - startTime) > 15000)
538 break;
539 Thread.sleep(100);
540 }
541 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
542 }
543 }