1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.Collections;
26 import java.util.Comparator;
27 import java.util.HashMap;
28 import java.util.HashSet;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.Set;
32 import java.util.concurrent.CountDownLatch;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.commons.logging.impl.Log4JLogger;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.LargeTests;
44 import org.apache.hadoop.hbase.TableName;
45 import org.apache.hadoop.hbase.TableNotFoundException;
46 import org.apache.hadoop.hbase.client.HBaseAdmin;
47 import org.apache.hadoop.hbase.client.HTable;
48 import org.apache.hadoop.hbase.client.ScannerCallable;
49 import org.apache.hadoop.hbase.ipc.RpcClient;
50 import org.apache.hadoop.hbase.ipc.RpcServer;
51 import org.apache.hadoop.hbase.master.HMaster;
52 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
53 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
54 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
55 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.FSUtils;
58 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
59 import org.apache.log4j.Level;
60 import org.junit.After;
61 import org.junit.AfterClass;
62 import org.junit.Before;
63 import org.junit.BeforeClass;
64 import org.junit.Test;
65 import org.junit.experimental.categories.Category;
66
67
68
69
70
71
72
73
74
75 @Category(LargeTests.class)
76 public class TestFlushSnapshotFromClient {
77 private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
78 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
79 private static final int NUM_RS = 2;
80 private static final String STRING_TABLE_NAME = "test";
81 private static final byte[] TEST_FAM = Bytes.toBytes("fam");
82 private static final byte[] TEST_QUAL = Bytes.toBytes("q");
83 private static final TableName TABLE_NAME =
84 TableName.valueOf(STRING_TABLE_NAME);
85 private final int DEFAULT_NUM_ROWS = 100;
86
87
88
89
90
91 @BeforeClass
92 public static void setupCluster() throws Exception {
93
94
95
96
97
98 setupConf(UTIL.getConfiguration());
99 UTIL.startMiniCluster(NUM_RS);
100 }
101
102 private static void setupConf(Configuration conf) {
103
104 conf.setInt("hbase.regionsever.info.port", -1);
105
106 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
107
108
109 conf.setInt("hbase.hstore.compaction.min", 10);
110 conf.setInt("hbase.hstore.compactionThreshold", 10);
111
112 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
113
114 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
115 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
116 ConstantSizeRegionSplitPolicy.class.getName());
117 }
118
119 @Before
120 public void setup() throws Exception {
121 SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
122 }
123
124 @After
125 public void tearDown() throws Exception {
126 UTIL.deleteTable(TABLE_NAME);
127
128 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
129 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
130 }
131
132 @AfterClass
133 public static void cleanupTest() throws Exception {
134 try {
135 UTIL.shutdownMiniCluster();
136 } catch (Exception e) {
137 LOG.warn("failure shutting down cluster", e);
138 }
139 }
140
141
142
143
144
145 @Test (timeout=300000)
146 public void testFlushTableSnapshot() throws Exception {
147 HBaseAdmin admin = UTIL.getHBaseAdmin();
148
149 SnapshotTestingUtils.assertNoSnapshots(admin);
150
151
152 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
153 SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
154
155 LOG.debug("FS state before snapshot:");
156 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
157 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
158
159
160 String snapshotString = "offlineTableSnapshot";
161 byte[] snapshot = Bytes.toBytes(snapshotString);
162 admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
163 LOG.debug("Snapshot completed.");
164
165
166 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
167 snapshot, TABLE_NAME);
168
169
170 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
171 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
172 LOG.debug("FS state after snapshot:");
173 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
174 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
175
176 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
177 admin, fs);
178 }
179
180
181
182
183
184 @Test
185 public void testSkipFlushTableSnapshot() throws Exception {
186 HBaseAdmin admin = UTIL.getHBaseAdmin();
187
188 SnapshotTestingUtils.assertNoSnapshots(admin);
189
190
191 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
192 UTIL.loadTable(table, TEST_FAM);
193
194 LOG.debug("FS state before snapshot:");
195 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
196 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
197
198
199 String snapshotString = "skipFlushTableSnapshot";
200 byte[] snapshot = Bytes.toBytes(snapshotString);
201 admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
202 LOG.debug("Snapshot completed.");
203
204
205 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
206 snapshot, TABLE_NAME);
207
208
209 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
210 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
211 LOG.debug("FS state after snapshot:");
212 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
213 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
214
215 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
216 admin, fs);
217
218 admin.deleteSnapshot(snapshot);
219 snapshots = admin.listSnapshots();
220 SnapshotTestingUtils.assertNoSnapshots(admin);
221 }
222
223
224
225
226
227
228 @Test (timeout=300000)
229 public void testFlushTableSnapshotWithProcedure() throws Exception {
230 HBaseAdmin admin = UTIL.getHBaseAdmin();
231
232 SnapshotTestingUtils.assertNoSnapshots(admin);
233
234
235 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
236 SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
237
238 LOG.debug("FS state before snapshot:");
239 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
240 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
241
242
243 String snapshotString = "offlineTableSnapshot";
244 byte[] snapshot = Bytes.toBytes(snapshotString);
245 Map<String, String> props = new HashMap<String, String>();
246 props.put("table", STRING_TABLE_NAME);
247 admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
248 snapshotString, props);
249
250
251 LOG.debug("Snapshot completed.");
252
253
254 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
255 snapshot, TABLE_NAME);
256
257
258 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
259 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
260 LOG.debug("FS state after snapshot:");
261 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
262 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
263
264 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
265 admin, fs);
266 }
267
268 @Test (timeout=300000)
269 public void testSnapshotFailsOnNonExistantTable() throws Exception {
270 HBaseAdmin admin = UTIL.getHBaseAdmin();
271
272 SnapshotTestingUtils.assertNoSnapshots(admin);
273 String tableName = "_not_a_table";
274
275
276 boolean fail = false;
277 do {
278 try {
279 admin.getTableDescriptor(Bytes.toBytes(tableName));
280 fail = true;
281 LOG.error("Table:" + tableName + " already exists, checking a new name");
282 tableName = tableName+"!";
283 } catch (TableNotFoundException e) {
284 fail = false;
285 }
286 } while (fail);
287
288
289 try {
290 admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
291 fail("Snapshot succeeded even though there is not table.");
292 } catch (SnapshotCreationException e) {
293 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
294 }
295 }
296
297 @Test(timeout = 300000)
298 public void testAsyncFlushSnapshot() throws Exception {
299 HBaseAdmin admin = UTIL.getHBaseAdmin();
300 SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
301 .setTable(TABLE_NAME.getNameAsString())
302 .setType(SnapshotDescription.Type.FLUSH)
303 .build();
304
305
306 admin.takeSnapshotAsync(snapshot);
307
308
309 HMaster master = UTIL.getMiniHBaseCluster().getMaster();
310 SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
311 LOG.info(" === Async Snapshot Completed ===");
312 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
313 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
314
315 SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
316 }
317
318 @Test (timeout=300000)
319 public void testSnapshotStateAfterMerge() throws Exception {
320 int numRows = DEFAULT_NUM_ROWS;
321 HBaseAdmin admin = UTIL.getHBaseAdmin();
322
323 SnapshotTestingUtils.assertNoSnapshots(admin);
324
325 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
326
327
328 String snapshotBeforeMergeName = "snapshotBeforeMerge";
329 admin.snapshot(snapshotBeforeMergeName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
330
331
332 String cloneBeforeMergeName = "cloneBeforeMerge";
333 admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
334 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneBeforeMergeName));
335
336
337 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
338 Collections.sort(regions, new Comparator<HRegionInfo>() {
339 public int compare(HRegionInfo r1, HRegionInfo r2) {
340 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
341 }
342 });
343
344 int numRegions = admin.getTableRegions(TABLE_NAME).size();
345 int numRegionsAfterMerge = numRegions - 2;
346 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
347 regions.get(2).getEncodedNameAsBytes(), true);
348 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
349 regions.get(6).getEncodedNameAsBytes(), true);
350
351
352 waitRegionsAfterMerge(numRegionsAfterMerge);
353 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
354
355
356 String cloneAfterMergeName = "cloneAfterMerge";
357 admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
358 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneAfterMergeName));
359
360 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
361 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneBeforeMergeName), numRows);
362 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneAfterMergeName), numRows);
363
364
365 UTIL.deleteTable(cloneAfterMergeName);
366 UTIL.deleteTable(cloneBeforeMergeName);
367 }
368
369 @Test (timeout=300000)
370 public void testTakeSnapshotAfterMerge() throws Exception {
371 int numRows = DEFAULT_NUM_ROWS;
372 HBaseAdmin admin = UTIL.getHBaseAdmin();
373
374 SnapshotTestingUtils.assertNoSnapshots(admin);
375
376 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
377
378
379 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
380 Collections.sort(regions, new Comparator<HRegionInfo>() {
381 public int compare(HRegionInfo r1, HRegionInfo r2) {
382 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
383 }
384 });
385
386 int numRegions = admin.getTableRegions(TABLE_NAME).size();
387 int numRegionsAfterMerge = numRegions - 2;
388 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
389 regions.get(2).getEncodedNameAsBytes(), true);
390 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
391 regions.get(6).getEncodedNameAsBytes(), true);
392
393 waitRegionsAfterMerge(numRegionsAfterMerge);
394 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
395
396
397 String snapshotName = "snapshotAfterMerge";
398 SnapshotTestingUtils.snapshot(admin, snapshotName, STRING_TABLE_NAME,
399 SnapshotDescription.Type.FLUSH, 3);
400
401
402 String cloneName = "cloneMerge";
403 admin.cloneSnapshot(snapshotName, cloneName);
404 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneName));
405
406 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
407 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneName), numRows);
408
409
410 UTIL.deleteTable(cloneName);
411 }
412
413
414
415
416 @Test (timeout=300000)
417 public void testFlushCreateListDestroy() throws Exception {
418 LOG.debug("------- Starting Snapshot test -------------");
419 HBaseAdmin admin = UTIL.getHBaseAdmin();
420
421 SnapshotTestingUtils.assertNoSnapshots(admin);
422
423 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
424
425 String snapshotName = "flushSnapshotCreateListDestroy";
426 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
427 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
428 SnapshotTestingUtils.createSnapshotAndValidate(admin,
429 TableName.valueOf(STRING_TABLE_NAME), Bytes.toString(TEST_FAM),
430 snapshotName, rootDir, fs, true);
431 }
432
433
434
435
436
437
438 @Test(timeout=300000)
439 public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
440 final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
441 final TableName TABLE2_NAME =
442 TableName.valueOf(STRING_TABLE2_NAME);
443
444 int ssNum = 20;
445 HBaseAdmin admin = UTIL.getHBaseAdmin();
446
447 SnapshotTestingUtils.assertNoSnapshots(admin);
448
449 SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
450
451 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
452 SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
453
454 final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
455
456 class SSRunnable implements Runnable {
457 SnapshotDescription ss;
458 SSRunnable(SnapshotDescription ss) {
459 this.ss = ss;
460 }
461
462 @Override
463 public void run() {
464 try {
465 HBaseAdmin admin = UTIL.getHBaseAdmin();
466 LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
467 admin.takeSnapshotAsync(ss);
468 } catch (Exception e) {
469 LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
470 ss)
471 + ". This is ok, we expect some", e);
472 }
473 LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
474 toBeSubmitted.countDown();
475 }
476 };
477
478
479 SnapshotDescription[] descs = new SnapshotDescription[ssNum];
480 for (int i = 0; i < ssNum; i++) {
481 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
482 builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
483 builder.setName("ss"+i);
484 builder.setType(SnapshotDescription.Type.FLUSH);
485 descs[i] = builder.build();
486 }
487
488
489 for (int i=0 ; i < ssNum; i++) {
490 new Thread(new SSRunnable(descs[i])).start();
491 }
492
493
494 toBeSubmitted.await();
495
496
497 while (true) {
498 int doneCount = 0;
499 for (SnapshotDescription ss : descs) {
500 try {
501 if (admin.isSnapshotFinished(ss)) {
502 doneCount++;
503 }
504 } catch (Exception e) {
505 LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
506 doneCount++;
507 }
508 }
509 if (doneCount == descs.length) {
510 break;
511 }
512 Thread.sleep(100);
513 }
514
515
516 logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
517
518 List<SnapshotDescription> taken = admin.listSnapshots();
519 int takenSize = taken.size();
520 LOG.info("Taken " + takenSize + " snapshots: " + taken);
521 assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
522 " issued many requests", takenSize < ssNum && takenSize > 0);
523
524
525 int t1SnapshotsCount = 0;
526 int t2SnapshotsCount = 0;
527 for (SnapshotDescription ss : taken) {
528 if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
529 t1SnapshotsCount++;
530 } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
531 t2SnapshotsCount++;
532 }
533 }
534 assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
535 assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
536
537 UTIL.deleteTable(TABLE2_NAME);
538 }
539
540 private void logFSTree(Path root) throws IOException {
541 FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
542 }
543
544 private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
545 throws IOException, InterruptedException {
546 HBaseAdmin admin = UTIL.getHBaseAdmin();
547
548 long startTime = System.currentTimeMillis();
549 while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
550
551
552 if ((System.currentTimeMillis() - startTime) > 15000)
553 break;
554 Thread.sleep(100);
555 }
556 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
557 }
558 }