1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.Collections;
26 import java.util.Comparator;
27 import java.util.HashMap;
28 import java.util.HashSet;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.Set;
32 import java.util.concurrent.CountDownLatch;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.commons.logging.impl.Log4JLogger;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.LargeTests;
44 import org.apache.hadoop.hbase.TableName;
45 import org.apache.hadoop.hbase.TableNotFoundException;
46 import org.apache.hadoop.hbase.client.HBaseAdmin;
47 import org.apache.hadoop.hbase.client.HTable;
48 import org.apache.hadoop.hbase.client.ScannerCallable;
49 import org.apache.hadoop.hbase.ipc.RpcClient;
50 import org.apache.hadoop.hbase.ipc.RpcServer;
51 import org.apache.hadoop.hbase.master.HMaster;
52 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
53 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
54 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
55 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.FSUtils;
58 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
59 import org.apache.log4j.Level;
60 import org.junit.After;
61 import org.junit.AfterClass;
62 import org.junit.Before;
63 import org.junit.BeforeClass;
64 import org.junit.Test;
65 import org.junit.experimental.categories.Category;
66
67
68
69
70
71
72
73
74
75 @Category(LargeTests.class)
76 public class TestFlushSnapshotFromClient {
77 private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
78 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
79 private static final int NUM_RS = 2;
80 private static final String STRING_TABLE_NAME = "test";
81 private static final byte[] TEST_FAM = Bytes.toBytes("fam");
82 private static final byte[] TEST_QUAL = Bytes.toBytes("q");
83 private static final TableName TABLE_NAME =
84 TableName.valueOf(STRING_TABLE_NAME);
85 private final int DEFAULT_NUM_ROWS = 100;
86
87
88
89
90
91 @BeforeClass
92 public static void setupCluster() throws Exception {
93 ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
94 ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
95 ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
96 setupConf(UTIL.getConfiguration());
97 UTIL.startMiniCluster(NUM_RS);
98 }
99
100 private static void setupConf(Configuration conf) {
101
102 conf.setInt("hbase.regionsever.info.port", -1);
103
104 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
105
106
107 conf.setInt("hbase.hstore.compaction.min", 10);
108 conf.setInt("hbase.hstore.compactionThreshold", 10);
109
110 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
111
112 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
113 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
114 ConstantSizeRegionSplitPolicy.class.getName());
115 }
116
117 @Before
118 public void setup() throws Exception {
119 SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
120 }
121
122 @After
123 public void tearDown() throws Exception {
124 UTIL.deleteTable(TABLE_NAME);
125
126 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
127 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
128 }
129
130 @AfterClass
131 public static void cleanupTest() throws Exception {
132 try {
133 UTIL.shutdownMiniCluster();
134 } catch (Exception e) {
135 LOG.warn("failure shutting down cluster", e);
136 }
137 }
138
139
140
141
142
143 @Test (timeout=300000)
144 public void testFlushTableSnapshot() throws Exception {
145 HBaseAdmin admin = UTIL.getHBaseAdmin();
146
147 SnapshotTestingUtils.assertNoSnapshots(admin);
148
149
150 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
151 SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
152
153 LOG.debug("FS state before snapshot:");
154 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
155 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
156
157
158 String snapshotString = "offlineTableSnapshot";
159 byte[] snapshot = Bytes.toBytes(snapshotString);
160 admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
161 LOG.debug("Snapshot completed.");
162
163
164 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
165 snapshot, TABLE_NAME);
166
167
168 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
169 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
170 LOG.debug("FS state after snapshot:");
171 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
172 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
173
174 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
175 admin, fs);
176 }
177
178
179
180
181
182 @Test
183 public void testSkipFlushTableSnapshot() throws Exception {
184 HBaseAdmin admin = UTIL.getHBaseAdmin();
185
186 SnapshotTestingUtils.assertNoSnapshots(admin);
187
188
189 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
190 UTIL.loadTable(table, TEST_FAM);
191
192 LOG.debug("FS state before snapshot:");
193 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
194 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
195
196
197 String snapshotString = "skipFlushTableSnapshot";
198 byte[] snapshot = Bytes.toBytes(snapshotString);
199 admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
200 LOG.debug("Snapshot completed.");
201
202
203 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
204 snapshot, TABLE_NAME);
205
206
207 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
208 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
209 LOG.debug("FS state after snapshot:");
210 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
211 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
212
213 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
214 admin, fs);
215
216 admin.deleteSnapshot(snapshot);
217 snapshots = admin.listSnapshots();
218 SnapshotTestingUtils.assertNoSnapshots(admin);
219 }
220
221
222
223
224
225
226 @Test (timeout=300000)
227 public void testFlushTableSnapshotWithProcedure() throws Exception {
228 HBaseAdmin admin = UTIL.getHBaseAdmin();
229
230 SnapshotTestingUtils.assertNoSnapshots(admin);
231
232
233 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
234 SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
235
236 LOG.debug("FS state before snapshot:");
237 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
238 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
239
240
241 String snapshotString = "offlineTableSnapshot";
242 byte[] snapshot = Bytes.toBytes(snapshotString);
243 Map<String, String> props = new HashMap<String, String>();
244 props.put("table", STRING_TABLE_NAME);
245 admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
246 snapshotString, props);
247
248
249 LOG.debug("Snapshot completed.");
250
251
252 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
253 snapshot, TABLE_NAME);
254
255
256 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
257 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
258 LOG.debug("FS state after snapshot:");
259 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
260 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
261
262 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
263 admin, fs);
264 }
265
266 @Test (timeout=300000)
267 public void testSnapshotFailsOnNonExistantTable() throws Exception {
268 HBaseAdmin admin = UTIL.getHBaseAdmin();
269
270 SnapshotTestingUtils.assertNoSnapshots(admin);
271 String tableName = "_not_a_table";
272
273
274 boolean fail = false;
275 do {
276 try {
277 admin.getTableDescriptor(Bytes.toBytes(tableName));
278 fail = true;
279 LOG.error("Table:" + tableName + " already exists, checking a new name");
280 tableName = tableName+"!";
281 } catch (TableNotFoundException e) {
282 fail = false;
283 }
284 } while (fail);
285
286
287 try {
288 admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
289 fail("Snapshot succeeded even though there is not table.");
290 } catch (SnapshotCreationException e) {
291 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
292 }
293 }
294
295 @Test(timeout = 300000)
296 public void testAsyncFlushSnapshot() throws Exception {
297 HBaseAdmin admin = UTIL.getHBaseAdmin();
298 SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
299 .setTable(TABLE_NAME.getNameAsString())
300 .setType(SnapshotDescription.Type.FLUSH)
301 .build();
302
303
304 admin.takeSnapshotAsync(snapshot);
305
306
307 HMaster master = UTIL.getMiniHBaseCluster().getMaster();
308 SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
309 LOG.info(" === Async Snapshot Completed ===");
310 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
311 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
312
313 SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
314 }
315
316 @Test (timeout=300000)
317 public void testSnapshotStateAfterMerge() throws Exception {
318 int numRows = DEFAULT_NUM_ROWS;
319 HBaseAdmin admin = UTIL.getHBaseAdmin();
320
321 SnapshotTestingUtils.assertNoSnapshots(admin);
322
323 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
324
325
326 String snapshotBeforeMergeName = "snapshotBeforeMerge";
327 admin.snapshot(snapshotBeforeMergeName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
328
329
330 String cloneBeforeMergeName = "cloneBeforeMerge";
331 admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
332 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneBeforeMergeName));
333
334
335 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
336 Collections.sort(regions, new Comparator<HRegionInfo>() {
337 public int compare(HRegionInfo r1, HRegionInfo r2) {
338 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
339 }
340 });
341
342 int numRegions = admin.getTableRegions(TABLE_NAME).size();
343 int numRegionsAfterMerge = numRegions - 2;
344 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
345 regions.get(2).getEncodedNameAsBytes(), true);
346 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
347 regions.get(6).getEncodedNameAsBytes(), true);
348
349
350 waitRegionsAfterMerge(numRegionsAfterMerge);
351 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
352
353
354 String cloneAfterMergeName = "cloneAfterMerge";
355 admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
356 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneAfterMergeName));
357
358 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
359 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneBeforeMergeName), numRows);
360 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneAfterMergeName), numRows);
361
362
363 UTIL.deleteTable(cloneAfterMergeName);
364 UTIL.deleteTable(cloneBeforeMergeName);
365 }
366
367 @Test (timeout=300000)
368 public void testTakeSnapshotAfterMerge() throws Exception {
369 int numRows = DEFAULT_NUM_ROWS;
370 HBaseAdmin admin = UTIL.getHBaseAdmin();
371
372 SnapshotTestingUtils.assertNoSnapshots(admin);
373
374 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
375
376
377 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
378 Collections.sort(regions, new Comparator<HRegionInfo>() {
379 public int compare(HRegionInfo r1, HRegionInfo r2) {
380 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
381 }
382 });
383
384 int numRegions = admin.getTableRegions(TABLE_NAME).size();
385 int numRegionsAfterMerge = numRegions - 2;
386 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
387 regions.get(2).getEncodedNameAsBytes(), true);
388 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
389 regions.get(6).getEncodedNameAsBytes(), true);
390
391 waitRegionsAfterMerge(numRegionsAfterMerge);
392 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
393
394
395 String snapshotName = "snapshotAfterMerge";
396 SnapshotTestingUtils.snapshot(admin, snapshotName, STRING_TABLE_NAME,
397 SnapshotDescription.Type.FLUSH, 3);
398
399
400 String cloneName = "cloneMerge";
401 admin.cloneSnapshot(snapshotName, cloneName);
402 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneName));
403
404 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
405 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneName), numRows);
406
407
408 UTIL.deleteTable(cloneName);
409 }
410
411
412
413
414 @Test (timeout=300000)
415 public void testFlushCreateListDestroy() throws Exception {
416 LOG.debug("------- Starting Snapshot test -------------");
417 HBaseAdmin admin = UTIL.getHBaseAdmin();
418
419 SnapshotTestingUtils.assertNoSnapshots(admin);
420
421 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
422
423 String snapshotName = "flushSnapshotCreateListDestroy";
424 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
425 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
426 SnapshotTestingUtils.createSnapshotAndValidate(admin,
427 TableName.valueOf(STRING_TABLE_NAME), Bytes.toString(TEST_FAM),
428 snapshotName, rootDir, fs, true);
429 }
430
431
432
433
434
435
436 @Test(timeout=300000)
437 public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
438 final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
439 final TableName TABLE2_NAME =
440 TableName.valueOf(STRING_TABLE2_NAME);
441
442 int ssNum = 20;
443 HBaseAdmin admin = UTIL.getHBaseAdmin();
444
445 SnapshotTestingUtils.assertNoSnapshots(admin);
446
447 SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
448
449 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
450 SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
451
452 final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
453
454 class SSRunnable implements Runnable {
455 SnapshotDescription ss;
456 SSRunnable(SnapshotDescription ss) {
457 this.ss = ss;
458 }
459
460 @Override
461 public void run() {
462 try {
463 HBaseAdmin admin = UTIL.getHBaseAdmin();
464 LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
465 admin.takeSnapshotAsync(ss);
466 } catch (Exception e) {
467 LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
468 ss)
469 + ". This is ok, we expect some", e);
470 }
471 LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
472 toBeSubmitted.countDown();
473 }
474 };
475
476
477 SnapshotDescription[] descs = new SnapshotDescription[ssNum];
478 for (int i = 0; i < ssNum; i++) {
479 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
480 builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
481 builder.setName("ss"+i);
482 builder.setType(SnapshotDescription.Type.FLUSH);
483 descs[i] = builder.build();
484 }
485
486
487 for (int i=0 ; i < ssNum; i++) {
488 new Thread(new SSRunnable(descs[i])).start();
489 }
490
491
492 toBeSubmitted.await();
493
494
495 while (true) {
496 int doneCount = 0;
497 for (SnapshotDescription ss : descs) {
498 try {
499 if (admin.isSnapshotFinished(ss)) {
500 doneCount++;
501 }
502 } catch (Exception e) {
503 LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
504 doneCount++;
505 }
506 }
507 if (doneCount == descs.length) {
508 break;
509 }
510 Thread.sleep(100);
511 }
512
513
514 logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
515
516 List<SnapshotDescription> taken = admin.listSnapshots();
517 int takenSize = taken.size();
518 LOG.info("Taken " + takenSize + " snapshots: " + taken);
519 assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
520 " issued many requests", takenSize < ssNum && takenSize > 0);
521
522
523 int t1SnapshotsCount = 0;
524 int t2SnapshotsCount = 0;
525 for (SnapshotDescription ss : taken) {
526 if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
527 t1SnapshotsCount++;
528 } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
529 t2SnapshotsCount++;
530 }
531 }
532 assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
533 assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
534
535 UTIL.deleteTable(TABLE2_NAME);
536 }
537
538 private void logFSTree(Path root) throws IOException {
539 FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
540 }
541
542 private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
543 throws IOException, InterruptedException {
544 HBaseAdmin admin = UTIL.getHBaseAdmin();
545
546 long startTime = System.currentTimeMillis();
547 while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
548
549
550 if ((System.currentTimeMillis() - startTime) > 15000)
551 break;
552 Thread.sleep(100);
553 }
554 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
555 }
556 }