1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.Collections;
26 import java.util.Comparator;
27 import java.util.HashSet;
28 import java.util.List;
29 import java.util.Set;
30 import java.util.concurrent.CountDownLatch;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.commons.logging.impl.Log4JLogger;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.TableName;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.LargeTests;
44 import org.apache.hadoop.hbase.TableNotFoundException;
45 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
46 import org.apache.hadoop.hbase.ipc.RpcClient;
47 import org.apache.hadoop.hbase.ipc.RpcServer;
48 import org.apache.hadoop.hbase.client.HBaseAdmin;
49 import org.apache.hadoop.hbase.client.HTable;
50 import org.apache.hadoop.hbase.client.ScannerCallable;
51 import org.apache.hadoop.hbase.master.HMaster;
52 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
53 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
54 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
55 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.FSTableDescriptors;
58 import org.apache.hadoop.hbase.util.FSUtils;
59 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
60 import org.apache.log4j.Level;
61 import org.junit.After;
62 import org.junit.AfterClass;
63 import org.junit.Before;
64 import org.junit.BeforeClass;
65 import org.junit.Test;
66 import org.junit.experimental.categories.Category;
67
68
69
70
71
72
73
74
75
76 @Category(LargeTests.class)
77 public class TestFlushSnapshotFromClient {
78 private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
79 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
80 private static final int NUM_RS = 2;
81 private static final String STRING_TABLE_NAME = "test";
82 private static final byte[] TEST_FAM = Bytes.toBytes("fam");
83 private static final byte[] TEST_QUAL = Bytes.toBytes("q");
84 private static final TableName TABLE_NAME =
85 TableName.valueOf(STRING_TABLE_NAME);
86 private final int DEFAULT_NUM_ROWS = 100;
87
88
89
90
91
92 @BeforeClass
93 public static void setupCluster() throws Exception {
94 ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
95 ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
96 ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
97 setupConf(UTIL.getConfiguration());
98 UTIL.startMiniCluster(NUM_RS);
99 }
100
101 private static void setupConf(Configuration conf) {
102
103 conf.setInt("hbase.regionsever.info.port", -1);
104
105 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
106
107
108 conf.setInt("hbase.hstore.compaction.min", 10);
109 conf.setInt("hbase.hstore.compactionThreshold", 10);
110
111 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
112
113 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
114 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
115 ConstantSizeRegionSplitPolicy.class.getName());
116 }
117
118 @Before
119 public void setup() throws Exception {
120 SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
121 }
122
123 @After
124 public void tearDown() throws Exception {
125 UTIL.deleteTable(TABLE_NAME);
126
127 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
128 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
129 }
130
131 @AfterClass
132 public static void cleanupTest() throws Exception {
133 try {
134 UTIL.shutdownMiniCluster();
135 } catch (Exception e) {
136 LOG.warn("failure shutting down cluster", e);
137 }
138 }
139
140
141
142
143
144 @Test (timeout=300000)
145 public void testFlushTableSnapshot() throws Exception {
146 HBaseAdmin admin = UTIL.getHBaseAdmin();
147
148 SnapshotTestingUtils.assertNoSnapshots(admin);
149
150
151 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
152 SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
153
154
155 Set<String> snapshotServers = new HashSet<String>();
156 List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
157 for (RegionServerThread server : servers) {
158 if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
159 snapshotServers.add(server.getRegionServer().getServerName().toString());
160 }
161 }
162
163 LOG.debug("FS state before snapshot:");
164 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
165 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
166
167
168 String snapshotString = "offlineTableSnapshot";
169 byte[] snapshot = Bytes.toBytes(snapshotString);
170 admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
171 LOG.debug("Snapshot completed.");
172
173
174 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
175 snapshot, TABLE_NAME);
176
177
178 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
179 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
180 LOG.debug("FS state after snapshot:");
181 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
182 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
183
184 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
185 admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
186
187 admin.deleteSnapshot(snapshot);
188 snapshots = admin.listSnapshots();
189 SnapshotTestingUtils.assertNoSnapshots(admin);
190 }
191
192 @Test (timeout=300000)
193 public void testSnapshotFailsOnNonExistantTable() throws Exception {
194 HBaseAdmin admin = UTIL.getHBaseAdmin();
195
196 SnapshotTestingUtils.assertNoSnapshots(admin);
197 String tableName = "_not_a_table";
198
199
200 boolean fail = false;
201 do {
202 try {
203 admin.getTableDescriptor(Bytes.toBytes(tableName));
204 fail = true;
205 LOG.error("Table:" + tableName + " already exists, checking a new name");
206 tableName = tableName+"!";
207 } catch (TableNotFoundException e) {
208 fail = false;
209 }
210 } while (fail);
211
212
213 try {
214 admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
215 fail("Snapshot succeeded even though there is not table.");
216 } catch (SnapshotCreationException e) {
217 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
218 }
219 }
220
221 @Test(timeout = 300000)
222 public void testAsyncFlushSnapshot() throws Exception {
223 HBaseAdmin admin = UTIL.getHBaseAdmin();
224 SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
225 .setTable(TABLE_NAME.getNameAsString())
226 .setType(SnapshotDescription.Type.FLUSH)
227 .build();
228
229
230 admin.takeSnapshotAsync(snapshot);
231
232
233 HMaster master = UTIL.getMiniHBaseCluster().getMaster();
234 SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
235 LOG.info(" === Async Snapshot Completed ===");
236 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
237 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
238
239 SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
240
241
242 admin.deleteSnapshot(snapshot.getName());
243 LOG.info(" === Async Snapshot Deleted ===");
244 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
245 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
246
247 SnapshotTestingUtils.assertNoSnapshots(admin);
248 LOG.info(" === Async Snapshot Test Completed ===");
249
250 }
251
252 @Test (timeout=300000)
253 public void testSnapshotStateAfterMerge() throws Exception {
254 int numRows = DEFAULT_NUM_ROWS;
255 HBaseAdmin admin = UTIL.getHBaseAdmin();
256
257 SnapshotTestingUtils.assertNoSnapshots(admin);
258
259 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
260
261
262 String snapshotBeforeMergeName = "snapshotBeforeMerge";
263 admin.snapshot(snapshotBeforeMergeName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
264
265
266 String cloneBeforeMergeName = "cloneBeforeMerge";
267 admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
268 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneBeforeMergeName));
269
270
271 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
272 Collections.sort(regions, new Comparator<HRegionInfo>() {
273 public int compare(HRegionInfo r1, HRegionInfo r2) {
274 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
275 }
276 });
277
278 int numRegions = admin.getTableRegions(TABLE_NAME).size();
279 int numRegionsAfterMerge = numRegions - 2;
280 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
281 regions.get(2).getEncodedNameAsBytes(), true);
282 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
283 regions.get(6).getEncodedNameAsBytes(), true);
284
285
286 waitRegionsAfterMerge(numRegionsAfterMerge);
287 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
288
289
290 String cloneAfterMergeName = "cloneAfterMerge";
291 admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
292 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneAfterMergeName));
293
294 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
295 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneBeforeMergeName), numRows);
296 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneAfterMergeName), numRows);
297
298
299 UTIL.deleteTable(cloneAfterMergeName);
300 UTIL.deleteTable(cloneBeforeMergeName);
301 admin.deleteSnapshot(snapshotBeforeMergeName);
302
303
304 SnapshotTestingUtils.assertNoSnapshots(admin);
305 }
306
307 @Test (timeout=300000)
308 public void testTakeSnapshotAfterMerge() throws Exception {
309 int numRows = DEFAULT_NUM_ROWS;
310 HBaseAdmin admin = UTIL.getHBaseAdmin();
311
312 SnapshotTestingUtils.assertNoSnapshots(admin);
313
314 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
315
316
317 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
318 Collections.sort(regions, new Comparator<HRegionInfo>() {
319 public int compare(HRegionInfo r1, HRegionInfo r2) {
320 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
321 }
322 });
323
324 int numRegions = admin.getTableRegions(TABLE_NAME).size();
325 int numRegionsAfterMerge = numRegions - 2;
326 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
327 regions.get(2).getEncodedNameAsBytes(), true);
328 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
329 regions.get(6).getEncodedNameAsBytes(), true);
330
331 waitRegionsAfterMerge(numRegionsAfterMerge);
332 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
333
334
335 String snapshotName = "snapshotAfterMerge";
336 admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
337
338
339 String cloneName = "cloneMerge";
340 admin.cloneSnapshot(snapshotName, cloneName);
341 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneName));
342
343 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
344 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneName), numRows);
345
346
347 UTIL.deleteTable(cloneName);
348 admin.deleteSnapshot(snapshotName);
349
350
351 SnapshotTestingUtils.assertNoSnapshots(admin);
352 }
353
354
355
356
357 @Test (timeout=300000)
358 public void testFlushCreateListDestroy() throws Exception {
359 LOG.debug("------- Starting Snapshot test -------------");
360 HBaseAdmin admin = UTIL.getHBaseAdmin();
361
362 SnapshotTestingUtils.assertNoSnapshots(admin);
363
364 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
365
366 String snapshotName = "flushSnapshotCreateListDestroy";
367
368 admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
369 logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
370
371
372 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
373 snapshotName, TABLE_NAME);
374
375
376 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
377 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
378 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir);
379 assertTrue(fs.exists(snapshotDir));
380 FSUtils.logFileSystemState(UTIL.getTestFileSystem(), snapshotDir, LOG);
381 Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
382 assertTrue(fs.exists(snapshotinfo));
383
384
385 HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs,
386 rootDir, TABLE_NAME);
387 HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs,
388 new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir), snapshotName));
389 assertEquals(desc, snapshotDesc);
390
391
392 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
393 assertTrue(regions.size() > 1);
394 for (HRegionInfo info : regions) {
395 String regionName = info.getEncodedName();
396 Path regionDir = new Path(snapshotDir, regionName);
397 HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
398 assertEquals(info, snapshotRegionInfo);
399
400 Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM));
401 assertTrue("Missing region " + Bytes.toString(snapshotRegionInfo.getStartKey()),
402 fs.exists(familyDir));
403
404
405 assertTrue(fs.listStatus(familyDir).length > 0);
406 }
407
408
409 admin.deleteSnapshot(snapshotName);
410 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
411 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
412
413
414 SnapshotTestingUtils.assertNoSnapshots(admin);
415 LOG.debug("------- Flush-Snapshot Create List Destroy-------------");
416 }
417
418
419
420
421
422
423 @Test(timeout=300000)
424 public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
425 final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
426 final TableName TABLE2_NAME =
427 TableName.valueOf(STRING_TABLE2_NAME);
428
429 int ssNum = 20;
430 HBaseAdmin admin = UTIL.getHBaseAdmin();
431
432 SnapshotTestingUtils.assertNoSnapshots(admin);
433
434 SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
435
436 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
437 SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
438
439 final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
440
441 class SSRunnable implements Runnable {
442 SnapshotDescription ss;
443 SSRunnable(SnapshotDescription ss) {
444 this.ss = ss;
445 }
446
447 @Override
448 public void run() {
449 try {
450 HBaseAdmin admin = UTIL.getHBaseAdmin();
451 LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
452 admin.takeSnapshotAsync(ss);
453 } catch (Exception e) {
454 LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
455 ss)
456 + ". This is ok, we expect some", e);
457 }
458 LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
459 toBeSubmitted.countDown();
460 }
461 };
462
463
464 SnapshotDescription[] descs = new SnapshotDescription[ssNum];
465 for (int i = 0; i < ssNum; i++) {
466 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
467 builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
468 builder.setName("ss"+i);
469 builder.setType(SnapshotDescription.Type.FLUSH);
470 descs[i] = builder.build();
471 }
472
473
474 for (int i=0 ; i < ssNum; i++) {
475 new Thread(new SSRunnable(descs[i])).start();
476 }
477
478
479 toBeSubmitted.await();
480
481
482 while (true) {
483 int doneCount = 0;
484 for (SnapshotDescription ss : descs) {
485 try {
486 if (admin.isSnapshotFinished(ss)) {
487 doneCount++;
488 }
489 } catch (Exception e) {
490 LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
491 doneCount++;
492 }
493 }
494 if (doneCount == descs.length) {
495 break;
496 }
497 Thread.sleep(100);
498 }
499
500
501 logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
502
503 List<SnapshotDescription> taken = admin.listSnapshots();
504 int takenSize = taken.size();
505 LOG.info("Taken " + takenSize + " snapshots: " + taken);
506 assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
507 " issued many requests", takenSize < ssNum && takenSize > 0);
508
509
510 int t1SnapshotsCount = 0;
511 int t2SnapshotsCount = 0;
512 for (SnapshotDescription ss : taken) {
513 if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
514 t1SnapshotsCount++;
515 } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
516 t2SnapshotsCount++;
517 }
518 }
519 assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
520 assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
521
522
523 for (SnapshotDescription ss : taken) {
524 admin.deleteSnapshot(ss.getName());
525 }
526 UTIL.deleteTable(TABLE2_NAME);
527 }
528
529 private void logFSTree(Path root) throws IOException {
530 FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
531 }
532
533 private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
534 throws IOException, InterruptedException {
535 HBaseAdmin admin = UTIL.getHBaseAdmin();
536
537 long startTime = System.currentTimeMillis();
538 while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
539
540
541 if ((System.currentTimeMillis() - startTime) > 15000)
542 break;
543 Thread.sleep(100);
544 }
545 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
546 }
547 }