1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.Collections;
26 import java.util.Comparator;
27 import java.util.HashSet;
28 import java.util.List;
29 import java.util.Set;
30 import java.util.concurrent.CountDownLatch;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.commons.logging.impl.Log4JLogger;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.LargeTests;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.TableNotFoundException;
45 import org.apache.hadoop.hbase.client.HBaseAdmin;
46 import org.apache.hadoop.hbase.client.HTable;
47 import org.apache.hadoop.hbase.client.ScannerCallable;
48 import org.apache.hadoop.hbase.ipc.RpcClient;
49 import org.apache.hadoop.hbase.ipc.RpcServer;
50 import org.apache.hadoop.hbase.master.HMaster;
51 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
52 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
53 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
54 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
55 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.FSTableDescriptors;
58 import org.apache.hadoop.hbase.util.FSUtils;
59 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
60 import org.apache.log4j.Level;
61 import org.junit.After;
62 import org.junit.AfterClass;
63 import org.junit.Before;
64 import org.junit.BeforeClass;
65 import org.junit.Test;
66 import org.junit.experimental.categories.Category;
67
68
69
70
71
72
73
74
75
76 @Category(LargeTests.class)
77 public class TestFlushSnapshotFromClient {
78 private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
79 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
80 private static final int NUM_RS = 2;
81 private static final String STRING_TABLE_NAME = "test";
82 private static final byte[] TEST_FAM = Bytes.toBytes("fam");
83 private static final byte[] TEST_QUAL = Bytes.toBytes("q");
84 private static final TableName TABLE_NAME =
85 TableName.valueOf(STRING_TABLE_NAME);
86 private final int DEFAULT_NUM_ROWS = 100;
87
88
89
90
91
92 @BeforeClass
93 public static void setupCluster() throws Exception {
94 ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
95 ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
96 ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
97 setupConf(UTIL.getConfiguration());
98 UTIL.startMiniCluster(NUM_RS);
99 }
100
101 private static void setupConf(Configuration conf) {
102
103 conf.setInt("hbase.regionsever.info.port", -1);
104
105 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
106
107
108 conf.setInt("hbase.hstore.compaction.min", 10);
109 conf.setInt("hbase.hstore.compactionThreshold", 10);
110
111 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
112
113 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
114 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
115 ConstantSizeRegionSplitPolicy.class.getName());
116 }
117
118 @Before
119 public void setup() throws Exception {
120 SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
121 }
122
123 @After
124 public void tearDown() throws Exception {
125 UTIL.deleteTable(TABLE_NAME);
126
127 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
128 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
129 }
130
131 @AfterClass
132 public static void cleanupTest() throws Exception {
133 try {
134 UTIL.shutdownMiniCluster();
135 } catch (Exception e) {
136 LOG.warn("failure shutting down cluster", e);
137 }
138 }
139
140
141
142
143
144 @Test (timeout=300000)
145 public void testFlushTableSnapshot() throws Exception {
146 HBaseAdmin admin = UTIL.getHBaseAdmin();
147
148 SnapshotTestingUtils.assertNoSnapshots(admin);
149
150
151 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
152 SnapshotTestingUtils.loadData(UTIL, table, DEFAULT_NUM_ROWS, TEST_FAM);
153
154
155 Set<String> snapshotServers = new HashSet<String>();
156 List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
157 for (RegionServerThread server : servers) {
158 if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
159 snapshotServers.add(server.getRegionServer().getServerName().toString());
160 }
161 }
162
163 LOG.debug("FS state before snapshot:");
164 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
165 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
166
167
168 String snapshotString = "offlineTableSnapshot";
169 byte[] snapshot = Bytes.toBytes(snapshotString);
170 admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
171 LOG.debug("Snapshot completed.");
172
173
174 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
175 snapshot, TABLE_NAME);
176
177
178 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
179 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
180 LOG.debug("FS state after snapshot:");
181 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
182 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
183
184 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
185 admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
186 }
187
188 @Test (timeout=300000)
189 public void testSnapshotFailsOnNonExistantTable() throws Exception {
190 HBaseAdmin admin = UTIL.getHBaseAdmin();
191
192 SnapshotTestingUtils.assertNoSnapshots(admin);
193 String tableName = "_not_a_table";
194
195
196 boolean fail = false;
197 do {
198 try {
199 admin.getTableDescriptor(Bytes.toBytes(tableName));
200 fail = true;
201 LOG.error("Table:" + tableName + " already exists, checking a new name");
202 tableName = tableName+"!";
203 } catch (TableNotFoundException e) {
204 fail = false;
205 }
206 } while (fail);
207
208
209 try {
210 admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
211 fail("Snapshot succeeded even though there is not table.");
212 } catch (SnapshotCreationException e) {
213 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
214 }
215 }
216
217 @Test(timeout = 300000)
218 public void testAsyncFlushSnapshot() throws Exception {
219 HBaseAdmin admin = UTIL.getHBaseAdmin();
220 SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
221 .setTable(TABLE_NAME.getNameAsString())
222 .setType(SnapshotDescription.Type.FLUSH)
223 .build();
224
225
226 admin.takeSnapshotAsync(snapshot);
227
228
229 HMaster master = UTIL.getMiniHBaseCluster().getMaster();
230 SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
231 LOG.info(" === Async Snapshot Completed ===");
232 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
233 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
234
235 SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
236 }
237
238 @Test (timeout=300000)
239 public void testSnapshotStateAfterMerge() throws Exception {
240 int numRows = DEFAULT_NUM_ROWS;
241 HBaseAdmin admin = UTIL.getHBaseAdmin();
242
243 SnapshotTestingUtils.assertNoSnapshots(admin);
244
245 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
246
247
248 String snapshotBeforeMergeName = "snapshotBeforeMerge";
249 admin.snapshot(snapshotBeforeMergeName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
250
251
252 String cloneBeforeMergeName = "cloneBeforeMerge";
253 admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
254 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneBeforeMergeName));
255
256
257 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
258 Collections.sort(regions, new Comparator<HRegionInfo>() {
259 public int compare(HRegionInfo r1, HRegionInfo r2) {
260 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
261 }
262 });
263
264 int numRegions = admin.getTableRegions(TABLE_NAME).size();
265 int numRegionsAfterMerge = numRegions - 2;
266 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
267 regions.get(2).getEncodedNameAsBytes(), true);
268 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
269 regions.get(6).getEncodedNameAsBytes(), true);
270
271
272 waitRegionsAfterMerge(numRegionsAfterMerge);
273 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
274
275
276 String cloneAfterMergeName = "cloneAfterMerge";
277 admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
278 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneAfterMergeName));
279
280 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
281 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneBeforeMergeName), numRows);
282 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneAfterMergeName), numRows);
283
284
285 UTIL.deleteTable(cloneAfterMergeName);
286 UTIL.deleteTable(cloneBeforeMergeName);
287 }
288
289 @Test (timeout=300000)
290 public void testTakeSnapshotAfterMerge() throws Exception {
291 int numRows = DEFAULT_NUM_ROWS;
292 HBaseAdmin admin = UTIL.getHBaseAdmin();
293
294 SnapshotTestingUtils.assertNoSnapshots(admin);
295
296 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
297
298
299 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
300 Collections.sort(regions, new Comparator<HRegionInfo>() {
301 public int compare(HRegionInfo r1, HRegionInfo r2) {
302 return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
303 }
304 });
305
306 int numRegions = admin.getTableRegions(TABLE_NAME).size();
307 int numRegionsAfterMerge = numRegions - 2;
308 admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
309 regions.get(2).getEncodedNameAsBytes(), true);
310 admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
311 regions.get(6).getEncodedNameAsBytes(), true);
312
313 waitRegionsAfterMerge(numRegionsAfterMerge);
314 assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
315
316
317 String snapshotName = "snapshotAfterMerge";
318 SnapshotTestingUtils.snapshot(admin, snapshotName, STRING_TABLE_NAME,
319 SnapshotDescription.Type.FLUSH, 3);
320
321
322 String cloneName = "cloneMerge";
323 admin.cloneSnapshot(snapshotName, cloneName);
324 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TableName.valueOf(cloneName));
325
326 SnapshotTestingUtils.verifyRowCount(UTIL, TABLE_NAME, numRows);
327 SnapshotTestingUtils.verifyRowCount(UTIL, TableName.valueOf(cloneName), numRows);
328
329
330 UTIL.deleteTable(cloneName);
331 }
332
333
334
335
336 @Test (timeout=300000)
337 public void testFlushCreateListDestroy() throws Exception {
338 LOG.debug("------- Starting Snapshot test -------------");
339 HBaseAdmin admin = UTIL.getHBaseAdmin();
340
341 SnapshotTestingUtils.assertNoSnapshots(admin);
342
343 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
344
345 String snapshotName = "flushSnapshotCreateListDestroy";
346
347 admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
348 logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
349
350
351 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
352 snapshotName, TABLE_NAME);
353
354
355 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
356 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
357 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir);
358 assertTrue(fs.exists(snapshotDir));
359 FSUtils.logFileSystemState(UTIL.getTestFileSystem(), snapshotDir, LOG);
360 Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
361 assertTrue(fs.exists(snapshotinfo));
362
363
364 HTableDescriptor desc = FSTableDescriptors.getTableDescriptorFromFs(fs,
365 rootDir, TABLE_NAME);
366 HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptorFromFs(fs,
367 new Path(SnapshotDescriptionUtils.getSnapshotsDir(rootDir), snapshotName));
368 assertEquals(desc, snapshotDesc);
369
370
371 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
372 assertTrue(regions.size() > 1);
373 for (HRegionInfo info : regions) {
374 String regionName = info.getEncodedName();
375 Path regionDir = new Path(snapshotDir, regionName);
376 HRegionInfo snapshotRegionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
377 assertEquals(info, snapshotRegionInfo);
378
379 Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM));
380 assertTrue("Missing region " + Bytes.toString(snapshotRegionInfo.getStartKey()),
381 fs.exists(familyDir));
382
383
384 assertTrue(fs.listStatus(familyDir).length > 0);
385 }
386 }
387
388
389
390
391
392
393 @Test(timeout=300000)
394 public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
395 final String STRING_TABLE2_NAME = STRING_TABLE_NAME + "2";
396 final TableName TABLE2_NAME =
397 TableName.valueOf(STRING_TABLE2_NAME);
398
399 int ssNum = 20;
400 HBaseAdmin admin = UTIL.getHBaseAdmin();
401
402 SnapshotTestingUtils.assertNoSnapshots(admin);
403
404 SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
405
406 SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
407 SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
408
409 final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
410
411 class SSRunnable implements Runnable {
412 SnapshotDescription ss;
413 SSRunnable(SnapshotDescription ss) {
414 this.ss = ss;
415 }
416
417 @Override
418 public void run() {
419 try {
420 HBaseAdmin admin = UTIL.getHBaseAdmin();
421 LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
422 admin.takeSnapshotAsync(ss);
423 } catch (Exception e) {
424 LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
425 ss)
426 + ". This is ok, we expect some", e);
427 }
428 LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
429 toBeSubmitted.countDown();
430 }
431 };
432
433
434 SnapshotDescription[] descs = new SnapshotDescription[ssNum];
435 for (int i = 0; i < ssNum; i++) {
436 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
437 builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
438 builder.setName("ss"+i);
439 builder.setType(SnapshotDescription.Type.FLUSH);
440 descs[i] = builder.build();
441 }
442
443
444 for (int i=0 ; i < ssNum; i++) {
445 new Thread(new SSRunnable(descs[i])).start();
446 }
447
448
449 toBeSubmitted.await();
450
451
452 while (true) {
453 int doneCount = 0;
454 for (SnapshotDescription ss : descs) {
455 try {
456 if (admin.isSnapshotFinished(ss)) {
457 doneCount++;
458 }
459 } catch (Exception e) {
460 LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
461 doneCount++;
462 }
463 }
464 if (doneCount == descs.length) {
465 break;
466 }
467 Thread.sleep(100);
468 }
469
470
471 logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
472
473 List<SnapshotDescription> taken = admin.listSnapshots();
474 int takenSize = taken.size();
475 LOG.info("Taken " + takenSize + " snapshots: " + taken);
476 assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
477 " issued many requests", takenSize < ssNum && takenSize > 0);
478
479
480 int t1SnapshotsCount = 0;
481 int t2SnapshotsCount = 0;
482 for (SnapshotDescription ss : taken) {
483 if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
484 t1SnapshotsCount++;
485 } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
486 t2SnapshotsCount++;
487 }
488 }
489 assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
490 assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
491
492 UTIL.deleteTable(TABLE2_NAME);
493 }
494
495 private void logFSTree(Path root) throws IOException {
496 FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
497 }
498
499 private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
500 throws IOException, InterruptedException {
501 HBaseAdmin admin = UTIL.getHBaseAdmin();
502
503 long startTime = System.currentTimeMillis();
504 while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
505
506
507 if ((System.currentTimeMillis() - startTime) > 15000)
508 break;
509 Thread.sleep(100);
510 }
511 SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
512 }
513 }