1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.HashSet;
26 import java.util.List;
27 import java.util.Set;
28 import java.util.concurrent.CountDownLatch;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.HRegionInfo;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.LargeTests;
40 import org.apache.hadoop.hbase.TableNotFoundException;
41 import org.apache.hadoop.hbase.client.HBaseAdmin;
42 import org.apache.hadoop.hbase.client.HTable;
43 import org.apache.hadoop.hbase.master.HMaster;
44 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
45 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
46 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
47 import org.apache.hadoop.hbase.regionserver.HRegion;
48 import org.apache.hadoop.hbase.regionserver.HRegionServer;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.FSTableDescriptors;
51 import org.apache.hadoop.hbase.util.FSUtils;
52 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
53 import org.junit.After;
54 import org.junit.AfterClass;
55 import org.junit.Before;
56 import org.junit.BeforeClass;
57 import org.junit.Test;
58 import org.junit.experimental.categories.Category;
59
60
61
62
63
64
65
66
67
68 @Category(LargeTests.class)
69 public class TestFlushSnapshotFromClient {
70 private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
71 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
72 private static final int NUM_RS = 2;
73 private static final String STRING_TABLE_NAME = "test";
74 private static final byte[] TEST_FAM = Bytes.toBytes("fam");
75 private static final byte[] TABLE_NAME = Bytes.toBytes(STRING_TABLE_NAME);
76
77
78
79
80
81 @BeforeClass
82 public static void setupCluster() throws Exception {
83 setupConf(UTIL.getConfiguration());
84 UTIL.startMiniCluster(NUM_RS);
85 }
86
87 private static void setupConf(Configuration conf) {
88
89 conf.setInt("hbase.regionsever.info.port", -1);
90
91 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
92
93
94 conf.setInt("hbase.hstore.compaction.min", 10);
95 conf.setInt("hbase.hstore.compactionThreshold", 10);
96
97 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
98
99 conf.setInt("hbase.client.retries.number", 1);
100
101 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
102
103 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
104 ConstantSizeRegionSplitPolicy.class.getName());
105 }
106
107 @Before
108 public void setup() throws Exception {
109 UTIL.createTable(TABLE_NAME, TEST_FAM);
110 }
111
112 @After
113 public void tearDown() throws Exception {
114 UTIL.deleteTable(TABLE_NAME);
115
116 try {
117 UTIL.getTestFileSystem().delete(new Path(UTIL.getDefaultRootDirPath(), ".archive"), true);
118 } catch (IOException e) {
119 LOG.warn("Failure to delete archive directory", e);
120 }
121 }
122
123 @AfterClass
124 public static void cleanupTest() throws Exception {
125 try {
126 UTIL.shutdownMiniCluster();
127 } catch (Exception e) {
128 LOG.warn("failure shutting down cluster", e);
129 }
130 }
131
132
133
134
135
136 @Test
137 public void testFlushTableSnapshot() throws Exception {
138 HBaseAdmin admin = UTIL.getHBaseAdmin();
139
140 SnapshotTestingUtils.assertNoSnapshots(admin);
141
142
143 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
144 UTIL.loadTable(table, TEST_FAM);
145
146
147 Set<String> snapshotServers = new HashSet<String>();
148 List<RegionServerThread> servers = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
149 for (RegionServerThread server : servers) {
150 if (server.getRegionServer().getOnlineRegions(TABLE_NAME).size() > 0) {
151 snapshotServers.add(server.getRegionServer().getServerName().toString());
152 }
153 }
154
155 LOG.debug("FS state before snapshot:");
156 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
157 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
158
159
160 String snapshotString = "offlineTableSnapshot";
161 byte[] snapshot = Bytes.toBytes(snapshotString);
162 admin.snapshot(snapshotString, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
163 LOG.debug("Snapshot completed.");
164
165
166 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
167 snapshot, TABLE_NAME);
168
169
170 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
171 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
172 LOG.debug("FS state after snapshot:");
173 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
174 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
175
176 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
177 admin, fs, false, new Path(rootDir, HConstants.HREGION_LOGDIR_NAME), snapshotServers);
178
179 admin.deleteSnapshot(snapshot);
180 snapshots = admin.listSnapshots();
181 SnapshotTestingUtils.assertNoSnapshots(admin);
182 }
183
184 @Test
185 public void testSnapshotFailsOnNonExistantTable() throws Exception {
186 HBaseAdmin admin = UTIL.getHBaseAdmin();
187
188 SnapshotTestingUtils.assertNoSnapshots(admin);
189 String tableName = "_not_a_table";
190
191
192 boolean fail = false;
193 do {
194 try {
195 admin.getTableDescriptor(Bytes.toBytes(tableName));
196 fail = true;
197 LOG.error("Table:" + tableName + " already exists, checking a new name");
198 tableName = tableName+"!";
199 } catch (TableNotFoundException e) {
200 fail = false;
201 }
202 } while (fail);
203
204
205 try {
206 admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
207 fail("Snapshot succeeded even though there is not table.");
208 } catch (SnapshotCreationException e) {
209 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
210 }
211 }
212
213 @Test(timeout = 15000)
214 public void testAsyncFlushSnapshot() throws Exception {
215 HBaseAdmin admin = UTIL.getHBaseAdmin();
216 SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
217 .setTable(STRING_TABLE_NAME).setType(SnapshotDescription.Type.FLUSH).build();
218
219
220 admin.takeSnapshotAsync(snapshot);
221
222
223 HMaster master = UTIL.getMiniHBaseCluster().getMaster();
224 SnapshotTestingUtils.waitForSnapshotToComplete(master, new HSnapshotDescription(snapshot), 200);
225 LOG.info(" === Async Snapshot Completed ===");
226 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
227 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
228
229 SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
230
231
232 admin.deleteSnapshot(snapshot.getName());
233 LOG.info(" === Async Snapshot Deleted ===");
234 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
235 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
236
237 SnapshotTestingUtils.assertNoSnapshots(admin);
238 LOG.info(" === Async Snapshot Test Completed ===");
239
240 }
241
242
243
244
245 @Test
246 public void testFlushCreateListDestroy() throws Exception {
247 LOG.debug("------- Starting Snapshot test -------------");
248 HBaseAdmin admin = UTIL.getHBaseAdmin();
249
250 SnapshotTestingUtils.assertNoSnapshots(admin);
251
252 UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
253
254 HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
255 List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
256 for (HRegion region : onlineRegions) {
257 region.waitForFlushesAndCompactions();
258 }
259 String snapshotName = "flushSnapshotCreateListDestroy";
260
261 admin.snapshot(snapshotName, STRING_TABLE_NAME, SnapshotDescription.Type.FLUSH);
262 logFSTree(new Path(UTIL.getConfiguration().get(HConstants.HBASE_DIR)));
263
264
265 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
266 snapshotName, STRING_TABLE_NAME);
267
268
269 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
270 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
271 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshots.get(0), rootDir);
272 assertTrue(fs.exists(snapshotDir));
273 FSUtils.logFileSystemState(UTIL.getTestFileSystem(), snapshotDir, LOG);
274 Path snapshotinfo = new Path(snapshotDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE);
275 assertTrue(fs.exists(snapshotinfo));
276
277
278 HTableDescriptor desc = FSTableDescriptors.getTableDescriptor(fs, rootDir, TABLE_NAME);
279 HTableDescriptor snapshotDesc = FSTableDescriptors.getTableDescriptor(fs,
280 SnapshotDescriptionUtils.getSnapshotsDir(rootDir), Bytes.toBytes(snapshotName));
281 assertEquals(desc, snapshotDesc);
282
283
284 List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
285 for (HRegionInfo info : regions) {
286 String regionName = info.getEncodedName();
287 Path regionDir = new Path(snapshotDir, regionName);
288 HRegionInfo snapshotRegionInfo = HRegion.loadDotRegionInfoFileContent(fs, regionDir);
289 assertEquals(info, snapshotRegionInfo);
290
291 Path familyDir = new Path(regionDir, Bytes.toString(TEST_FAM));
292 assertTrue(fs.exists(familyDir));
293
294 assertTrue(fs.listStatus(familyDir).length > 0);
295 }
296
297
298 admin.deleteSnapshot(snapshotName);
299 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
300 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
301
302
303 SnapshotTestingUtils.assertNoSnapshots(admin);
304 LOG.debug("------- Flush-Snapshot Create List Destroy-------------");
305 }
306
307
308
309
310 @Test(timeout=60000)
311 public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
312 int ssNum = 10;
313 HBaseAdmin admin = UTIL.getHBaseAdmin();
314
315 SnapshotTestingUtils.assertNoSnapshots(admin);
316
317 UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
318
319 HRegionServer rs = UTIL.getRSForFirstRegionInTable(TABLE_NAME);
320 List<HRegion> onlineRegions = rs.getOnlineRegions(TABLE_NAME);
321 for (HRegion region : onlineRegions) {
322 region.waitForFlushesAndCompactions();
323 }
324
325
326 SnapshotDescription[] descs = new SnapshotDescription[ssNum];
327 for (int i = 0; i < ssNum; i++) {
328 SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
329 builder.setTable(STRING_TABLE_NAME);
330 builder.setName("ss"+i);
331 builder.setType(SnapshotDescription.Type.FLUSH);
332 descs[i] = builder.build();
333 }
334
335 final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
336
337 class SSRunnable implements Runnable {
338 SnapshotDescription ss;
339 SSRunnable(SnapshotDescription ss) {
340 this.ss = ss;
341 }
342
343 @Override
344 public void run() {
345 try {
346 HBaseAdmin admin = UTIL.getHBaseAdmin();
347 LOG.info("Submitting snapshot request: " + SnapshotDescriptionUtils.toString(ss));
348 admin.takeSnapshotAsync(ss);
349 } catch (Exception e) {
350 LOG.info("Exception during snapshot request: " + SnapshotDescriptionUtils.toString(ss)
351 + ". This is ok, we expect some", e);
352 }
353 LOG.info("Submitted snapshot request: " + SnapshotDescriptionUtils.toString(ss));
354 toBeSubmitted.countDown();
355 }
356 };
357
358
359 for (int i=0 ; i < ssNum; i++) {
360 new Thread(new SSRunnable(descs[i])).start();
361 }
362
363
364 toBeSubmitted.await();
365
366
367 while (true) {
368 int doneCount = 0;
369 for (SnapshotDescription ss : descs) {
370 try {
371 if (admin.isSnapshotFinished(ss)) {
372 doneCount++;
373 }
374 } catch (Exception e) {
375 LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
376 doneCount++;
377 }
378 }
379 if (doneCount == descs.length) {
380 break;
381 }
382 Thread.sleep(100);
383 }
384
385
386 logFSTree(new Path(UTIL.getConfiguration().get(HConstants.HBASE_DIR)));
387
388 List<SnapshotDescription> taken = admin.listSnapshots();
389 int takenSize = taken.size();
390 LOG.info("Taken " + takenSize + " snapshots: " + taken);
391 assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
392 " issued many requests", takenSize < ssNum && takenSize > 0);
393
394 for (SnapshotDescription ss : taken) {
395 admin.deleteSnapshot(ss.getName());
396 }
397 }
398
399 private void logFSTree(Path root) throws IOException {
400 FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
401 }
402 }