1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertTrue;
23 import static org.junit.Assert.fail;
24
25 import java.util.ArrayList;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HTableDescriptor;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.testclassification.LargeTests;
38 import org.apache.hadoop.hbase.TableNotFoundException;
39 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
40 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
41 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
42 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
43 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
44 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
45 import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.FSUtils;
48 import org.junit.After;
49 import org.junit.AfterClass;
50 import org.junit.Before;
51 import org.junit.BeforeClass;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55 import com.google.common.collect.Lists;
56
57
58
59
60
61
62 @Category(LargeTests.class)
63 public class TestSnapshotFromClient {
64 private static final Log LOG = LogFactory.getLog(TestSnapshotFromClient.class);
65 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
66 private static final int NUM_RS = 2;
67 private static final String STRING_TABLE_NAME = "test";
68 private static final byte[] TEST_FAM = Bytes.toBytes("fam");
69 private static final TableName TABLE_NAME =
70 TableName.valueOf(STRING_TABLE_NAME);
71
72
73
74
75
76 @BeforeClass
77 public static void setupCluster() throws Exception {
78 setupConf(UTIL.getConfiguration());
79 UTIL.startMiniCluster(NUM_RS);
80 }
81
82 private static void setupConf(Configuration conf) {
83
84 conf.setInt("hbase.regionsever.info.port", -1);
85
86 conf.setInt("hbase.hregion.memstore.flush.size", 25000);
87
88
89 conf.setInt("hbase.hstore.compaction.min", 10);
90 conf.setInt("hbase.hstore.compactionThreshold", 10);
91
92 conf.setInt("hbase.hstore.blockingStoreFiles", 12);
93
94 conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
95 conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
96 ConstantSizeRegionSplitPolicy.class.getName());
97 }
98
99 @Before
100 public void setup() throws Exception {
101 UTIL.createTable(TABLE_NAME, TEST_FAM);
102 }
103
104 @After
105 public void tearDown() throws Exception {
106 UTIL.deleteTable(TABLE_NAME);
107 SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
108 SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
109 }
110
111 @AfterClass
112 public static void cleanupTest() throws Exception {
113 try {
114 UTIL.shutdownMiniCluster();
115 } catch (Exception e) {
116 LOG.warn("failure shutting down cluster", e);
117 }
118 }
119
120
121
122
123
124 @Test (timeout=300000)
125 public void testMetaTablesSnapshot() throws Exception {
126 HBaseAdmin admin = UTIL.getHBaseAdmin();
127 byte[] snapshotName = Bytes.toBytes("metaSnapshot");
128
129 try {
130 admin.snapshot(snapshotName, TableName.META_TABLE_NAME);
131 fail("taking a snapshot of hbase:meta should not be allowed");
132 } catch (IllegalArgumentException e) {
133
134 }
135 }
136
137
138
139
140
141
142 @Test (timeout=300000)
143 public void testSnapshotDeletionWithRegex() throws Exception {
144 HBaseAdmin admin = UTIL.getHBaseAdmin();
145
146 SnapshotTestingUtils.assertNoSnapshots(admin);
147
148
149 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
150 UTIL.loadTable(table, TEST_FAM);
151 table.close();
152
153 byte[] snapshot1 = Bytes.toBytes("TableSnapshot1");
154 admin.snapshot(snapshot1, TABLE_NAME);
155 LOG.debug("Snapshot1 completed.");
156
157 byte[] snapshot2 = Bytes.toBytes("TableSnapshot2");
158 admin.snapshot(snapshot2, TABLE_NAME);
159 LOG.debug("Snapshot2 completed.");
160
161 String snapshot3 = "3rdTableSnapshot";
162 admin.snapshot(Bytes.toBytes(snapshot3), TABLE_NAME);
163 LOG.debug(snapshot3 + " completed.");
164
165
166 admin.deleteSnapshots("TableSnapshot.*");
167 List<SnapshotDescription> snapshots = admin.listSnapshots();
168 assertEquals(1, snapshots.size());
169 assertEquals(snapshots.get(0).getName(), snapshot3);
170
171 admin.deleteSnapshot(snapshot3);
172 admin.close();
173 }
174
175
176
177
178 @Test (timeout=300000)
179 public void testOfflineTableSnapshot() throws Exception {
180 HBaseAdmin admin = UTIL.getHBaseAdmin();
181
182 SnapshotTestingUtils.assertNoSnapshots(admin);
183
184
185 HTable table = new HTable(UTIL.getConfiguration(), TABLE_NAME);
186 UTIL.loadTable(table, TEST_FAM, false);
187
188 LOG.debug("FS state before disable:");
189 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
190 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
191
192
193 admin.disableTable(TABLE_NAME);
194
195 LOG.debug("FS state before snapshot:");
196 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
197 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
198
199
200 final String SNAPSHOT_NAME = "offlineTableSnapshot";
201 byte[] snapshot = Bytes.toBytes(SNAPSHOT_NAME);
202
203 SnapshotDescription desc = SnapshotDescription.newBuilder()
204 .setType(SnapshotDescription.Type.DISABLED)
205 .setTable(STRING_TABLE_NAME)
206 .setName(SNAPSHOT_NAME)
207 .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
208 .build();
209 admin.snapshot(desc);
210 LOG.debug("Snapshot completed.");
211
212
213 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
214 snapshot, TABLE_NAME);
215
216
217 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
218 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
219 LOG.debug("FS state after snapshot:");
220 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
221 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
222
223 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
224 admin, fs);
225
226 admin.deleteSnapshot(snapshot);
227 snapshots = admin.listSnapshots();
228 SnapshotTestingUtils.assertNoSnapshots(admin);
229 }
230
231 @Test (timeout=300000)
232 public void testSnapshotFailsOnNonExistantTable() throws Exception {
233 HBaseAdmin admin = UTIL.getHBaseAdmin();
234
235 SnapshotTestingUtils.assertNoSnapshots(admin);
236 String tableName = "_not_a_table";
237
238
239 boolean fail = false;
240 do {
241 try {
242 admin.getTableDescriptor(Bytes.toBytes(tableName));
243 fail = true;
244 LOG.error("Table:" + tableName + " already exists, checking a new name");
245 tableName = tableName+"!";
246 } catch (TableNotFoundException e) {
247 fail = false;
248 }
249 } while (fail);
250
251
252 try {
253 admin.snapshot("fail", tableName);
254 fail("Snapshot succeeded even though there is not table.");
255 } catch (SnapshotCreationException e) {
256 LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
257 }
258 }
259
260 @Test (timeout=300000)
261 public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
262
263
264 HBaseAdmin admin = UTIL.getHBaseAdmin();
265
266 SnapshotTestingUtils.assertNoSnapshots(admin);
267
268 LOG.debug("FS state before disable:");
269 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
270 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
271 admin.disableTable(TABLE_NAME);
272
273 LOG.debug("FS state before snapshot:");
274 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
275 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
276
277
278 byte[] snapshot = Bytes.toBytes("testOfflineTableSnapshotWithEmptyRegions");
279 admin.snapshot(snapshot, TABLE_NAME);
280 LOG.debug("Snapshot completed.");
281
282
283 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
284 snapshot, TABLE_NAME);
285
286
287 FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
288 Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
289 LOG.debug("FS state after snapshot:");
290 FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
291 FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
292
293 List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM);
294 List<byte[]> nonEmptyCfs = Lists.newArrayList();
295 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, nonEmptyCfs, emptyCfs,
296 rootDir, admin, fs);
297
298 admin.deleteSnapshot(snapshot);
299 snapshots = admin.listSnapshots();
300 SnapshotTestingUtils.assertNoSnapshots(admin);
301 }
302
303 @Test(timeout = 300000)
304 public void testListTableSnapshots() throws Exception {
305 HBaseAdmin admin = null;
306 TableName tableName2 = TableName.valueOf("testListTableSnapshots");
307 try {
308 admin = UTIL.getHBaseAdmin();
309
310 HTableDescriptor htd = new HTableDescriptor(tableName2);
311 UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
312
313 String table1Snapshot1 = "Table1Snapshot1";
314 admin.snapshot(table1Snapshot1, TABLE_NAME);
315 LOG.debug("Snapshot1 completed.");
316
317 String table1Snapshot2 = "Table1Snapshot2";
318 admin.snapshot(table1Snapshot2, TABLE_NAME);
319 LOG.debug("Snapshot2 completed.");
320
321 String table2Snapshot1 = "Table2Snapshot1";
322 admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName2);
323 LOG.debug(table2Snapshot1 + " completed.");
324
325 List<SnapshotDescription> listTableSnapshots = admin.listTableSnapshots("test.*", ".*");
326 List<String> listTableSnapshotNames = new ArrayList<String>();
327 assertEquals(3, listTableSnapshots.size());
328 for (SnapshotDescription s : listTableSnapshots) {
329 listTableSnapshotNames.add(s.getName());
330 }
331 assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
332 assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
333 assertTrue(listTableSnapshotNames.contains(table2Snapshot1));
334 } finally {
335 if (admin != null) {
336 try {
337 admin.deleteSnapshots("Table.*");
338 } catch (SnapshotDoesNotExistException ignore) {
339 }
340 if (admin.tableExists(tableName2)) {
341 UTIL.deleteTable(tableName2);
342 }
343 admin.close();
344 }
345 }
346 }
347
348 @Test(timeout = 300000)
349 public void testListTableSnapshotsWithRegex() throws Exception {
350 HBaseAdmin admin = null;
351 try {
352 admin = UTIL.getHBaseAdmin();
353
354 String table1Snapshot1 = "Table1Snapshot1";
355 admin.snapshot(table1Snapshot1, TABLE_NAME);
356 LOG.debug("Snapshot1 completed.");
357
358 String table1Snapshot2 = "Table1Snapshot2";
359 admin.snapshot(table1Snapshot2, TABLE_NAME);
360 LOG.debug("Snapshot2 completed.");
361
362 String table2Snapshot1 = "Table2Snapshot1";
363 admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
364 LOG.debug(table2Snapshot1 + " completed.");
365
366 List<SnapshotDescription> listTableSnapshots = admin.listTableSnapshots("test.*", "Table1.*");
367 List<String> listTableSnapshotNames = new ArrayList<String>();
368 assertEquals(2, listTableSnapshots.size());
369 for (SnapshotDescription s : listTableSnapshots) {
370 listTableSnapshotNames.add(s.getName());
371 }
372 assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
373 assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
374 assertFalse(listTableSnapshotNames.contains(table2Snapshot1));
375 } finally {
376 if (admin != null) {
377 try {
378 admin.deleteSnapshots("Table.*");
379 } catch (SnapshotDoesNotExistException ignore) {
380 }
381 admin.close();
382 }
383 }
384 }
385
386 @Test(timeout = 300000)
387 public void testDeleteTableSnapshots() throws Exception {
388 HBaseAdmin admin = null;
389 TableName tableName2 = TableName.valueOf("testListTableSnapshots");
390 try {
391 admin = UTIL.getHBaseAdmin();
392
393 HTableDescriptor htd = new HTableDescriptor(tableName2);
394 UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
395
396 String table1Snapshot1 = "Table1Snapshot1";
397 admin.snapshot(table1Snapshot1, TABLE_NAME);
398 LOG.debug("Snapshot1 completed.");
399
400 String table1Snapshot2 = "Table1Snapshot2";
401 admin.snapshot(table1Snapshot2, TABLE_NAME);
402 LOG.debug("Snapshot2 completed.");
403
404 String table2Snapshot1 = "Table2Snapshot1";
405 admin.snapshot(Bytes.toBytes(table2Snapshot1), tableName2);
406 LOG.debug(table2Snapshot1 + " completed.");
407
408 admin.deleteTableSnapshots("test.*", ".*");
409 assertEquals(0, admin.listTableSnapshots("test.*", ".*").size());
410 } finally {
411 if (admin != null) {
412 if (admin.tableExists(tableName2)) {
413 UTIL.deleteTable(tableName2);
414 }
415 admin.close();
416 }
417 }
418 }
419
420 @Test(timeout = 300000)
421 public void testDeleteTableSnapshotsWithRegex() throws Exception {
422 HBaseAdmin admin = null;
423 try {
424 admin = UTIL.getHBaseAdmin();
425
426 String table1Snapshot1 = "Table1Snapshot1";
427 admin.snapshot(table1Snapshot1, TABLE_NAME);
428 LOG.debug("Snapshot1 completed.");
429
430 String table1Snapshot2 = "Table1Snapshot2";
431 admin.snapshot(table1Snapshot2, TABLE_NAME);
432 LOG.debug("Snapshot2 completed.");
433
434 String table2Snapshot1 = "Table2Snapshot1";
435 admin.snapshot(Bytes.toBytes(table2Snapshot1), TABLE_NAME);
436 LOG.debug(table2Snapshot1 + " completed.");
437
438 admin.deleteTableSnapshots("test.*", "Table1.*");
439 assertEquals(1, admin.listTableSnapshots("test.*", ".*").size());
440 } finally {
441 if (admin != null) {
442 try {
443 admin.deleteTableSnapshots("test.*", ".*");
444 } catch (SnapshotDoesNotExistException ignore) {
445 }
446 admin.close();
447 }
448 }
449 }
450 }