1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.util.HashSet;
26 import java.util.Set;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.fs.Path;
31 import org.apache.hadoop.hbase.TableName;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HColumnDescriptor;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HTableDescriptor;
36 import org.apache.hadoop.hbase.testclassification.LargeTests;
37 import org.apache.hadoop.hbase.master.MasterFileSystem;
38 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
39 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
40 import org.apache.hadoop.hbase.snapshot.CorruptedSnapshotException;
41 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.FSUtils;
44 import org.junit.After;
45 import org.junit.AfterClass;
46 import org.junit.Before;
47 import org.junit.BeforeClass;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51
52
53
54 @Category(LargeTests.class)
55 public class TestRestoreSnapshotFromClient {
56 final Log LOG = LogFactory.getLog(getClass());
57
58 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59
60 private final byte[] FAMILY = Bytes.toBytes("cf");
61
62 private byte[] emptySnapshot;
63 private byte[] snapshotName0;
64 private byte[] snapshotName1;
65 private byte[] snapshotName2;
66 private int snapshot0Rows;
67 private int snapshot1Rows;
68 private TableName tableName;
69 private Admin admin;
70
71 @BeforeClass
72 public static void setUpBeforeClass() throws Exception {
73 TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
74 TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
75 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
76 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
77 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
78 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 6);
79 TEST_UTIL.getConfiguration().setBoolean(
80 "hbase.master.enabletable.roundrobin", true);
81 TEST_UTIL.startMiniCluster(3);
82 }
83
84 @AfterClass
85 public static void tearDownAfterClass() throws Exception {
86 TEST_UTIL.shutdownMiniCluster();
87 }
88
89
90
91
92
93
94 @Before
95 public void setup() throws Exception {
96 this.admin = TEST_UTIL.getHBaseAdmin();
97
98 long tid = System.currentTimeMillis();
99 tableName =
100 TableName.valueOf("testtb-" + tid);
101 emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
102 snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
103 snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
104 snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
105
106
107 SnapshotTestingUtils.createTable(TEST_UTIL, tableName, getNumReplicas(), FAMILY);
108 admin.disableTable(tableName);
109
110
111 admin.snapshot(emptySnapshot, tableName);
112
113
114 admin.enableTable(tableName);
115 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
116 try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
117 snapshot0Rows = TEST_UTIL.countRows(table);
118 }
119 admin.disableTable(tableName);
120
121
122 admin.snapshot(snapshotName0, tableName);
123
124
125 admin.enableTable(tableName);
126 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, FAMILY);
127 try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
128 snapshot1Rows = TEST_UTIL.countRows(table);
129 }
130 }
131
132 @After
133 public void tearDown() throws Exception {
134 TEST_UTIL.deleteTable(tableName);
135 SnapshotTestingUtils.deleteAllSnapshots(TEST_UTIL.getHBaseAdmin());
136 SnapshotTestingUtils.deleteArchiveDirectory(TEST_UTIL);
137 }
138
139 @Test
140 public void testRestoreSnapshot() throws IOException {
141 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
142 admin.disableTable(tableName);
143 admin.snapshot(snapshotName1, tableName);
144
145 admin.restoreSnapshot(snapshotName0);
146 admin.enableTable(tableName);
147 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
148 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
149
150
151 admin.disableTable(tableName);
152 admin.restoreSnapshot(emptySnapshot);
153 admin.enableTable(tableName);
154 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, 0);
155 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
156
157
158 admin.disableTable(tableName);
159 admin.restoreSnapshot(snapshotName1);
160 admin.enableTable(tableName);
161 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
162 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
163
164
165 TEST_UTIL.deleteTable(tableName);
166 admin.restoreSnapshot(snapshotName1);
167 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot1Rows);
168 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
169 }
170
171 protected int getNumReplicas() {
172 return 1;
173 }
174
175 @Test
176 public void testRestoreSchemaChange() throws Exception {
177 byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
178
179 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
180
181
182 admin.disableTable(tableName);
183 admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2));
184 admin.enableTable(tableName);
185 assertEquals(2, table.getTableDescriptor().getFamilies().size());
186 HTableDescriptor htd = admin.getTableDescriptor(tableName);
187 assertEquals(2, htd.getFamilies().size());
188 SnapshotTestingUtils.loadData(TEST_UTIL, tableName, 500, TEST_FAMILY2);
189 long snapshot2Rows = snapshot1Rows + 500;
190 assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
191 assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
192 Set<String> fsFamilies = getFamiliesFromFS(tableName);
193 assertEquals(2, fsFamilies.size());
194
195
196 admin.disableTable(tableName);
197 admin.snapshot(snapshotName2, tableName);
198
199
200 admin.restoreSnapshot(snapshotName0);
201 admin.enableTable(tableName);
202 assertEquals(1, table.getTableDescriptor().getFamilies().size());
203 try {
204 TEST_UTIL.countRows(table, TEST_FAMILY2);
205 fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
206 } catch (NoSuchColumnFamilyException e) {
207
208 }
209 assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
210 htd = admin.getTableDescriptor(tableName);
211 assertEquals(1, htd.getFamilies().size());
212 fsFamilies = getFamiliesFromFS(tableName);
213 assertEquals(1, fsFamilies.size());
214
215
216 admin.disableTable(tableName);
217 admin.restoreSnapshot(snapshotName2);
218 admin.enableTable(tableName);
219 htd = admin.getTableDescriptor(tableName);
220 assertEquals(2, htd.getFamilies().size());
221 assertEquals(2, table.getTableDescriptor().getFamilies().size());
222 assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
223 assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
224 fsFamilies = getFamiliesFromFS(tableName);
225 assertEquals(2, fsFamilies.size());
226 table.close();
227 }
228
229 @Test
230 public void testCloneSnapshotOfCloned() throws IOException, InterruptedException {
231 TableName clonedTableName =
232 TableName.valueOf("clonedtb-" + System.currentTimeMillis());
233 admin.cloneSnapshot(snapshotName0, clonedTableName);
234 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
235 SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
236 admin.disableTable(clonedTableName);
237 admin.snapshot(snapshotName2, clonedTableName);
238 TEST_UTIL.deleteTable(clonedTableName);
239 waitCleanerRun();
240
241 admin.cloneSnapshot(snapshotName2, clonedTableName);
242 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, clonedTableName, snapshot0Rows);
243 SnapshotTestingUtils.verifyReplicasCameOnline(clonedTableName, admin, getNumReplicas());
244 TEST_UTIL.deleteTable(clonedTableName);
245 }
246
247 @Test
248 public void testCloneAndRestoreSnapshot() throws IOException, InterruptedException {
249 TEST_UTIL.deleteTable(tableName);
250 waitCleanerRun();
251
252 admin.cloneSnapshot(snapshotName0, tableName);
253 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
254 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
255 waitCleanerRun();
256
257 admin.disableTable(tableName);
258 admin.restoreSnapshot(snapshotName0);
259 admin.enableTable(tableName);
260 SnapshotTestingUtils.verifyRowCount(TEST_UTIL, tableName, snapshot0Rows);
261 SnapshotTestingUtils.verifyReplicasCameOnline(tableName, admin, getNumReplicas());
262 }
263
264 @Test
265 public void testCorruptedSnapshot() throws IOException, InterruptedException {
266 SnapshotTestingUtils.corruptSnapshot(TEST_UTIL, Bytes.toString(snapshotName0));
267 TableName cloneName = TableName.valueOf("corruptedClone-" + System.currentTimeMillis());
268 try {
269 admin.cloneSnapshot(snapshotName0, cloneName);
270 fail("Expected CorruptedSnapshotException, got succeeded cloneSnapshot()");
271 } catch (CorruptedSnapshotException e) {
272
273
274 assertFalse(admin.tableExists(cloneName));
275 } catch (Exception e) {
276 fail("Expected CorruptedSnapshotException got: " + e);
277 }
278 }
279
280
281
282
283 private void waitCleanerRun() throws InterruptedException {
284 TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
285 }
286
287 private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
288 MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
289 Set<String> families = new HashSet<String>();
290 Path tableDir = FSUtils.getTableDir(mfs.getRootDir(), tableName);
291 for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
292 for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
293 families.add(familyDir.getName());
294 }
295 }
296 return families;
297 }
298 }