1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.fail;
22
23 import java.io.IOException;
24 import java.util.HashSet;
25 import java.util.Set;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.fs.Path;
30 import org.apache.hadoop.hbase.HBaseTestingUtility;
31 import org.apache.hadoop.hbase.HColumnDescriptor;
32 import org.apache.hadoop.hbase.HConstants;
33 import org.apache.hadoop.hbase.HTableDescriptor;
34 import org.apache.hadoop.hbase.LargeTests;
35 import org.apache.hadoop.hbase.master.MasterFileSystem;
36 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
37 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.FSUtils;
40 import org.apache.hadoop.hbase.util.MD5Hash;
41 import org.junit.After;
42 import org.junit.AfterClass;
43 import org.junit.Before;
44 import org.junit.BeforeClass;
45 import org.junit.Test;
46 import org.junit.experimental.categories.Category;
47
48
49
50
51 @Category(LargeTests.class)
52 public class TestRestoreSnapshotFromClient {
53 final Log LOG = LogFactory.getLog(getClass());
54
55 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
56
57 private final byte[] FAMILY = Bytes.toBytes("cf");
58
59 private byte[] emptySnapshot;
60 private byte[] snapshotName0;
61 private byte[] snapshotName1;
62 private byte[] snapshotName2;
63 private int snapshot0Rows;
64 private int snapshot1Rows;
65 private byte[] tableName;
66 private HBaseAdmin admin;
67
68 @BeforeClass
69 public static void setUpBeforeClass() throws Exception {
70 TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
71 TEST_UTIL.getConfiguration().setBoolean("hbase.online.schema.update.enable", true);
72 TEST_UTIL.getConfiguration().setInt("hbase.hstore.compactionThreshold", 10);
73 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
74 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
75 TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
76 TEST_UTIL.getConfiguration().setBoolean(
77 "hbase.master.enabletable.roundrobin", true);
78 TEST_UTIL.startMiniCluster(3);
79 }
80
81 @AfterClass
82 public static void tearDownAfterClass() throws Exception {
83 TEST_UTIL.shutdownMiniCluster();
84 }
85
86
87
88
89
90
91 @Before
92 public void setup() throws Exception {
93 this.admin = TEST_UTIL.getHBaseAdmin();
94
95 long tid = System.currentTimeMillis();
96 tableName = Bytes.toBytes("testtb-" + tid);
97 emptySnapshot = Bytes.toBytes("emptySnaptb-" + tid);
98 snapshotName0 = Bytes.toBytes("snaptb0-" + tid);
99 snapshotName1 = Bytes.toBytes("snaptb1-" + tid);
100 snapshotName2 = Bytes.toBytes("snaptb2-" + tid);
101
102
103 createTable(tableName, FAMILY);
104 admin.disableTable(tableName);
105
106
107 admin.snapshot(emptySnapshot, tableName);
108
109 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
110 try {
111
112 admin.enableTable(tableName);
113 loadData(table, 500, FAMILY);
114 snapshot0Rows = TEST_UTIL.countRows(table);
115 admin.disableTable(tableName);
116
117
118 admin.snapshot(snapshotName0, tableName);
119
120
121 admin.enableTable(tableName);
122 loadData(table, 500, FAMILY);
123 snapshot1Rows = TEST_UTIL.countRows(table);
124 admin.disableTable(tableName);
125
126
127 admin.snapshot(snapshotName1, tableName);
128
129
130 admin.enableTable(tableName);
131 } finally {
132 table.close();
133 }
134 }
135
136 @After
137 public void tearDown() throws Exception {
138 if (admin.tableExists(tableName)) {
139 TEST_UTIL.deleteTable(tableName);
140 }
141 admin.deleteSnapshot(snapshotName0);
142 admin.deleteSnapshot(snapshotName1);
143
144
145 MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
146 mfs.getFileSystem().delete(
147 new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY), true);
148 }
149
150 @Test
151 public void testRestoreSnapshot() throws IOException {
152 verifyRowCount(tableName, snapshot1Rows);
153
154
155 admin.disableTable(tableName);
156 admin.restoreSnapshot(snapshotName0);
157 admin.enableTable(tableName);
158 verifyRowCount(tableName, snapshot0Rows);
159
160
161 admin.disableTable(tableName);
162 admin.restoreSnapshot(emptySnapshot);
163 admin.enableTable(tableName);
164 verifyRowCount(tableName, 0);
165
166
167 admin.disableTable(tableName);
168 admin.restoreSnapshot(snapshotName1);
169 admin.enableTable(tableName);
170 verifyRowCount(tableName, snapshot1Rows);
171 }
172
173 @Test
174 public void testRestoreSchemaChange() throws IOException {
175 byte[] TEST_FAMILY2 = Bytes.toBytes("cf2");
176
177 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
178
179
180 admin.disableTable(tableName);
181 admin.addColumn(tableName, new HColumnDescriptor(TEST_FAMILY2));
182 admin.enableTable(tableName);
183 assertEquals(2, table.getTableDescriptor().getFamilies().size());
184 HTableDescriptor htd = admin.getTableDescriptor(tableName);
185 assertEquals(2, htd.getFamilies().size());
186 loadData(table, 500, TEST_FAMILY2);
187 long snapshot2Rows = snapshot1Rows + 500;
188 assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
189 assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
190 Set<String> fsFamilies = getFamiliesFromFS(tableName);
191 assertEquals(2, fsFamilies.size());
192 table.close();
193
194
195 admin.disableTable(tableName);
196 admin.snapshot(snapshotName2, tableName);
197
198
199 admin.restoreSnapshot(snapshotName0);
200 assertEquals(1, table.getTableDescriptor().getFamilies().size());
201 admin.enableTable(tableName);
202 try {
203 TEST_UTIL.countRows(table, TEST_FAMILY2);
204 fail("family '" + Bytes.toString(TEST_FAMILY2) + "' should not exists");
205 } catch (NoSuchColumnFamilyException e) {
206
207 }
208 assertEquals(snapshot0Rows, TEST_UTIL.countRows(table));
209 htd = admin.getTableDescriptor(tableName);
210 assertEquals(1, htd.getFamilies().size());
211 fsFamilies = getFamiliesFromFS(tableName);
212 assertEquals(1, fsFamilies.size());
213 table.close();
214
215
216 admin.disableTable(tableName);
217 admin.restoreSnapshot(snapshotName2);
218 admin.enableTable(tableName);
219 htd = admin.getTableDescriptor(tableName);
220 assertEquals(2, htd.getFamilies().size());
221 assertEquals(2, table.getTableDescriptor().getFamilies().size());
222 assertEquals(500, TEST_UTIL.countRows(table, TEST_FAMILY2));
223 assertEquals(snapshot2Rows, TEST_UTIL.countRows(table));
224 fsFamilies = getFamiliesFromFS(tableName);
225 assertEquals(2, fsFamilies.size());
226 table.close();
227 }
228
229 @Test
230 public void testRestoreSnapshotOfCloned() throws IOException, InterruptedException {
231 byte[] clonedTableName = Bytes.toBytes("clonedtb-" + System.currentTimeMillis());
232 admin.cloneSnapshot(snapshotName0, clonedTableName);
233 verifyRowCount(clonedTableName, snapshot0Rows);
234 admin.disableTable(clonedTableName);
235 admin.snapshot(snapshotName2, clonedTableName);
236 admin.deleteTable(clonedTableName);
237 waitCleanerRun();
238
239 admin.cloneSnapshot(snapshotName2, clonedTableName);
240 verifyRowCount(clonedTableName, snapshot0Rows);
241 admin.disableTable(clonedTableName);
242 admin.deleteTable(clonedTableName);
243 }
244
245
246
247
248 private void createTable(final byte[] tableName, final byte[]... families) throws IOException {
249 HTableDescriptor htd = new HTableDescriptor(tableName);
250 for (byte[] family: families) {
251 HColumnDescriptor hcd = new HColumnDescriptor(family);
252 htd.addFamily(hcd);
253 }
254 byte[][] splitKeys = new byte[16][];
255 byte[] hex = Bytes.toBytes("0123456789abcdef");
256 for (int i = 0; i < 16; ++i) {
257 splitKeys[i] = new byte[] { hex[i] };
258 }
259 admin.createTable(htd, splitKeys);
260 }
261
262 public void loadData(final HTable table, int rows, byte[]... families) throws IOException {
263 byte[] qualifier = Bytes.toBytes("q");
264 table.setAutoFlush(false);
265 while (rows-- > 0) {
266 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
267 byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
268 Put put = new Put(key);
269 put.setWriteToWAL(false);
270 for (byte[] family: families) {
271 put.add(family, qualifier, value);
272 }
273 table.put(put);
274 }
275 table.flushCommits();
276 }
277
278 private void waitCleanerRun() throws InterruptedException {
279 TEST_UTIL.getMiniHBaseCluster().getMaster().getHFileCleaner().choreForTesting();
280 }
281
282 private Set<String> getFamiliesFromFS(final byte[] tableName) throws IOException {
283 MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
284 Set<String> families = new HashSet<String>();
285 Path tableDir = HTableDescriptor.getTableDir(mfs.getRootDir(), tableName);
286 for (Path regionDir: FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
287 for (Path familyDir: FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
288 families.add(familyDir.getName());
289 }
290 }
291 return families;
292 }
293
294 private void verifyRowCount(final byte[] tableName, long expectedRows) throws IOException {
295 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
296 assertEquals(expectedRows, TEST_UTIL.countRows(table));
297 table.close();
298 }
299 }