1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import static org.junit.Assert.*;
23
24 import java.io.File;
25 import java.io.IOException;
26 import java.util.ArrayList;
27 import java.util.Arrays;
28 import java.util.List;
29 import java.util.Set;
30
31 import junit.framework.Assert;
32 import junit.framework.AssertionFailedError;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.FileUtil;
39 import org.apache.hadoop.fs.FsShell;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.*;
42 import org.apache.hadoop.hbase.catalog.CatalogTracker;
43 import org.apache.hadoop.hbase.catalog.MetaMigrationRemovingHTD;
44 import org.apache.hadoop.hbase.catalog.MetaReader;
45 import org.apache.hadoop.hbase.migration.HRegionInfo090x;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.Writables;
48 import org.junit.AfterClass;
49 import org.junit.BeforeClass;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52
53
54
55
56
57 @Category(MediumTests.class)
58 public class TestMetaMigrationRemovingHTD {
59 static final Log LOG = LogFactory.getLog(TestMetaMigrationRemovingHTD.class);
60 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
61 private final static String TESTTABLE = "TestTable";
62 private final static int ROWCOUNT = 100;
63
64 @BeforeClass
65 public static void setUpBeforeClass() throws Exception {
66
67
68
69 TEST_UTIL.startMiniZKCluster();
70 TEST_UTIL.startMiniDFSCluster(1);
71 Path testdir = TEST_UTIL.getDataTestDir("TestMetaMigrationRemovingHTD");
72
73 File untar = untar(new File(testdir.toString()));
74
75 Configuration conf = TEST_UTIL.getConfiguration();
76 FsShell shell = new FsShell(conf);
77 FileSystem fs = FileSystem.get(conf);
78
79 Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
80 if (!fs.isDirectory(hbaseRootDir.getParent())) {
81
82 fs.mkdirs(hbaseRootDir.getParent());
83 }
84 doFsCommand(shell,
85 new String [] {"-put", untar.toURI().toString(), hbaseRootDir.toString()});
86
87 doFsCommand(shell, new String [] {"-lsr", "/"});
88 TEST_UTIL.startMiniHBaseCluster(1, 1);
89
90
91
92 HTable t = new HTable(TEST_UTIL.getConfiguration(), TESTTABLE);
93 ResultScanner scanner = t.getScanner(new Scan());
94 int count = 0;
95 while (scanner.next() != null) {
96 count++;
97 }
98
99
100 Assert.assertEquals(ROWCOUNT, count);
101 scanner.close();
102 t.close();
103 }
104
105 private static File untar(final File testdir) throws IOException {
106
107 final String datafile = "hbase-4388-root.dir";
108 String srcTarFile =
109 System.getProperty("project.build.testSourceDirectory", "src/test") +
110 File.separator + "data" + File.separator + datafile + ".tgz";
111 File homedir = new File(testdir.toString());
112 File tgtUntarDir = new File(homedir, datafile);
113 if (tgtUntarDir.exists()) {
114 if (!FileUtil.fullyDelete(tgtUntarDir)) {
115 throw new IOException("Failed delete of " + tgtUntarDir.toString());
116 }
117 }
118 LOG.info("Untarring " + srcTarFile + " into " + homedir.toString());
119 FileUtil.unTar(new File(srcTarFile), homedir);
120 Assert.assertTrue(tgtUntarDir.exists());
121 return tgtUntarDir;
122 }
123
124 private static void doFsCommand(final FsShell shell, final String [] args)
125 throws Exception {
126
127 int errcode = shell.run(args);
128 if (errcode != 0) throw new IOException("Failed put; errcode=" + errcode);
129 }
130
131
132
133
134 @AfterClass
135 public static void tearDownAfterClass() throws Exception {
136 TEST_UTIL.shutdownMiniCluster();
137 }
138
139 @Test
140 public void testMetaUpdatedFlagInROOT() throws Exception {
141 boolean metaUpdated = MetaMigrationRemovingHTD.
142 isMetaHRIUpdated(TEST_UTIL.getMiniHBaseCluster().getMaster());
143 assertEquals(true, metaUpdated);
144 }
145
146 @Test
147 public void testMetaMigration() throws Exception {
148 LOG.info("Starting testMetaWithLegacyHRI");
149 final byte [] FAMILY = Bytes.toBytes("family");
150 HTableDescriptor htd = new HTableDescriptor("testMetaMigration");
151 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
152 htd.addFamily(hcd);
153 Configuration conf = TEST_UTIL.getConfiguration();
154 createMultiRegionsWithLegacyHRI(conf, htd, FAMILY,
155 new byte[][]{
156 HConstants.EMPTY_START_ROW,
157 Bytes.toBytes("region_a"),
158 Bytes.toBytes("region_b")});
159 CatalogTracker ct =
160 TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker();
161
162 undoVersionInMeta();
163 MetaReader.fullScanMetaAndPrint(ct);
164 LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI");
165
166 Set<HTableDescriptor> htds =
167 MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo(
168 TEST_UTIL.getHBaseCluster().getMaster());
169 MetaReader.fullScanMetaAndPrint(ct);
170
171 assertEquals(1, htds.size());
172 assertTrue(htds.contains(htd));
173
174 boolean metaUpdated =
175 MetaMigrationRemovingHTD.isMetaHRIUpdated(
176 TEST_UTIL.getMiniHBaseCluster().getMaster());
177 assertEquals(true, metaUpdated);
178 }
179
180
181
182
183
184
185
186
187
188
189
190
191
192 @Test
193 public void testMasterCrashDuringMetaMigration() throws Exception {
194 final byte[] FAMILY = Bytes.toBytes("family");
195 HTableDescriptor htd = new HTableDescriptor("testMasterCrashDuringMetaMigration");
196 HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
197 htd.addFamily(hcd);
198 Configuration conf = TEST_UTIL.getConfiguration();
199
200 createMultiRegionsWithNewHRI(conf, htd, FAMILY, 10);
201
202 createMultiRegionsWithLegacyHRI(conf, htd, FAMILY, 10);
203 CatalogTracker ct =
204 TEST_UTIL.getMiniHBaseCluster().getMaster().getCatalogTracker();
205
206 undoVersionInMeta();
207 MetaMigrationRemovingHTD.updateRootWithMetaMigrationStatus(ct);
208
209 LOG.info("Meta Print completed.testUpdatesOnMetaWithLegacyHRI");
210
211 Set<HTableDescriptor> htds =
212 MetaMigrationRemovingHTD.updateMetaWithNewRegionInfo(
213 TEST_UTIL.getHBaseCluster().getMaster());
214 assertEquals(1, htds.size());
215 assertTrue(htds.contains(htd));
216
217 boolean metaUpdated = MetaMigrationRemovingHTD.
218 isMetaHRIUpdated(TEST_UTIL.getMiniHBaseCluster().getMaster());
219 assertEquals(true, metaUpdated);
220 LOG.info("END testMetaWithLegacyHRI");
221 }
222
223 private void undoVersionInMeta() throws IOException {
224 Delete d = new Delete(HRegionInfo.ROOT_REGIONINFO.getRegionName());
225
226 d.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.META_VERSION_QUALIFIER);
227 HTable rootTable =
228 new HTable(TEST_UTIL.getConfiguration(), HConstants.ROOT_TABLE_NAME);
229 try {
230 rootTable.delete(d);
231 } finally {
232 rootTable.close();
233 }
234 }
235
236 public static void assertEquals(int expected, int actual) {
237 if (expected != actual) {
238 throw new AssertionFailedError("expected:<" +
239 expected + "> but was:<" +
240 actual + ">");
241 }
242 }
243
244 public static void assertEquals(boolean expected, boolean actual) {
245 if (expected != actual) {
246 throw new AssertionFailedError("expected:<" +
247 expected + "> but was:<" +
248 actual + ">");
249 }
250 }
251
252
253
254
255
256
257
258
259
260
261
262
263 public int createMultiRegionsWithLegacyHRI(final Configuration c,
264 final HTableDescriptor htd, final byte [] family, int numRegions)
265 throws IOException {
266 if (numRegions < 3) throw new IOException("Must create at least 3 regions");
267 byte [] startKey = Bytes.toBytes("aaaaa");
268 byte [] endKey = Bytes.toBytes("zzzzz");
269 byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
270 byte [][] regionStartKeys = new byte[splitKeys.length+1][];
271 for (int i=0;i<splitKeys.length;i++) {
272 regionStartKeys[i+1] = splitKeys[i];
273 }
274 regionStartKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
275 return createMultiRegionsWithLegacyHRI(c, htd, family, regionStartKeys);
276 }
277
278
279
280
281
282
283
284
285
286
287
288 public int createMultiRegionsWithLegacyHRI(final Configuration c,
289 final HTableDescriptor htd, final byte[] columnFamily, byte [][] startKeys)
290 throws IOException {
291 Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
292 HTable meta = new HTable(c, HConstants.META_TABLE_NAME);
293 if(!htd.hasFamily(columnFamily)) {
294 HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
295 htd.addFamily(hcd);
296 }
297 List<HRegionInfo090x> newRegions
298 = new ArrayList<HRegionInfo090x>(startKeys.length);
299 int count = 0;
300 for (int i = 0; i < startKeys.length; i++) {
301 int j = (i + 1) % startKeys.length;
302 HRegionInfo090x hri = new HRegionInfo090x(htd,
303 startKeys[i], startKeys[j]);
304 Put put = new Put(hri.getRegionName());
305 put.setWriteToWAL(false);
306 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
307 Writables.getBytes(hri));
308 meta.put(put);
309 LOG.info("createMultiRegions: PUT inserted " + hri.toString());
310
311 newRegions.add(hri);
312 count++;
313 }
314 meta.close();
315 return count;
316 }
317
318 int createMultiRegionsWithNewHRI(final Configuration c,
319 final HTableDescriptor htd, final byte [] family, int numRegions)
320 throws IOException {
321 if (numRegions < 3) throw new IOException("Must create at least 3 regions");
322 byte [] startKey = Bytes.toBytes("aaaaa");
323 byte [] endKey = Bytes.toBytes("zzzzz");
324 byte [][] splitKeys = Bytes.split(startKey, endKey, numRegions - 3);
325 byte [][] regionStartKeys = new byte[splitKeys.length+1][];
326 for (int i=0;i<splitKeys.length;i++) {
327 regionStartKeys[i+1] = splitKeys[i];
328 }
329 regionStartKeys[0] = HConstants.EMPTY_BYTE_ARRAY;
330 return createMultiRegionsWithNewHRI(c, htd, family, regionStartKeys);
331 }
332
333 int createMultiRegionsWithNewHRI(final Configuration c, final HTableDescriptor htd,
334 final byte[] columnFamily, byte [][] startKeys)
335 throws IOException {
336 Arrays.sort(startKeys, Bytes.BYTES_COMPARATOR);
337 HTable meta = new HTable(c, HConstants.META_TABLE_NAME);
338 if(!htd.hasFamily(columnFamily)) {
339 HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
340 htd.addFamily(hcd);
341 }
342 List<HRegionInfo> newRegions
343 = new ArrayList<HRegionInfo>(startKeys.length);
344 int count = 0;
345 for (int i = 0; i < startKeys.length; i++) {
346 int j = (i + 1) % startKeys.length;
347 HRegionInfo hri = new HRegionInfo(htd.getName(),
348 startKeys[i], startKeys[j]);
349 Put put = new Put(hri.getRegionName());
350 put.setWriteToWAL(false);
351 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
352 Writables.getBytes(hri));
353 meta.put(put);
354 LOG.info("createMultiRegions: PUT inserted " + hri.toString());
355
356 newRegions.add(hri);
357 count++;
358 }
359 meta.close();
360 return count;
361 }
362
363 @org.junit.Rule
364 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
365 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
366 }
367