1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.util.List;
28
29 import org.apache.commons.lang.math.RandomUtils;
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.fs.FileSystem;
33 import org.apache.hadoop.fs.Path;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HTableDescriptor;
38 import org.apache.hadoop.hbase.testclassification.LargeTests;
39 import org.apache.hadoop.hbase.MiniHBaseCluster;
40 import org.apache.hadoop.hbase.ServerName;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.UnknownRegionException;
43 import org.apache.hadoop.hbase.MetaTableAccessor;
44 import org.apache.hadoop.hbase.client.Admin;
45 import org.apache.hadoop.hbase.client.Put;
46 import org.apache.hadoop.hbase.client.Result;
47 import org.apache.hadoop.hbase.client.ResultScanner;
48 import org.apache.hadoop.hbase.client.Scan;
49 import org.apache.hadoop.hbase.client.Table;
50 import org.apache.hadoop.hbase.exceptions.MergeRegionException;
51 import org.apache.hadoop.hbase.master.AssignmentManager;
52 import org.apache.hadoop.hbase.master.HMaster;
53 import org.apache.hadoop.hbase.master.RegionState.State;
54 import org.apache.hadoop.hbase.master.RegionStates;
55 import org.apache.hadoop.hbase.util.Bytes;
56 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
57 import org.apache.hadoop.hbase.util.FSUtils;
58 import org.apache.hadoop.hbase.util.Pair;
59 import org.apache.hadoop.hbase.util.PairOfSameType;
60 import org.apache.hadoop.util.StringUtils;
61 import org.junit.AfterClass;
62 import org.junit.BeforeClass;
63 import org.junit.Test;
64 import org.junit.experimental.categories.Category;
65
66 import com.google.common.base.Joiner;
67
68
69
70
71
72
73
74 @Category(LargeTests.class)
75 public class TestRegionMergeTransactionOnCluster {
76 private static final Log LOG = LogFactory
77 .getLog(TestRegionMergeTransactionOnCluster.class);
78 private static final int NB_SERVERS = 3;
79
80 private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
81 private static final byte[] QUALIFIER = Bytes.toBytes("q");
82
83 private static byte[] ROW = Bytes.toBytes("testRow");
84 private static final int INITIAL_REGION_NUM = 10;
85 private static final int ROWSIZE = 200;
86 private static byte[][] ROWS = makeN(ROW, ROWSIZE);
87
88 private static int waitTime = 60 * 1000;
89
90 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
91
92 private static HMaster master;
93 private static Admin admin;
94
95 static void setupOnce() throws Exception {
96
97 TEST_UTIL.startMiniCluster(NB_SERVERS);
98 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
99 master = cluster.getMaster();
100 master.balanceSwitch(false);
101 admin = TEST_UTIL.getHBaseAdmin();
102 }
103
104 @BeforeClass
105 public static void beforeAllTests() throws Exception {
106
107 TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
108 setupOnce();
109 }
110
111 @AfterClass
112 public static void afterAllTests() throws Exception {
113 TEST_UTIL.shutdownMiniCluster();
114 }
115
116 @Test
117 public void testWholesomeMerge() throws Exception {
118 LOG.info("Starting testWholesomeMerge");
119 final TableName tableName =
120 TableName.valueOf("testWholesomeMerge");
121
122
123 Table table = createTableAndLoadData(master, tableName);
124
125 mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1,
126 INITIAL_REGION_NUM - 1);
127
128
129 PairOfSameType<HRegionInfo> mergedRegions =
130 mergeRegionsAndVerifyRegionNum(master, tableName, 1, 2,
131 INITIAL_REGION_NUM - 2);
132
133 verifyRowCount(table, ROWSIZE);
134
135
136 HRegionInfo hri = RandomUtils.nextBoolean() ?
137 mergedRegions.getFirst() : mergedRegions.getSecond();
138 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
139 AssignmentManager am = cluster.getMaster().getAssignmentManager();
140 RegionStates regionStates = am.getRegionStates();
141 long start = EnvironmentEdgeManager.currentTime();
142 while (!regionStates.isRegionInState(hri, State.MERGED)) {
143 assertFalse("Timed out in waiting one merged region to be in state MERGED",
144 EnvironmentEdgeManager.currentTime() - start > 60000);
145 Thread.sleep(500);
146 }
147
148
149 am.assign(hri, true, true);
150 assertFalse("Merged region can't be assigned",
151 regionStates.isRegionInTransition(hri));
152 assertTrue(regionStates.isRegionInState(hri, State.MERGED));
153
154
155 am.unassign(hri, true, null);
156 assertFalse("Merged region can't be unassigned",
157 regionStates.isRegionInTransition(hri));
158 assertTrue(regionStates.isRegionInState(hri, State.MERGED));
159
160 table.close();
161 }
162
163 @Test
164 public void testCleanMergeReference() throws Exception {
165 LOG.info("Starting testCleanMergeReference");
166 admin.enableCatalogJanitor(false);
167 try {
168 final TableName tableName =
169 TableName.valueOf("testCleanMergeReference");
170
171 Table table = createTableAndLoadData(master, tableName);
172
173 mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1,
174 INITIAL_REGION_NUM - 1);
175 verifyRowCount(table, ROWSIZE);
176 table.close();
177
178 List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor
179 .getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(), tableName);
180 HRegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
181 HTableDescriptor tableDescritor = master.getTableDescriptors().get(
182 tableName);
183 Result mergedRegionResult = MetaTableAccessor.getRegionResult(
184 master.getConnection(), mergedRegionInfo.getRegionName());
185
186
187 assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
188 HConstants.MERGEA_QUALIFIER) != null);
189 assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
190 HConstants.MERGEB_QUALIFIER) != null);
191
192
193 HRegionInfo regionA = HRegionInfo.getHRegionInfo(mergedRegionResult,
194 HConstants.MERGEA_QUALIFIER);
195 HRegionInfo regionB = HRegionInfo.getHRegionInfo(mergedRegionResult,
196 HConstants.MERGEB_QUALIFIER);
197 FileSystem fs = master.getMasterFileSystem().getFileSystem();
198 Path rootDir = master.getMasterFileSystem().getRootDir();
199
200 Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
201 Path regionAdir = new Path(tabledir, regionA.getEncodedName());
202 Path regionBdir = new Path(tabledir, regionB.getEncodedName());
203 assertTrue(fs.exists(regionAdir));
204 assertTrue(fs.exists(regionBdir));
205
206 admin.compactRegion(mergedRegionInfo.getRegionName());
207
208 long timeout = System.currentTimeMillis() + waitTime;
209 HRegionFileSystem hrfs = new HRegionFileSystem(
210 TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
211 while (System.currentTimeMillis() < timeout) {
212 if (!hrfs.hasReferences(tableDescritor)) {
213 break;
214 }
215 Thread.sleep(50);
216 }
217 assertFalse(hrfs.hasReferences(tableDescritor));
218
219
220
221 int cleaned = admin.runCatalogScan();
222 assertTrue(cleaned > 0);
223 assertFalse(fs.exists(regionAdir));
224 assertFalse(fs.exists(regionBdir));
225
226 mergedRegionResult = MetaTableAccessor.getRegionResult(
227 master.getConnection(), mergedRegionInfo.getRegionName());
228 assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
229 HConstants.MERGEA_QUALIFIER) != null);
230 assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
231 HConstants.MERGEB_QUALIFIER) != null);
232
233 } finally {
234 admin.enableCatalogJanitor(true);
235 }
236 }
237
238
239
240
241
242
243
244 @Test
245 public void testMerge() throws Exception {
246 LOG.info("Starting testMerge");
247 final TableName tableName = TableName.valueOf("testMerge");
248
249 try {
250
251 Table table = createTableAndLoadData(master, tableName);
252 RegionStates regionStates = master.getAssignmentManager().getRegionStates();
253 List<HRegionInfo> regions = regionStates.getRegionsOfTable(tableName);
254
255 HRegionInfo a = regions.get(0);
256 HRegionInfo b = regions.get(1);
257 regionStates.regionOffline(a);
258 try {
259
260 admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
261 fail("Offline regions should not be able to merge");
262 } catch (IOException ie) {
263 System.out.println(ie);
264 assertTrue("Exception should mention regions not online",
265 StringUtils.stringifyException(ie).contains("regions not online")
266 && ie instanceof MergeRegionException);
267 }
268 try {
269
270 admin.mergeRegions(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true);
271 fail("A region should not be able to merge with itself, even forcifully");
272 } catch (IOException ie) {
273 assertTrue("Exception should mention regions not online",
274 StringUtils.stringifyException(ie).contains("region to itself")
275 && ie instanceof MergeRegionException);
276 }
277 try {
278
279 admin.mergeRegions(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true);
280 fail("Unknown region could not be merged");
281 } catch (IOException ie) {
282 assertTrue("UnknownRegionException should be thrown",
283 ie instanceof UnknownRegionException);
284 }
285 table.close();
286 } finally {
287 TEST_UTIL.deleteTable(tableName);
288 }
289 }
290
291 private PairOfSameType<HRegionInfo> mergeRegionsAndVerifyRegionNum(
292 HMaster master, TableName tablename,
293 int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
294 PairOfSameType<HRegionInfo> mergedRegions =
295 requestMergeRegion(master, tablename, regionAnum, regionBnum);
296 waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
297 return mergedRegions;
298 }
299
300 private PairOfSameType<HRegionInfo> requestMergeRegion(
301 HMaster master, TableName tablename,
302 int regionAnum, int regionBnum) throws Exception {
303 List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor
304 .getTableRegionsAndLocations(master.getZooKeeper(),
305 master.getConnection(), tablename);
306 HRegionInfo regionA = tableRegions.get(regionAnum).getFirst();
307 HRegionInfo regionB = tableRegions.get(regionBnum).getFirst();
308 TEST_UTIL.getHBaseAdmin().mergeRegions(
309 regionA.getEncodedNameAsBytes(),
310 regionB.getEncodedNameAsBytes(), false);
311 return new PairOfSameType<HRegionInfo>(regionA, regionB);
312 }
313
314 private void waitAndVerifyRegionNum(HMaster master, TableName tablename,
315 int expectedRegionNum) throws Exception {
316 List<Pair<HRegionInfo, ServerName>> tableRegionsInMeta;
317 List<HRegionInfo> tableRegionsInMaster;
318 long timeout = System.currentTimeMillis() + waitTime;
319 while (System.currentTimeMillis() < timeout) {
320 tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
321 master.getConnection(), tablename);
322 tableRegionsInMaster = master.getAssignmentManager().getRegionStates()
323 .getRegionsOfTable(tablename);
324 if (tableRegionsInMeta.size() == expectedRegionNum
325 && tableRegionsInMaster.size() == expectedRegionNum) {
326 break;
327 }
328 Thread.sleep(250);
329 }
330
331 tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
332 master.getConnection(), tablename);
333 LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
334 assertEquals(expectedRegionNum, tableRegionsInMeta.size());
335 }
336
337 private Table createTableAndLoadData(HMaster master, TableName tablename)
338 throws Exception {
339 return createTableAndLoadData(master, tablename, INITIAL_REGION_NUM);
340 }
341
342 private Table createTableAndLoadData(HMaster master, TableName tablename,
343 int numRegions) throws Exception {
344 assertTrue("ROWSIZE must > numregions:" + numRegions, ROWSIZE > numRegions);
345 byte[][] splitRows = new byte[numRegions - 1][];
346 for (int i = 0; i < splitRows.length; i++) {
347 splitRows[i] = ROWS[(i + 1) * ROWSIZE / numRegions];
348 }
349
350 Table table = TEST_UTIL.createTable(tablename, FAMILYNAME, splitRows);
351 loadData(table);
352 verifyRowCount(table, ROWSIZE);
353
354
355 long timeout = System.currentTimeMillis() + waitTime;
356 List<Pair<HRegionInfo, ServerName>> tableRegions;
357 while (System.currentTimeMillis() < timeout) {
358 tableRegions = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
359 master.getConnection(), tablename);
360 if (tableRegions.size() == numRegions)
361 break;
362 Thread.sleep(250);
363 }
364
365 tableRegions = MetaTableAccessor.getTableRegionsAndLocations(
366 master.getZooKeeper(),
367 master.getConnection(), tablename);
368 LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
369 assertEquals(numRegions, tableRegions.size());
370 return table;
371 }
372
373 private static byte[][] makeN(byte[] base, int n) {
374 byte[][] ret = new byte[n][];
375 for (int i = 0; i < n; i++) {
376 ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%04d", i)));
377 }
378 return ret;
379 }
380
381 private void loadData(Table table) throws IOException {
382 for (int i = 0; i < ROWSIZE; i++) {
383 Put put = new Put(ROWS[i]);
384 put.add(FAMILYNAME, QUALIFIER, Bytes.toBytes(i));
385 table.put(put);
386 }
387 }
388
389 private void verifyRowCount(Table table, int expectedRegionNum)
390 throws IOException {
391 ResultScanner scanner = table.getScanner(new Scan());
392 int rowCount = 0;
393 while (scanner.next() != null) {
394 rowCount++;
395 }
396 assertEquals(expectedRegionNum, rowCount);
397 scanner.close();
398 }
399 }