1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.mockito.Mockito.*;
25
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.List;
29
30 import org.apache.hadoop.fs.FileSystem;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.TableName;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HTableDescriptor;
38 import org.apache.hadoop.hbase.KeyValue;
39 import org.apache.hadoop.hbase.Server;
40 import org.apache.hadoop.hbase.SmallTests;
41 import org.apache.hadoop.hbase.client.Durability;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.Scan;
44 import org.apache.hadoop.hbase.regionserver.wal.HLog;
45 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.zookeeper.KeeperException;
48 import org.junit.After;
49 import org.junit.Before;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52 import org.mockito.Mockito;
53
54 import com.google.common.collect.ImmutableList;
55
56
57
58
59
60 @Category(SmallTests.class)
61 public class TestRegionMergeTransaction {
62 private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63 private final Path testdir = TEST_UTIL.getDataTestDir(this.getClass()
64 .getName());
65 private HRegion region_a;
66 private HRegion region_b;
67 private HRegion region_c;
68 private HLog wal;
69 private FileSystem fs;
70
71 private static final byte[] STARTROW_A = new byte[] { 'a', 'a', 'a' };
72 private static final byte[] STARTROW_B = new byte[] { 'g', 'g', 'g' };
73 private static final byte[] STARTROW_C = new byte[] { 'w', 'w', 'w' };
74 private static final byte[] ENDROW = new byte[] { '{', '{', '{' };
75 private static final byte[] CF = HConstants.CATALOG_FAMILY;
76
77 @Before
78 public void setup() throws IOException {
79 this.fs = FileSystem.get(TEST_UTIL.getConfiguration());
80 this.fs.delete(this.testdir, true);
81 this.wal = HLogFactory.createHLog(fs, this.testdir, "logs",
82 TEST_UTIL.getConfiguration());
83 this.region_a = createRegion(this.testdir, this.wal, STARTROW_A, STARTROW_B);
84 this.region_b = createRegion(this.testdir, this.wal, STARTROW_B, STARTROW_C);
85 this.region_c = createRegion(this.testdir, this.wal, STARTROW_C, ENDROW);
86 assert region_a != null && region_b != null && region_c != null;
87 TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true);
88 }
89
90 @After
91 public void teardown() throws IOException {
92 for (HRegion region : new HRegion[] { region_a, region_b, region_c }) {
93 if (region != null && !region.isClosed()) region.close();
94 if (this.fs.exists(region.getRegionFileSystem().getRegionDir())
95 && !this.fs.delete(region.getRegionFileSystem().getRegionDir(), true)) {
96 throw new IOException("Failed deleting of "
97 + region.getRegionFileSystem().getRegionDir());
98 }
99 }
100 if (this.wal != null)
101 this.wal.closeAndDelete();
102 this.fs.delete(this.testdir, true);
103 }
104
105
106
107
108
109
110 @Test
111 public void testPrepare() throws IOException {
112 prepareOnGoodRegions();
113 }
114
115 private RegionMergeTransaction prepareOnGoodRegions() throws IOException {
116 RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_b,
117 false);
118 RegionMergeTransaction spyMT = Mockito.spy(mt);
119 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
120 region_a.getRegionName());
121 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
122 region_b.getRegionName());
123 assertTrue(spyMT.prepare(null));
124 return spyMT;
125 }
126
127
128
129
130 @Test
131 public void testPrepareWithSameRegion() throws IOException {
132 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
133 this.region_a, true);
134 assertFalse("should not merge the same region even if it is forcible ",
135 mt.prepare(null));
136 }
137
138
139
140
141 @Test
142 public void testPrepareWithRegionsNotAdjacent() throws IOException {
143 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
144 this.region_c, false);
145 assertFalse("should not merge two regions if they are adjacent except it is forcible",
146 mt.prepare(null));
147 }
148
149
150
151
152 @Test
153 public void testPrepareWithRegionsNotAdjacentUnderCompulsory()
154 throws IOException {
155 RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_c,
156 true);
157 RegionMergeTransaction spyMT = Mockito.spy(mt);
158 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
159 region_a.getRegionName());
160 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
161 region_c.getRegionName());
162 assertTrue("Since focible is true, should merge two regions even if they are not adjacent",
163 spyMT.prepare(null));
164 }
165
166
167
168
169 @Test
170 public void testPrepareWithRegionsWithReference() throws IOException {
171 HStore storeMock = Mockito.mock(HStore.class);
172 when(storeMock.hasReferences()).thenReturn(true);
173 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
174 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
175 this.region_a.stores.put(Bytes.toBytes(""), storeMock);
176 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
177 this.region_b, false);
178 assertFalse(
179 "a region should not be mergeable if it has instances of store file references",
180 mt.prepare(null));
181 }
182
183 @Test
184 public void testPrepareWithClosedRegion() throws IOException {
185 this.region_a.close();
186 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
187 this.region_b, false);
188 assertFalse(mt.prepare(null));
189 }
190
191
192
193
194
195 @Test
196 public void testPrepareWithRegionsWithMergeReference() throws IOException {
197 RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_b,
198 false);
199 RegionMergeTransaction spyMT = Mockito.spy(mt);
200 doReturn(true).when(spyMT).hasMergeQualifierInMeta(null,
201 region_a.getRegionName());
202 doReturn(true).when(spyMT).hasMergeQualifierInMeta(null,
203 region_b.getRegionName());
204 assertFalse(spyMT.prepare(null));
205 }
206
207 @Test
208 public void testWholesomeMerge() throws IOException {
209 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
210 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
211 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
212 assertEquals(rowCountOfRegionA, countRows(this.region_a));
213 assertEquals(rowCountOfRegionB, countRows(this.region_b));
214
215
216 RegionMergeTransaction mt = prepareOnGoodRegions();
217
218
219 Server mockServer = Mockito.mock(Server.class);
220 when(mockServer.getConfiguration())
221 .thenReturn(TEST_UTIL.getConfiguration());
222 HRegion mergedRegion = mt.execute(mockServer, null);
223
224 assertTrue(this.fs.exists(mt.getMergesDir()));
225
226 assertTrue(region_a.isClosed());
227 assertTrue(region_b.isClosed());
228
229
230
231 assertEquals(0, this.fs.listStatus(mt.getMergesDir()).length);
232
233 assertTrue(Bytes.equals(this.region_a.getStartKey(),
234 mergedRegion.getStartKey()));
235 assertTrue(Bytes.equals(this.region_b.getEndKey(),
236 mergedRegion.getEndKey()));
237
238 try {
239 int mergedRegionRowCount = countRows(mergedRegion);
240 assertEquals((rowCountOfRegionA + rowCountOfRegionB),
241 mergedRegionRowCount);
242 } finally {
243 HRegion.closeHRegion(mergedRegion);
244 }
245
246 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
247 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
248 }
249
250 @Test
251 public void testRollback() throws IOException {
252 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
253 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
254 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
255 assertEquals(rowCountOfRegionA, countRows(this.region_a));
256 assertEquals(rowCountOfRegionB, countRows(this.region_b));
257
258
259 RegionMergeTransaction mt = prepareOnGoodRegions();
260
261 when(mt.createMergedRegionFromMerges(region_a, region_b,
262 mt.getMergedRegionInfo())).thenThrow(
263 new MockedFailedMergedRegionCreation());
264
265
266 boolean expectedException = false;
267 Server mockServer = Mockito.mock(Server.class);
268 when(mockServer.getConfiguration())
269 .thenReturn(TEST_UTIL.getConfiguration());
270 try {
271 mt.execute(mockServer, null);
272 } catch (MockedFailedMergedRegionCreation e) {
273 expectedException = true;
274 }
275 assertTrue(expectedException);
276
277 assertTrue(mt.rollback(null, null));
278
279
280 int rowCountOfRegionA2 = countRows(this.region_a);
281 assertEquals(rowCountOfRegionA, rowCountOfRegionA2);
282 int rowCountOfRegionB2 = countRows(this.region_b);
283 assertEquals(rowCountOfRegionB, rowCountOfRegionB2);
284
285
286 assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir,
287 mt.getMergedRegionInfo())));
288
289 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
290 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
291
292
293 assertTrue(mt.prepare(null));
294 HRegion mergedRegion = mt.execute(mockServer, null);
295
296
297 try {
298 int mergedRegionRowCount = countRows(mergedRegion);
299 assertEquals((rowCountOfRegionA + rowCountOfRegionB),
300 mergedRegionRowCount);
301 } finally {
302 HRegion.closeHRegion(mergedRegion);
303 }
304
305 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
306 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
307 }
308
309 @Test
310 public void testFailAfterPONR() throws IOException, KeeperException {
311 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
312 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
313 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
314 assertEquals(rowCountOfRegionA, countRows(this.region_a));
315 assertEquals(rowCountOfRegionB, countRows(this.region_b));
316
317
318 RegionMergeTransaction mt = prepareOnGoodRegions();
319 Mockito.doThrow(new MockedFailedMergedRegionOpen())
320 .when(mt)
321 .openMergedRegion((Server) Mockito.anyObject(),
322 (RegionServerServices) Mockito.anyObject(),
323 (HRegion) Mockito.anyObject());
324
325
326 boolean expectedException = false;
327 Server mockServer = Mockito.mock(Server.class);
328 when(mockServer.getConfiguration())
329 .thenReturn(TEST_UTIL.getConfiguration());
330 try {
331 mt.execute(mockServer, null);
332 } catch (MockedFailedMergedRegionOpen e) {
333 expectedException = true;
334 }
335 assertTrue(expectedException);
336
337 assertFalse(mt.rollback(null, null));
338
339
340
341 Path tableDir = this.region_a.getRegionFileSystem().getRegionDir()
342 .getParent();
343 Path mergedRegionDir = new Path(tableDir, mt.getMergedRegionInfo()
344 .getEncodedName());
345 assertTrue(TEST_UTIL.getTestFileSystem().exists(mergedRegionDir));
346 }
347
348 @Test
349 public void testMeregedRegionBoundary() {
350 TableName tableName =
351 TableName.valueOf("testMeregedRegionBoundary");
352 byte[] a = Bytes.toBytes("a");
353 byte[] b = Bytes.toBytes("b");
354 byte[] z = Bytes.toBytes("z");
355 HRegionInfo r1 = new HRegionInfo(tableName);
356 HRegionInfo r2 = new HRegionInfo(tableName, a, z);
357 HRegionInfo m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
358 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
359 && Bytes.equals(m.getEndKey(), r1.getEndKey()));
360
361 r1 = new HRegionInfo(tableName, null, a);
362 r2 = new HRegionInfo(tableName, a, z);
363 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
364 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
365 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
366
367 r1 = new HRegionInfo(tableName, null, a);
368 r2 = new HRegionInfo(tableName, z, null);
369 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
370 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
371 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
372
373 r1 = new HRegionInfo(tableName, a, z);
374 r2 = new HRegionInfo(tableName, z, null);
375 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
376 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
377 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
378
379 r1 = new HRegionInfo(tableName, a, b);
380 r2 = new HRegionInfo(tableName, b, z);
381 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
382 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
383 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
384 }
385
386
387
388
389 @SuppressWarnings("serial")
390 private class MockedFailedMergedRegionCreation extends IOException {
391 }
392
393 @SuppressWarnings("serial")
394 private class MockedFailedMergedRegionOpen extends IOException {
395 }
396
397 private HRegion createRegion(final Path testdir, final HLog wal,
398 final byte[] startrow, final byte[] endrow)
399 throws IOException {
400
401 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
402 HColumnDescriptor hcd = new HColumnDescriptor(CF);
403 htd.addFamily(hcd);
404 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startrow, endrow);
405 HRegion a = HRegion.createHRegion(hri, testdir,
406 TEST_UTIL.getConfiguration(), htd);
407 HRegion.closeHRegion(a);
408 return HRegion.openHRegion(testdir, hri, htd, wal,
409 TEST_UTIL.getConfiguration());
410 }
411
412 private int countRows(final HRegion r) throws IOException {
413 int rowcount = 0;
414 InternalScanner scanner = r.getScanner(new Scan());
415 try {
416 List<KeyValue> kvs = new ArrayList<KeyValue>();
417 boolean hasNext = true;
418 while (hasNext) {
419 hasNext = scanner.next(kvs);
420 if (!kvs.isEmpty())
421 rowcount++;
422 }
423 } finally {
424 scanner.close();
425 }
426 return rowcount;
427 }
428
429
430
431
432
433
434
435
436
437
438 private int loadRegion(final HRegion r, final byte[] f, final boolean flush)
439 throws IOException {
440 byte[] k = new byte[3];
441 int rowCount = 0;
442 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
443 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
444 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
445 k[0] = b1;
446 k[1] = b2;
447 k[2] = b3;
448 if (!HRegion.rowIsInRange(r.getRegionInfo(), k)) {
449 continue;
450 }
451 Put put = new Put(k);
452 put.add(f, null, k);
453 if (r.getLog() == null)
454 put.setDurability(Durability.SKIP_WAL);
455 r.put(put);
456 rowCount++;
457 }
458 }
459 if (flush) {
460 r.flushcache();
461 }
462 }
463 return rowCount;
464 }
465
466 }
467