1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.mockito.Mockito.doReturn;
25 import static org.mockito.Mockito.when;
26
27 import java.io.IOException;
28 import java.util.ArrayList;
29 import java.util.List;
30
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.Cell;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.HColumnDescriptor;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.HRegionInfo;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.Server;
40 import org.apache.hadoop.hbase.SmallTests;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.client.Durability;
43 import org.apache.hadoop.hbase.client.Put;
44 import org.apache.hadoop.hbase.client.Scan;
45 import org.apache.hadoop.hbase.regionserver.wal.HLog;
46 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.zookeeper.KeeperException;
49 import org.junit.After;
50 import org.junit.Before;
51 import org.junit.Test;
52 import org.junit.experimental.categories.Category;
53 import org.mockito.Mockito;
54
55 import com.google.common.collect.ImmutableList;
56
57
58
59
60
61 @Category(SmallTests.class)
62 public class TestRegionMergeTransaction {
63 private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
64 private final Path testdir = TEST_UTIL.getDataTestDir(this.getClass()
65 .getName());
66 private HRegion region_a;
67 private HRegion region_b;
68 private HRegion region_c;
69 private HLog wal;
70 private FileSystem fs;
71
72 private static final byte[] STARTROW_A = new byte[] { 'a', 'a', 'a' };
73 private static final byte[] STARTROW_B = new byte[] { 'g', 'g', 'g' };
74 private static final byte[] STARTROW_C = new byte[] { 'w', 'w', 'w' };
75 private static final byte[] ENDROW = new byte[] { '{', '{', '{' };
76 private static final byte[] CF = HConstants.CATALOG_FAMILY;
77
78 @Before
79 public void setup() throws IOException {
80 this.fs = FileSystem.get(TEST_UTIL.getConfiguration());
81 this.fs.delete(this.testdir, true);
82 this.wal = HLogFactory.createHLog(fs, this.testdir, "logs",
83 TEST_UTIL.getConfiguration());
84 this.region_a = createRegion(this.testdir, this.wal, STARTROW_A, STARTROW_B);
85 this.region_b = createRegion(this.testdir, this.wal, STARTROW_B, STARTROW_C);
86 this.region_c = createRegion(this.testdir, this.wal, STARTROW_C, ENDROW);
87 assert region_a != null && region_b != null && region_c != null;
88 TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true);
89 }
90
91 @After
92 public void teardown() throws IOException {
93 for (HRegion region : new HRegion[] { region_a, region_b, region_c }) {
94 if (region != null && !region.isClosed()) region.close();
95 if (this.fs.exists(region.getRegionFileSystem().getRegionDir())
96 && !this.fs.delete(region.getRegionFileSystem().getRegionDir(), true)) {
97 throw new IOException("Failed deleting of "
98 + region.getRegionFileSystem().getRegionDir());
99 }
100 }
101 if (this.wal != null)
102 this.wal.closeAndDelete();
103 this.fs.delete(this.testdir, true);
104 }
105
106
107
108
109
110
111 @Test
112 public void testPrepare() throws IOException {
113 prepareOnGoodRegions();
114 }
115
116 private RegionMergeTransaction prepareOnGoodRegions() throws IOException {
117 RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_b,
118 false);
119 RegionMergeTransaction spyMT = Mockito.spy(mt);
120 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
121 region_a.getRegionName());
122 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
123 region_b.getRegionName());
124 assertTrue(spyMT.prepare(null));
125 return spyMT;
126 }
127
128
129
130
131 @Test
132 public void testPrepareWithSameRegion() throws IOException {
133 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
134 this.region_a, true);
135 assertFalse("should not merge the same region even if it is forcible ",
136 mt.prepare(null));
137 }
138
139
140
141
142 @Test
143 public void testPrepareWithRegionsNotAdjacent() throws IOException {
144 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
145 this.region_c, false);
146 assertFalse("should not merge two regions if they are adjacent except it is forcible",
147 mt.prepare(null));
148 }
149
150
151
152
153 @Test
154 public void testPrepareWithRegionsNotAdjacentUnderCompulsory()
155 throws IOException {
156 RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_c,
157 true);
158 RegionMergeTransaction spyMT = Mockito.spy(mt);
159 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
160 region_a.getRegionName());
161 doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
162 region_c.getRegionName());
163 assertTrue("Since focible is true, should merge two regions even if they are not adjacent",
164 spyMT.prepare(null));
165 }
166
167
168
169
170 @Test
171 public void testPrepareWithRegionsWithReference() throws IOException {
172 HStore storeMock = Mockito.mock(HStore.class);
173 when(storeMock.hasReferences()).thenReturn(true);
174 when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
175 when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
176 this.region_a.stores.put(Bytes.toBytes(""), storeMock);
177 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
178 this.region_b, false);
179 assertFalse(
180 "a region should not be mergeable if it has instances of store file references",
181 mt.prepare(null));
182 }
183
184 @Test
185 public void testPrepareWithClosedRegion() throws IOException {
186 this.region_a.close();
187 RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
188 this.region_b, false);
189 assertFalse(mt.prepare(null));
190 }
191
192
193
194
195
196 @Test
197 public void testPrepareWithRegionsWithMergeReference() throws IOException {
198 RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_b,
199 false);
200 RegionMergeTransaction spyMT = Mockito.spy(mt);
201 doReturn(true).when(spyMT).hasMergeQualifierInMeta(null,
202 region_a.getRegionName());
203 doReturn(true).when(spyMT).hasMergeQualifierInMeta(null,
204 region_b.getRegionName());
205 assertFalse(spyMT.prepare(null));
206 }
207
208 @Test
209 public void testWholesomeMerge() throws IOException {
210 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
211 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
212 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
213 assertEquals(rowCountOfRegionA, countRows(this.region_a));
214 assertEquals(rowCountOfRegionB, countRows(this.region_b));
215
216
217 RegionMergeTransaction mt = prepareOnGoodRegions();
218
219
220 Server mockServer = Mockito.mock(Server.class);
221 when(mockServer.getConfiguration())
222 .thenReturn(TEST_UTIL.getConfiguration());
223 HRegion mergedRegion = mt.execute(mockServer, null);
224
225 assertTrue(this.fs.exists(mt.getMergesDir()));
226
227 assertTrue(region_a.isClosed());
228 assertTrue(region_b.isClosed());
229
230
231
232 assertEquals(0, this.fs.listStatus(mt.getMergesDir()).length);
233
234 assertTrue(Bytes.equals(this.region_a.getStartKey(),
235 mergedRegion.getStartKey()));
236 assertTrue(Bytes.equals(this.region_b.getEndKey(),
237 mergedRegion.getEndKey()));
238
239 try {
240 int mergedRegionRowCount = countRows(mergedRegion);
241 assertEquals((rowCountOfRegionA + rowCountOfRegionB),
242 mergedRegionRowCount);
243 } finally {
244 HRegion.closeHRegion(mergedRegion);
245 }
246
247 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
248 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
249 }
250
251 @Test
252 public void testRollback() throws IOException {
253 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
254 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
255 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
256 assertEquals(rowCountOfRegionA, countRows(this.region_a));
257 assertEquals(rowCountOfRegionB, countRows(this.region_b));
258
259
260 RegionMergeTransaction mt = prepareOnGoodRegions();
261
262 when(mt.createMergedRegionFromMerges(region_a, region_b,
263 mt.getMergedRegionInfo())).thenThrow(
264 new MockedFailedMergedRegionCreation());
265
266
267 boolean expectedException = false;
268 Server mockServer = Mockito.mock(Server.class);
269 when(mockServer.getConfiguration())
270 .thenReturn(TEST_UTIL.getConfiguration());
271 try {
272 mt.execute(mockServer, null);
273 } catch (MockedFailedMergedRegionCreation e) {
274 expectedException = true;
275 }
276 assertTrue(expectedException);
277
278 assertTrue(mt.rollback(null, null));
279
280
281 int rowCountOfRegionA2 = countRows(this.region_a);
282 assertEquals(rowCountOfRegionA, rowCountOfRegionA2);
283 int rowCountOfRegionB2 = countRows(this.region_b);
284 assertEquals(rowCountOfRegionB, rowCountOfRegionB2);
285
286
287 assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir,
288 mt.getMergedRegionInfo())));
289
290 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
291 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
292
293
294 assertTrue(mt.prepare(null));
295 HRegion mergedRegion = mt.execute(mockServer, null);
296
297
298 try {
299 int mergedRegionRowCount = countRows(mergedRegion);
300 assertEquals((rowCountOfRegionA + rowCountOfRegionB),
301 mergedRegionRowCount);
302 } finally {
303 HRegion.closeHRegion(mergedRegion);
304 }
305
306 assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
307 assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
308 }
309
310 @Test
311 public void testFailAfterPONR() throws IOException, KeeperException {
312 final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
313 final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
314 assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
315 assertEquals(rowCountOfRegionA, countRows(this.region_a));
316 assertEquals(rowCountOfRegionB, countRows(this.region_b));
317
318
319 RegionMergeTransaction mt = prepareOnGoodRegions();
320 Mockito.doThrow(new MockedFailedMergedRegionOpen())
321 .when(mt)
322 .openMergedRegion((Server) Mockito.anyObject(),
323 (RegionServerServices) Mockito.anyObject(),
324 (HRegion) Mockito.anyObject());
325
326
327 boolean expectedException = false;
328 Server mockServer = Mockito.mock(Server.class);
329 when(mockServer.getConfiguration())
330 .thenReturn(TEST_UTIL.getConfiguration());
331 try {
332 mt.execute(mockServer, null);
333 } catch (MockedFailedMergedRegionOpen e) {
334 expectedException = true;
335 }
336 assertTrue(expectedException);
337
338 assertFalse(mt.rollback(null, null));
339
340
341
342 Path tableDir = this.region_a.getRegionFileSystem().getRegionDir()
343 .getParent();
344 Path mergedRegionDir = new Path(tableDir, mt.getMergedRegionInfo()
345 .getEncodedName());
346 assertTrue(TEST_UTIL.getTestFileSystem().exists(mergedRegionDir));
347 }
348
349 @Test
350 public void testMeregedRegionBoundary() {
351 TableName tableName =
352 TableName.valueOf("testMeregedRegionBoundary");
353 byte[] a = Bytes.toBytes("a");
354 byte[] b = Bytes.toBytes("b");
355 byte[] z = Bytes.toBytes("z");
356 HRegionInfo r1 = new HRegionInfo(tableName);
357 HRegionInfo r2 = new HRegionInfo(tableName, a, z);
358 HRegionInfo m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
359 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
360 && Bytes.equals(m.getEndKey(), r1.getEndKey()));
361
362 r1 = new HRegionInfo(tableName, null, a);
363 r2 = new HRegionInfo(tableName, a, z);
364 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
365 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
366 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
367
368 r1 = new HRegionInfo(tableName, null, a);
369 r2 = new HRegionInfo(tableName, z, null);
370 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
371 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
372 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
373
374 r1 = new HRegionInfo(tableName, a, z);
375 r2 = new HRegionInfo(tableName, z, null);
376 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
377 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
378 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
379
380 r1 = new HRegionInfo(tableName, a, b);
381 r2 = new HRegionInfo(tableName, b, z);
382 m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
383 assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
384 && Bytes.equals(m.getEndKey(), r2.getEndKey()));
385 }
386
387
388
389
390 @SuppressWarnings("serial")
391 private class MockedFailedMergedRegionCreation extends IOException {
392 }
393
394 @SuppressWarnings("serial")
395 private class MockedFailedMergedRegionOpen extends IOException {
396 }
397
398 private HRegion createRegion(final Path testdir, final HLog wal,
399 final byte[] startrow, final byte[] endrow)
400 throws IOException {
401
402 HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
403 HColumnDescriptor hcd = new HColumnDescriptor(CF);
404 htd.addFamily(hcd);
405 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startrow, endrow);
406 HRegion a = HRegion.createHRegion(hri, testdir,
407 TEST_UTIL.getConfiguration(), htd);
408 HRegion.closeHRegion(a);
409 return HRegion.openHRegion(testdir, hri, htd, wal,
410 TEST_UTIL.getConfiguration());
411 }
412
413 private int countRows(final HRegion r) throws IOException {
414 int rowcount = 0;
415 InternalScanner scanner = r.getScanner(new Scan());
416 try {
417 List<Cell> kvs = new ArrayList<Cell>();
418 boolean hasNext = true;
419 while (hasNext) {
420 hasNext = scanner.next(kvs);
421 if (!kvs.isEmpty())
422 rowcount++;
423 }
424 } finally {
425 scanner.close();
426 }
427 return rowcount;
428 }
429
430
431
432
433
434
435
436
437
438
439 private int loadRegion(final HRegion r, final byte[] f, final boolean flush)
440 throws IOException {
441 byte[] k = new byte[3];
442 int rowCount = 0;
443 for (byte b1 = 'a'; b1 <= 'z'; b1++) {
444 for (byte b2 = 'a'; b2 <= 'z'; b2++) {
445 for (byte b3 = 'a'; b3 <= 'z'; b3++) {
446 k[0] = b1;
447 k[1] = b2;
448 k[2] = b3;
449 if (!HRegion.rowIsInRange(r.getRegionInfo(), k)) {
450 continue;
451 }
452 Put put = new Put(k);
453 put.add(f, null, k);
454 if (r.getLog() == null)
455 put.setDurability(Durability.SKIP_WAL);
456 r.put(put);
457 rowCount++;
458 }
459 }
460 if (flush) {
461 r.flushcache();
462 }
463 }
464 return rowCount;
465 }
466
467 }
468