View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one or more
5    * contributor license agreements. See the NOTICE file distributed with this
6    * work for additional information regarding copyright ownership. The ASF
7    * licenses this file to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance with the License.
9    * You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16   * License for the specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  import static org.mockito.Mockito.*;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.List;
29  
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.TableName;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.HColumnDescriptor;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HTableDescriptor;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.Server;
40  import org.apache.hadoop.hbase.SmallTests;
41  import org.apache.hadoop.hbase.client.Durability;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.client.Scan;
44  import org.apache.hadoop.hbase.regionserver.wal.HLog;
45  import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.zookeeper.KeeperException;
48  import org.junit.After;
49  import org.junit.Before;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  import org.mockito.Mockito;
53  
54  import com.google.common.collect.ImmutableList;
55  
56  /**
57   * Test the {@link RegionMergeTransaction} class against two HRegions (as
58   * opposed to running cluster).
59   */
60  @Category(SmallTests.class)
61  public class TestRegionMergeTransaction {
62    private final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
63    private final Path testdir = TEST_UTIL.getDataTestDir(this.getClass()
64        .getName());
65    private HRegion region_a;
66    private HRegion region_b;
67    private HRegion region_c;
68    private HLog wal;
69    private FileSystem fs;
70    // Start rows of region_a,region_b,region_c
71    private static final byte[] STARTROW_A = new byte[] { 'a', 'a', 'a' };
72    private static final byte[] STARTROW_B = new byte[] { 'g', 'g', 'g' };
73    private static final byte[] STARTROW_C = new byte[] { 'w', 'w', 'w' };
74    private static final byte[] ENDROW = new byte[] { '{', '{', '{' };
75    private static final byte[] CF = HConstants.CATALOG_FAMILY;
76  
77    @Before
78    public void setup() throws IOException {
79      this.fs = FileSystem.get(TEST_UTIL.getConfiguration());
80      this.fs.delete(this.testdir, true);
81      this.wal = HLogFactory.createHLog(fs, this.testdir, "logs",
82          TEST_UTIL.getConfiguration());
83      this.region_a = createRegion(this.testdir, this.wal, STARTROW_A, STARTROW_B);
84      this.region_b = createRegion(this.testdir, this.wal, STARTROW_B, STARTROW_C);
85      this.region_c = createRegion(this.testdir, this.wal, STARTROW_C, ENDROW);
86      assert region_a != null && region_b != null && region_c != null;
87      TEST_UTIL.getConfiguration().setBoolean("hbase.testing.nocluster", true);
88    }
89  
90    @After
91    public void teardown() throws IOException {
92      for (HRegion region : new HRegion[] { region_a, region_b, region_c }) {
93        if (region != null && !region.isClosed()) region.close();
94        if (this.fs.exists(region.getRegionFileSystem().getRegionDir())
95            && !this.fs.delete(region.getRegionFileSystem().getRegionDir(), true)) {
96          throw new IOException("Failed deleting of "
97              + region.getRegionFileSystem().getRegionDir());
98        }
99      }
100     if (this.wal != null)
101       this.wal.closeAndDelete();
102     this.fs.delete(this.testdir, true);
103   }
104 
105   /**
106    * Test straight prepare works. Tries to merge on {@link #region_a} and
107    * {@link #region_b}
108    * @throws IOException
109    */
110   @Test
111   public void testPrepare() throws IOException {
112     prepareOnGoodRegions();
113   }
114 
115   private RegionMergeTransaction prepareOnGoodRegions() throws IOException {
116     RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_b,
117         false);
118     RegionMergeTransaction spyMT = Mockito.spy(mt);
119     doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
120         region_a.getRegionName());
121     doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
122         region_b.getRegionName());
123     assertTrue(spyMT.prepare(null));
124     return spyMT;
125   }
126 
127   /**
128    * Test merging the same region
129    */
130   @Test
131   public void testPrepareWithSameRegion() throws IOException {
132     RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
133         this.region_a, true);
134     assertFalse("should not merge the same region even if it is forcible ",
135         mt.prepare(null));
136   }
137 
138   /**
139    * Test merging two not adjacent regions under a common merge
140    */
141   @Test
142   public void testPrepareWithRegionsNotAdjacent() throws IOException {
143     RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
144         this.region_c, false);
145     assertFalse("should not merge two regions if they are adjacent except it is forcible",
146         mt.prepare(null));
147   }
148 
149   /**
150    * Test merging two not adjacent regions under a compulsory merge
151    */
152   @Test
153   public void testPrepareWithRegionsNotAdjacentUnderCompulsory()
154       throws IOException {
155     RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_c,
156         true);
157     RegionMergeTransaction spyMT = Mockito.spy(mt);
158     doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
159         region_a.getRegionName());
160     doReturn(false).when(spyMT).hasMergeQualifierInMeta(null,
161         region_c.getRegionName());
162     assertTrue("Since focible is true, should merge two regions even if they are not adjacent",
163         spyMT.prepare(null));
164   }
165 
166   /**
167    * Pass a reference store
168    */
169   @Test
170   public void testPrepareWithRegionsWithReference() throws IOException {
171     HStore storeMock = Mockito.mock(HStore.class);
172     when(storeMock.hasReferences()).thenReturn(true);
173     when(storeMock.getFamily()).thenReturn(new HColumnDescriptor("cf"));
174     when(storeMock.close()).thenReturn(ImmutableList.<StoreFile>of());
175     this.region_a.stores.put(Bytes.toBytes(""), storeMock);
176     RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
177         this.region_b, false);
178     assertFalse(
179         "a region should not be mergeable if it has instances of store file references",
180         mt.prepare(null));
181   }
182 
183   @Test
184   public void testPrepareWithClosedRegion() throws IOException {
185     this.region_a.close();
186     RegionMergeTransaction mt = new RegionMergeTransaction(this.region_a,
187         this.region_b, false);
188     assertFalse(mt.prepare(null));
189   }
190 
191   /**
192    * Test merging regions which are merged regions and has reference in META all
193    * the same
194    */
195   @Test
196   public void testPrepareWithRegionsWithMergeReference() throws IOException {
197     RegionMergeTransaction mt = new RegionMergeTransaction(region_a, region_b,
198         false);
199     RegionMergeTransaction spyMT = Mockito.spy(mt);
200     doReturn(true).when(spyMT).hasMergeQualifierInMeta(null,
201         region_a.getRegionName());
202     doReturn(true).when(spyMT).hasMergeQualifierInMeta(null,
203         region_b.getRegionName());
204     assertFalse(spyMT.prepare(null));
205   }
206 
207   @Test
208   public void testWholesomeMerge() throws IOException {
209     final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
210     final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
211     assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
212     assertEquals(rowCountOfRegionA, countRows(this.region_a));
213     assertEquals(rowCountOfRegionB, countRows(this.region_b));
214 
215     // Start transaction.
216     RegionMergeTransaction mt = prepareOnGoodRegions();
217 
218     // Run the execute. Look at what it returns.
219     Server mockServer = Mockito.mock(Server.class);
220     when(mockServer.getConfiguration())
221         .thenReturn(TEST_UTIL.getConfiguration());
222     HRegion mergedRegion = mt.execute(mockServer, null);
223     // Do some assertions about execution.
224     assertTrue(this.fs.exists(mt.getMergesDir()));
225     // Assert region_a and region_b is closed.
226     assertTrue(region_a.isClosed());
227     assertTrue(region_b.isClosed());
228 
229     // Assert mergedir is empty -- because its content will have been moved out
230     // to be under the merged region dirs.
231     assertEquals(0, this.fs.listStatus(mt.getMergesDir()).length);
232     // Check merged region have correct key span.
233     assertTrue(Bytes.equals(this.region_a.getStartKey(),
234         mergedRegion.getStartKey()));
235     assertTrue(Bytes.equals(this.region_b.getEndKey(),
236         mergedRegion.getEndKey()));
237     // Count rows. merged region are already open
238     try {
239       int mergedRegionRowCount = countRows(mergedRegion);
240       assertEquals((rowCountOfRegionA + rowCountOfRegionB),
241           mergedRegionRowCount);
242     } finally {
243       HRegion.closeHRegion(mergedRegion);
244     }
245     // Assert the write lock is no longer held on region_a and region_b
246     assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
247     assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
248   }
249 
250   @Test
251   public void testRollback() throws IOException {
252     final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
253     final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
254     assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
255     assertEquals(rowCountOfRegionA, countRows(this.region_a));
256     assertEquals(rowCountOfRegionB, countRows(this.region_b));
257 
258     // Start transaction.
259     RegionMergeTransaction mt = prepareOnGoodRegions();
260 
261     when(mt.createMergedRegionFromMerges(region_a, region_b,
262         mt.getMergedRegionInfo())).thenThrow(
263         new MockedFailedMergedRegionCreation());
264 
265     // Run the execute. Look at what it returns.
266     boolean expectedException = false;
267     Server mockServer = Mockito.mock(Server.class);
268     when(mockServer.getConfiguration())
269         .thenReturn(TEST_UTIL.getConfiguration());
270     try {
271       mt.execute(mockServer, null);
272     } catch (MockedFailedMergedRegionCreation e) {
273       expectedException = true;
274     }
275     assertTrue(expectedException);
276     // Run rollback
277     assertTrue(mt.rollback(null, null));
278 
279     // Assert I can scan region_a and region_b.
280     int rowCountOfRegionA2 = countRows(this.region_a);
281     assertEquals(rowCountOfRegionA, rowCountOfRegionA2);
282     int rowCountOfRegionB2 = countRows(this.region_b);
283     assertEquals(rowCountOfRegionB, rowCountOfRegionB2);
284 
285     // Assert rollback cleaned up stuff in fs
286     assertTrue(!this.fs.exists(HRegion.getRegionDir(this.testdir,
287         mt.getMergedRegionInfo())));
288 
289     assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
290     assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
291 
292     // Now retry the merge but do not throw an exception this time.
293     assertTrue(mt.prepare(null));
294     HRegion mergedRegion = mt.execute(mockServer, null);
295     // Count rows. daughters are already open
296     // Count rows. merged region are already open
297     try {
298       int mergedRegionRowCount = countRows(mergedRegion);
299       assertEquals((rowCountOfRegionA + rowCountOfRegionB),
300           mergedRegionRowCount);
301     } finally {
302       HRegion.closeHRegion(mergedRegion);
303     }
304     // Assert the write lock is no longer held on region_a and region_b
305     assertTrue(!this.region_a.lock.writeLock().isHeldByCurrentThread());
306     assertTrue(!this.region_b.lock.writeLock().isHeldByCurrentThread());
307   }
308 
309   @Test
310   public void testFailAfterPONR() throws IOException, KeeperException {
311     final int rowCountOfRegionA = loadRegion(this.region_a, CF, true);
312     final int rowCountOfRegionB = loadRegion(this.region_b, CF, true);
313     assertTrue(rowCountOfRegionA > 0 && rowCountOfRegionB > 0);
314     assertEquals(rowCountOfRegionA, countRows(this.region_a));
315     assertEquals(rowCountOfRegionB, countRows(this.region_b));
316 
317     // Start transaction.
318     RegionMergeTransaction mt = prepareOnGoodRegions();
319     Mockito.doThrow(new MockedFailedMergedRegionOpen())
320         .when(mt)
321         .openMergedRegion((Server) Mockito.anyObject(),
322             (RegionServerServices) Mockito.anyObject(),
323             (HRegion) Mockito.anyObject());
324 
325     // Run the execute. Look at what it returns.
326     boolean expectedException = false;
327     Server mockServer = Mockito.mock(Server.class);
328     when(mockServer.getConfiguration())
329         .thenReturn(TEST_UTIL.getConfiguration());
330     try {
331       mt.execute(mockServer, null);
332     } catch (MockedFailedMergedRegionOpen e) {
333       expectedException = true;
334     }
335     assertTrue(expectedException);
336     // Run rollback returns false that we should restart.
337     assertFalse(mt.rollback(null, null));
338     // Make sure that merged region is still in the filesystem, that
339     // they have not been removed; this is supposed to be the case if we go
340     // past point of no return.
341     Path tableDir = this.region_a.getRegionFileSystem().getRegionDir()
342         .getParent();
343     Path mergedRegionDir = new Path(tableDir, mt.getMergedRegionInfo()
344         .getEncodedName());
345     assertTrue(TEST_UTIL.getTestFileSystem().exists(mergedRegionDir));
346   }
347 
348   @Test
349   public void testMeregedRegionBoundary() {
350     TableName tableName =
351         TableName.valueOf("testMeregedRegionBoundary");
352     byte[] a = Bytes.toBytes("a");
353     byte[] b = Bytes.toBytes("b");
354     byte[] z = Bytes.toBytes("z");
355     HRegionInfo r1 = new HRegionInfo(tableName);
356     HRegionInfo r2 = new HRegionInfo(tableName, a, z);
357     HRegionInfo m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
358     assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
359         && Bytes.equals(m.getEndKey(), r1.getEndKey()));
360 
361     r1 = new HRegionInfo(tableName, null, a);
362     r2 = new HRegionInfo(tableName, a, z);
363     m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
364     assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
365         && Bytes.equals(m.getEndKey(), r2.getEndKey()));
366 
367     r1 = new HRegionInfo(tableName, null, a);
368     r2 = new HRegionInfo(tableName, z, null);
369     m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
370     assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
371         && Bytes.equals(m.getEndKey(), r2.getEndKey()));
372 
373     r1 = new HRegionInfo(tableName, a, z);
374     r2 = new HRegionInfo(tableName, z, null);
375     m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
376     assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
377       && Bytes.equals(m.getEndKey(), r2.getEndKey()));
378 
379     r1 = new HRegionInfo(tableName, a, b);
380     r2 = new HRegionInfo(tableName, b, z);
381     m = RegionMergeTransaction.getMergedRegionInfo(r1, r2);
382     assertTrue(Bytes.equals(m.getStartKey(), r1.getStartKey())
383       && Bytes.equals(m.getEndKey(), r2.getEndKey()));
384   }
385 
386   /**
387    * Exception used in this class only.
388    */
389   @SuppressWarnings("serial")
390   private class MockedFailedMergedRegionCreation extends IOException {
391   }
392 
393   @SuppressWarnings("serial")
394   private class MockedFailedMergedRegionOpen extends IOException {
395   }
396 
397   private HRegion createRegion(final Path testdir, final HLog wal,
398       final byte[] startrow, final byte[] endrow)
399       throws IOException {
400     // Make a region with start and end keys.
401     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("table"));
402     HColumnDescriptor hcd = new HColumnDescriptor(CF);
403     htd.addFamily(hcd);
404     HRegionInfo hri = new HRegionInfo(htd.getTableName(), startrow, endrow);
405     HRegion a = HRegion.createHRegion(hri, testdir,
406         TEST_UTIL.getConfiguration(), htd);
407     HRegion.closeHRegion(a);
408     return HRegion.openHRegion(testdir, hri, htd, wal,
409         TEST_UTIL.getConfiguration());
410   }
411 
412   private int countRows(final HRegion r) throws IOException {
413     int rowcount = 0;
414     InternalScanner scanner = r.getScanner(new Scan());
415     try {
416       List<KeyValue> kvs = new ArrayList<KeyValue>();
417       boolean hasNext = true;
418       while (hasNext) {
419         hasNext = scanner.next(kvs);
420         if (!kvs.isEmpty())
421           rowcount++;
422       }
423     } finally {
424       scanner.close();
425     }
426     return rowcount;
427   }
428 
429   /**
430    * Load region with rows from 'aaa' to 'zzz', skip the rows which are out of
431    * range of the region
432    * @param r Region
433    * @param f Family
434    * @param flush flush the cache if true
435    * @return Count of rows loaded.
436    * @throws IOException
437    */
438   private int loadRegion(final HRegion r, final byte[] f, final boolean flush)
439       throws IOException {
440     byte[] k = new byte[3];
441     int rowCount = 0;
442     for (byte b1 = 'a'; b1 <= 'z'; b1++) {
443       for (byte b2 = 'a'; b2 <= 'z'; b2++) {
444         for (byte b3 = 'a'; b3 <= 'z'; b3++) {
445           k[0] = b1;
446           k[1] = b2;
447           k[2] = b3;
448           if (!HRegion.rowIsInRange(r.getRegionInfo(), k)) {
449             continue;
450           }
451           Put put = new Put(k);
452           put.add(f, null, k);
453           if (r.getLog() == null)
454             put.setDurability(Durability.SKIP_WAL);
455           r.put(put);
456           rowCount++;
457         }
458       }
459       if (flush) {
460         r.flushcache();
461       }
462     }
463     return rowCount;
464   }
465 
466 }
467