View Javadoc

1   /**
2    * Copyright The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one or more
5    * contributor license agreements. See the NOTICE file distributed with this
6    * work for additional information regarding copyright ownership. The ASF
7    * licenses this file to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance with the License.
9    * You may obtain a copy of the License at
10   *
11   * http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16   * License for the specific language governing permissions and limitations
17   * under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.util.List;
28  
29  import org.apache.commons.lang.math.RandomUtils;
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HTableDescriptor;
38  import org.apache.hadoop.hbase.testclassification.LargeTests;
39  import org.apache.hadoop.hbase.MiniHBaseCluster;
40  import org.apache.hadoop.hbase.ServerName;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.UnknownRegionException;
43  import org.apache.hadoop.hbase.MetaTableAccessor;
44  import org.apache.hadoop.hbase.client.Admin;
45  import org.apache.hadoop.hbase.client.Put;
46  import org.apache.hadoop.hbase.client.Result;
47  import org.apache.hadoop.hbase.client.ResultScanner;
48  import org.apache.hadoop.hbase.client.Scan;
49  import org.apache.hadoop.hbase.client.Table;
50  import org.apache.hadoop.hbase.exceptions.MergeRegionException;
51  import org.apache.hadoop.hbase.master.AssignmentManager;
52  import org.apache.hadoop.hbase.master.HMaster;
53  import org.apache.hadoop.hbase.master.RegionState.State;
54  import org.apache.hadoop.hbase.master.RegionStates;
55  import org.apache.hadoop.hbase.util.Bytes;
56  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
57  import org.apache.hadoop.hbase.util.FSUtils;
58  import org.apache.hadoop.hbase.util.Pair;
59  import org.apache.hadoop.hbase.util.PairOfSameType;
60  import org.apache.hadoop.util.StringUtils;
61  import org.junit.AfterClass;
62  import org.junit.BeforeClass;
63  import org.junit.Test;
64  import org.junit.experimental.categories.Category;
65  
66  import com.google.common.base.Joiner;
67  
68  /**
69   * Like {@link TestRegionMergeTransaction} in that we're testing
70   * {@link RegionMergeTransaction} only the below tests are against a running
71   * cluster where {@link TestRegionMergeTransaction} is tests against bare
72   * {@link HRegion}.
73   */
74  @Category(LargeTests.class)
75  public class TestRegionMergeTransactionOnCluster {
76    private static final Log LOG = LogFactory
77        .getLog(TestRegionMergeTransactionOnCluster.class);
78    private static final int NB_SERVERS = 3;
79  
80    private static final byte[] FAMILYNAME = Bytes.toBytes("fam");
81    private static final byte[] QUALIFIER = Bytes.toBytes("q");
82  
83    private static byte[] ROW = Bytes.toBytes("testRow");
84    private static final int INITIAL_REGION_NUM = 10;
85    private static final int ROWSIZE = 200;
86    private static byte[][] ROWS = makeN(ROW, ROWSIZE);
87  
88    private static int waitTime = 60 * 1000;
89  
90    static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
91  
92    private static HMaster master;
93    private static Admin admin;
94  
95    static void setupOnce() throws Exception {
96      // Start a cluster
97      TEST_UTIL.startMiniCluster(NB_SERVERS);
98      MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
99      master = cluster.getMaster();
100     master.balanceSwitch(false);
101     admin = TEST_UTIL.getHBaseAdmin();
102   }
103 
104   @BeforeClass
105   public static void beforeAllTests() throws Exception {
106     // Use ZK for region assignment
107     TEST_UTIL.getConfiguration().setBoolean("hbase.assignment.usezk", true);
108     setupOnce();
109   }
110 
111   @AfterClass
112   public static void afterAllTests() throws Exception {
113     TEST_UTIL.shutdownMiniCluster();
114   }
115 
116   @Test
117   public void testWholesomeMerge() throws Exception {
118     LOG.info("Starting testWholesomeMerge");
119     final TableName tableName =
120         TableName.valueOf("testWholesomeMerge");
121 
122     // Create table and load data.
123     Table table = createTableAndLoadData(master, tableName);
124     // Merge 1st and 2nd region
125     mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1,
126         INITIAL_REGION_NUM - 1);
127 
128     // Merge 2nd and 3th region
129     PairOfSameType<HRegionInfo> mergedRegions =
130       mergeRegionsAndVerifyRegionNum(master, tableName, 1, 2,
131         INITIAL_REGION_NUM - 2);
132 
133     verifyRowCount(table, ROWSIZE);
134 
135     // Randomly choose one of the two merged regions
136     HRegionInfo hri = RandomUtils.nextBoolean() ?
137       mergedRegions.getFirst() : mergedRegions.getSecond();
138     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
139     AssignmentManager am = cluster.getMaster().getAssignmentManager();
140     RegionStates regionStates = am.getRegionStates();
141     long start = EnvironmentEdgeManager.currentTime();
142     while (!regionStates.isRegionInState(hri, State.MERGED)) {
143       assertFalse("Timed out in waiting one merged region to be in state MERGED",
144         EnvironmentEdgeManager.currentTime() - start > 60000);
145       Thread.sleep(500);
146     }
147 
148     // We should not be able to assign it again
149     am.assign(hri, true, true);
150     assertFalse("Merged region can't be assigned",
151       regionStates.isRegionInTransition(hri));
152     assertTrue(regionStates.isRegionInState(hri, State.MERGED));
153 
154     // We should not be able to unassign it either
155     am.unassign(hri, true, null);
156     assertFalse("Merged region can't be unassigned",
157       regionStates.isRegionInTransition(hri));
158     assertTrue(regionStates.isRegionInState(hri, State.MERGED));
159 
160     table.close();
161   }
162 
163   @Test
164   public void testCleanMergeReference() throws Exception {
165     LOG.info("Starting testCleanMergeReference");
166     admin.enableCatalogJanitor(false);
167     try {
168       final TableName tableName =
169           TableName.valueOf("testCleanMergeReference");
170       // Create table and load data.
171       Table table = createTableAndLoadData(master, tableName);
172       // Merge 1st and 2nd region
173       mergeRegionsAndVerifyRegionNum(master, tableName, 0, 1,
174           INITIAL_REGION_NUM - 1);
175       verifyRowCount(table, ROWSIZE);
176       table.close();
177 
178       List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor
179           .getTableRegionsAndLocations(master.getZooKeeper(), master.getConnection(), tableName);
180       HRegionInfo mergedRegionInfo = tableRegions.get(0).getFirst();
181       HTableDescriptor tableDescritor = master.getTableDescriptors().get(
182           tableName);
183       Result mergedRegionResult = MetaTableAccessor.getRegionResult(
184         master.getConnection(), mergedRegionInfo.getRegionName());
185 
186       // contains merge reference in META
187       assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
188           HConstants.MERGEA_QUALIFIER) != null);
189       assertTrue(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
190           HConstants.MERGEB_QUALIFIER) != null);
191 
192       // merging regions' directory are in the file system all the same
193       HRegionInfo regionA = HRegionInfo.getHRegionInfo(mergedRegionResult,
194           HConstants.MERGEA_QUALIFIER);
195       HRegionInfo regionB = HRegionInfo.getHRegionInfo(mergedRegionResult,
196           HConstants.MERGEB_QUALIFIER);
197       FileSystem fs = master.getMasterFileSystem().getFileSystem();
198       Path rootDir = master.getMasterFileSystem().getRootDir();
199 
200       Path tabledir = FSUtils.getTableDir(rootDir, mergedRegionInfo.getTable());
201       Path regionAdir = new Path(tabledir, regionA.getEncodedName());
202       Path regionBdir = new Path(tabledir, regionB.getEncodedName());
203       assertTrue(fs.exists(regionAdir));
204       assertTrue(fs.exists(regionBdir));
205 
206       admin.compactRegion(mergedRegionInfo.getRegionName());
207       // wait until merged region doesn't have reference file
208       long timeout = System.currentTimeMillis() + waitTime;
209       HRegionFileSystem hrfs = new HRegionFileSystem(
210           TEST_UTIL.getConfiguration(), fs, tabledir, mergedRegionInfo);
211       while (System.currentTimeMillis() < timeout) {
212         if (!hrfs.hasReferences(tableDescritor)) {
213           break;
214         }
215         Thread.sleep(50);
216       }
217       assertFalse(hrfs.hasReferences(tableDescritor));
218 
219       // run CatalogJanitor to clean merge references in hbase:meta and archive the
220       // files of merging regions
221       int cleaned = admin.runCatalogScan();
222       assertTrue(cleaned > 0);
223       assertFalse(fs.exists(regionAdir));
224       assertFalse(fs.exists(regionBdir));
225 
226       mergedRegionResult = MetaTableAccessor.getRegionResult(
227         master.getConnection(), mergedRegionInfo.getRegionName());
228       assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
229           HConstants.MERGEA_QUALIFIER) != null);
230       assertFalse(mergedRegionResult.getValue(HConstants.CATALOG_FAMILY,
231           HConstants.MERGEB_QUALIFIER) != null);
232 
233     } finally {
234       admin.enableCatalogJanitor(true);
235     }
236   }
237 
238   /**
239    * This test tests 1, merging region not online;
240    * 2, merging same two regions; 3, merging unknown regions.
241    * They are in one test case so that we don't have to create
242    * many tables, and these tests are simple.
243    */
244   @Test
245   public void testMerge() throws Exception {
246     LOG.info("Starting testMerge");
247     final TableName tableName = TableName.valueOf("testMerge");
248 
249     try {
250       // Create table and load data.
251       Table table = createTableAndLoadData(master, tableName);
252       RegionStates regionStates = master.getAssignmentManager().getRegionStates();
253       List<HRegionInfo> regions = regionStates.getRegionsOfTable(tableName);
254       // Fake offline one region
255       HRegionInfo a = regions.get(0);
256       HRegionInfo b = regions.get(1);
257       regionStates.regionOffline(a);
258       try {
259         // Merge offline region. Region a is offline here
260         admin.mergeRegions(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
261         fail("Offline regions should not be able to merge");
262       } catch (IOException ie) {
263         System.out.println(ie);
264         assertTrue("Exception should mention regions not online",
265           StringUtils.stringifyException(ie).contains("regions not online")
266             && ie instanceof MergeRegionException);
267       }
268       try {
269         // Merge the same region: b and b.
270         admin.mergeRegions(b.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), true);
271         fail("A region should not be able to merge with itself, even forcifully");
272       } catch (IOException ie) {
273         assertTrue("Exception should mention regions not online",
274           StringUtils.stringifyException(ie).contains("region to itself")
275             && ie instanceof MergeRegionException);
276       }
277       try {
278         // Merge unknown regions
279         admin.mergeRegions(Bytes.toBytes("-f1"), Bytes.toBytes("-f2"), true);
280         fail("Unknown region could not be merged");
281       } catch (IOException ie) {
282         assertTrue("UnknownRegionException should be thrown",
283           ie instanceof UnknownRegionException);
284       }
285       table.close();
286     } finally {
287       TEST_UTIL.deleteTable(tableName);
288     }
289   }
290 
291   private PairOfSameType<HRegionInfo> mergeRegionsAndVerifyRegionNum(
292       HMaster master, TableName tablename,
293       int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
294     PairOfSameType<HRegionInfo> mergedRegions =
295       requestMergeRegion(master, tablename, regionAnum, regionBnum);
296     waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
297     return mergedRegions;
298   }
299 
300   private PairOfSameType<HRegionInfo> requestMergeRegion(
301       HMaster master, TableName tablename,
302       int regionAnum, int regionBnum) throws Exception {
303     List<Pair<HRegionInfo, ServerName>> tableRegions = MetaTableAccessor
304         .getTableRegionsAndLocations(master.getZooKeeper(),
305           master.getConnection(), tablename);
306     HRegionInfo regionA = tableRegions.get(regionAnum).getFirst();
307     HRegionInfo regionB = tableRegions.get(regionBnum).getFirst();
308     TEST_UTIL.getHBaseAdmin().mergeRegions(
309       regionA.getEncodedNameAsBytes(),
310       regionB.getEncodedNameAsBytes(), false);
311     return new PairOfSameType<HRegionInfo>(regionA, regionB);
312   }
313 
314   private void waitAndVerifyRegionNum(HMaster master, TableName tablename,
315       int expectedRegionNum) throws Exception {
316     List<Pair<HRegionInfo, ServerName>> tableRegionsInMeta;
317     List<HRegionInfo> tableRegionsInMaster;
318     long timeout = System.currentTimeMillis() + waitTime;
319     while (System.currentTimeMillis() < timeout) {
320       tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
321         master.getConnection(), tablename);
322       tableRegionsInMaster = master.getAssignmentManager().getRegionStates()
323           .getRegionsOfTable(tablename);
324       if (tableRegionsInMeta.size() == expectedRegionNum
325           && tableRegionsInMaster.size() == expectedRegionNum) {
326         break;
327       }
328       Thread.sleep(250);
329     }
330 
331     tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
332       master.getConnection(), tablename);
333     LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
334     assertEquals(expectedRegionNum, tableRegionsInMeta.size());
335   }
336 
337   private Table createTableAndLoadData(HMaster master, TableName tablename)
338       throws Exception {
339     return createTableAndLoadData(master, tablename, INITIAL_REGION_NUM);
340   }
341 
342   private Table createTableAndLoadData(HMaster master, TableName tablename,
343       int numRegions) throws Exception {
344     assertTrue("ROWSIZE must > numregions:" + numRegions, ROWSIZE > numRegions);
345     byte[][] splitRows = new byte[numRegions - 1][];
346     for (int i = 0; i < splitRows.length; i++) {
347       splitRows[i] = ROWS[(i + 1) * ROWSIZE / numRegions];
348     }
349 
350     Table table = TEST_UTIL.createTable(tablename, FAMILYNAME, splitRows);
351     loadData(table);
352     verifyRowCount(table, ROWSIZE);
353 
354     // sleep here is an ugly hack to allow region transitions to finish
355     long timeout = System.currentTimeMillis() + waitTime;
356     List<Pair<HRegionInfo, ServerName>> tableRegions;
357     while (System.currentTimeMillis() < timeout) {
358       tableRegions = MetaTableAccessor.getTableRegionsAndLocations(master.getZooKeeper(),
359         master.getConnection(), tablename);
360       if (tableRegions.size() == numRegions)
361         break;
362       Thread.sleep(250);
363     }
364 
365     tableRegions = MetaTableAccessor.getTableRegionsAndLocations(
366       master.getZooKeeper(),
367       master.getConnection(), tablename);
368     LOG.info("Regions after load: " + Joiner.on(',').join(tableRegions));
369     assertEquals(numRegions, tableRegions.size());
370     return table;
371   }
372 
373   private static byte[][] makeN(byte[] base, int n) {
374     byte[][] ret = new byte[n][];
375     for (int i = 0; i < n; i++) {
376       ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%04d", i)));
377     }
378     return ret;
379   }
380 
381   private void loadData(Table table) throws IOException {
382     for (int i = 0; i < ROWSIZE; i++) {
383       Put put = new Put(ROWS[i]);
384       put.add(FAMILYNAME, QUALIFIER, Bytes.toBytes(i));
385       table.put(put);
386     }
387   }
388 
389   private void verifyRowCount(Table table, int expectedRegionNum)
390       throws IOException {
391     ResultScanner scanner = table.getScanner(new Scan());
392     int rowCount = 0;
393     while (scanner.next() != null) {
394       rowCount++;
395     }
396     assertEquals(expectedRegionNum, rowCount);
397     scanner.close();
398   }
399 }