View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.util;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.Cell;
30  import org.apache.hadoop.hbase.CellUtil;
31  import org.apache.hadoop.hbase.HBaseTestCase;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HColumnDescriptor;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.HRegionInfo;
36  import org.apache.hadoop.hbase.HTableDescriptor;
37  import org.apache.hadoop.hbase.LargeTests;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.client.Get;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.client.Result;
42  import org.apache.hadoop.hbase.client.Scan;
43  import org.apache.hadoop.hbase.regionserver.HRegion;
44  import org.apache.hadoop.hbase.regionserver.InternalScanner;
45  import org.apache.hadoop.hbase.regionserver.wal.HLog;
46  import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
47  import org.apache.hadoop.hdfs.MiniDFSCluster;
48  import org.apache.hadoop.util.ToolRunner;
49  import org.junit.experimental.categories.Category;
50  
51  /** Test stand alone merge tool that can merge arbitrary regions */
52  @Category(LargeTests.class)
53  public class TestMergeTool extends HBaseTestCase {
54    static final Log LOG = LogFactory.getLog(TestMergeTool.class);
55    HBaseTestingUtility TEST_UTIL;
56  //  static final byte [] COLUMN_NAME = Bytes.toBytes("contents:");
57    static final byte [] FAMILY = Bytes.toBytes("contents");
58    static final byte [] QUALIFIER = Bytes.toBytes("dc");
59  
60    private final HRegionInfo[] sourceRegions = new HRegionInfo[5];
61    private final HRegion[] regions = new HRegion[5];
62    private HTableDescriptor desc;
63    private byte [][][] rows;
64    private MiniDFSCluster dfsCluster = null;
65  
66    @Override
67    public void setUp() throws Exception {
68      // Set the timeout down else this test will take a while to complete.
69      this.conf.setLong("hbase.zookeeper.recoverable.waittime", 10);
70      // Make it so we try and connect to a zk that is not there (else we might
71      // find a zk ensemble put up by another concurrent test and this will
72      // mess up this test.  Choose unlikely port. Default test port is 21818.
73      // Default zk port is 2181.
74      this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, 10001);
75  
76      this.conf.set("hbase.hstore.compactionThreshold", "2");
77  
78      // Create table description
79      this.desc = new HTableDescriptor(TableName.valueOf("TestMergeTool"));
80      this.desc.addFamily(new HColumnDescriptor(FAMILY));
81  
82      /*
83       * Create the HRegionInfos for the regions.
84       */
85      // Region 0 will contain the key range [row_0200,row_0300)
86      sourceRegions[0] = new HRegionInfo(this.desc.getTableName(),
87          Bytes.toBytes("row_0200"),
88        Bytes.toBytes("row_0300"));
89  
90      // Region 1 will contain the key range [row_0250,row_0400) and overlaps
91      // with Region 0
92      sourceRegions[1] =
93        new HRegionInfo(this.desc.getTableName(),
94            Bytes.toBytes("row_0250"),
95            Bytes.toBytes("row_0400"));
96  
97      // Region 2 will contain the key range [row_0100,row_0200) and is adjacent
98      // to Region 0 or the region resulting from the merge of Regions 0 and 1
99      sourceRegions[2] =
100       new HRegionInfo(this.desc.getTableName(),
101           Bytes.toBytes("row_0100"),
102           Bytes.toBytes("row_0200"));
103 
104     // Region 3 will contain the key range [row_0500,row_0600) and is not
105     // adjacent to any of Regions 0, 1, 2 or the merged result of any or all
106     // of those regions
107     sourceRegions[3] =
108       new HRegionInfo(this.desc.getTableName(),
109           Bytes.toBytes("row_0500"),
110           Bytes.toBytes("row_0600"));
111 
112     // Region 4 will have empty start and end keys and overlaps all regions.
113     sourceRegions[4] =
114       new HRegionInfo(this.desc.getTableName(),
115           HConstants.EMPTY_BYTE_ARRAY,
116           HConstants.EMPTY_BYTE_ARRAY);
117 
118     /*
119      * Now create some row keys
120      */
121     this.rows = new byte [5][][];
122     this.rows[0] = Bytes.toByteArrays(new String[] { "row_0210", "row_0280" });
123     this.rows[1] = Bytes.toByteArrays(new String[] { "row_0260", "row_0350",
124         "row_035" });
125     this.rows[2] = Bytes.toByteArrays(new String[] { "row_0110", "row_0175",
126         "row_0175", "row_0175"});
127     this.rows[3] = Bytes.toByteArrays(new String[] { "row_0525", "row_0560",
128         "row_0560", "row_0560", "row_0560"});
129     this.rows[4] = Bytes.toByteArrays(new String[] { "row_0050", "row_1000",
130         "row_1000", "row_1000", "row_1000", "row_1000" });
131 
132     // Start up dfs
133     TEST_UTIL = new HBaseTestingUtility(conf);
134     this.dfsCluster = TEST_UTIL.startMiniDFSCluster(2);
135     this.fs = this.dfsCluster.getFileSystem();
136     System.out.println("fs=" + this.fs);
137     FSUtils.setFsDefault(this.conf, new Path(fs.getUri()));
138     Path parentdir = fs.getHomeDirectory();
139     FSUtils.setRootDir(conf, parentdir);
140     fs.mkdirs(parentdir);
141     FSUtils.setVersion(fs, parentdir);
142 
143     // Note: we must call super.setUp after starting the mini cluster or
144     // we will end up with a local file system
145 
146     super.setUp();
147     try {
148       // Create meta region
149       createMetaRegion();
150       new FSTableDescriptors(this.fs, this.testDir).createTableDescriptor(this.desc);
151       /*
152        * Create the regions we will merge
153        */
154       for (int i = 0; i < sourceRegions.length; i++) {
155         regions[i] =
156           HRegion.createHRegion(this.sourceRegions[i], this.testDir, this.conf,
157               this.desc);
158         /*
159          * Insert data
160          */
161         for (int j = 0; j < rows[i].length; j++) {
162           byte [] row = rows[i][j];
163           Put put = new Put(row);
164           put.add(FAMILY, QUALIFIER, row);
165           regions[i].put(put);
166         }
167         HRegion.addRegionToMETA(meta, regions[i]);
168       }
169       // Close root and meta regions
170       closeRootAndMeta();
171 
172     } catch (Exception e) {
173       TEST_UTIL.shutdownMiniCluster();
174       throw e;
175     }
176   }
177 
178   @Override
179   public void tearDown() throws Exception {
180     super.tearDown();
181     for (int i = 0; i < sourceRegions.length; i++) {
182       HRegion r = regions[i];
183       if (r != null) {
184         HRegion.closeHRegion(r);
185       }
186     }
187     TEST_UTIL.shutdownMiniCluster();
188   }
189 
190   /*
191    * @param msg Message that describes this merge
192    * @param regionName1
193    * @param regionName2
194    * @param log Log to use merging.
195    * @param upperbound Verifying, how high up in this.rows to go.
196    * @return Merged region.
197    * @throws Exception
198    */
199   private HRegion mergeAndVerify(final String msg, final String regionName1,
200     final String regionName2, final HLog log, final int upperbound)
201   throws Exception {
202     Merge merger = new Merge(this.conf);
203     LOG.info(msg);
204     LOG.info("fs2=" + this.conf.get("fs.defaultFS"));
205     int errCode = ToolRunner.run(this.conf, merger,
206       new String[] {this.desc.getTableName().getNameAsString(), regionName1, regionName2}
207     );
208     assertTrue("'" + msg + "' failed with errCode " + errCode, errCode == 0);
209     HRegionInfo mergedInfo = merger.getMergedHRegionInfo();
210 
211     // Now verify that we can read all the rows from regions 0, 1
212     // in the new merged region.
213     HRegion merged = HRegion.openHRegion(mergedInfo, this.desc, log, this.conf);
214     verifyMerge(merged, upperbound);
215     merged.close();
216     LOG.info("Verified " + msg);
217     return merged;
218   }
219 
220   private void verifyMerge(final HRegion merged, final int upperbound)
221   throws IOException {
222     //Test
223     Scan scan = new Scan();
224     scan.addFamily(FAMILY);
225     InternalScanner scanner = merged.getScanner(scan);
226     try {
227     List<Cell> testRes = null;
228       while (true) {
229         testRes = new ArrayList<Cell>();
230         boolean hasNext = scanner.next(testRes);
231         if (!hasNext) {
232           break;
233         }
234       }
235     } finally {
236       scanner.close();
237     }
238 
239     //!Test
240 
241     for (int i = 0; i < upperbound; i++) {
242       for (int j = 0; j < rows[i].length; j++) {
243         Get get = new Get(rows[i][j]);
244         get.addFamily(FAMILY);
245         Result result = merged.get(get);
246         assertEquals(1, result.size());
247         byte [] bytes = CellUtil.cloneValue(result.rawCells()[0]);
248         assertNotNull(Bytes.toStringBinary(rows[i][j]), bytes);
249         assertTrue(Bytes.equals(bytes, rows[i][j]));
250       }
251     }
252   }
253 
254   /**
255    * Test merge tool.
256    * @throws Exception
257    */
258   public void testMergeTool() throws Exception {
259     // First verify we can read the rows from the source regions and that they
260     // contain the right data.
261     for (int i = 0; i < regions.length; i++) {
262       for (int j = 0; j < rows[i].length; j++) {
263         Get get = new Get(rows[i][j]);
264         get.addFamily(FAMILY);
265         Result result = regions[i].get(get);
266         byte [] bytes =  CellUtil.cloneValue(result.rawCells()[0]);
267         assertNotNull(bytes);
268         assertTrue(Bytes.equals(bytes, rows[i][j]));
269       }
270       // Close the region and delete the log
271       HRegion.closeHRegion(regions[i]);
272     }
273 
274     // Create a log that we can reuse when we need to open regions
275     Path logPath = new Path("/tmp");
276     String logName = HConstants.HREGION_LOGDIR_NAME + "_"
277       + System.currentTimeMillis();
278     LOG.info("Creating log " + logPath.toString() + "/" + logName);
279 
280     HLog log = HLogFactory.createHLog(this.fs, logPath, 
281         logName, this.conf);
282     
283     try {
284        // Merge Region 0 and Region 1
285       HRegion merged = mergeAndVerify("merging regions 0 and 1 ",
286         this.sourceRegions[0].getRegionNameAsString(),
287         this.sourceRegions[1].getRegionNameAsString(), log, 2);
288 
289       // Merge the result of merging regions 0 and 1 with region 2
290       merged = mergeAndVerify("merging regions 0+1 and 2",
291         merged.getRegionInfo().getRegionNameAsString(),
292         this.sourceRegions[2].getRegionNameAsString(), log, 3);
293 
294       // Merge the result of merging regions 0, 1 and 2 with region 3
295       merged = mergeAndVerify("merging regions 0+1+2 and 3",
296         merged.getRegionInfo().getRegionNameAsString(),
297         this.sourceRegions[3].getRegionNameAsString(), log, 4);
298 
299       // Merge the result of merging regions 0, 1, 2 and 3 with region 4
300       merged = mergeAndVerify("merging regions 0+1+2+3 and 4",
301         merged.getRegionInfo().getRegionNameAsString(),
302         this.sourceRegions[4].getRegionNameAsString(), log, rows.length);
303     } finally {
304       log.closeAndDelete();
305     }
306   }
307 
308 }
309