1   /**
2    * Copyright 2008 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.util;
22  
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HBaseTestCase;
31  import org.apache.hadoop.hbase.HColumnDescriptor;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.KeyValue;
36  import org.apache.hadoop.hbase.client.Get;
37  import org.apache.hadoop.hbase.client.Put;
38  import org.apache.hadoop.hbase.client.Result;
39  import org.apache.hadoop.hbase.client.Scan;
40  import org.apache.hadoop.hbase.regionserver.wal.HLog;
41  import org.apache.hadoop.hbase.regionserver.HRegion;
42  import org.apache.hadoop.hbase.regionserver.InternalScanner;
43  import org.apache.hadoop.hdfs.MiniDFSCluster;
44  import org.apache.hadoop.util.ToolRunner;
45  
46  /** Test stand alone merge tool that can merge arbitrary regions */
47  public class TestMergeTool extends HBaseTestCase {
48    static final Log LOG = LogFactory.getLog(TestMergeTool.class);
49  //  static final byte [] COLUMN_NAME = Bytes.toBytes("contents:");
50    static final byte [] FAMILY = Bytes.toBytes("contents");
51    static final byte [] QUALIFIER = Bytes.toBytes("dc");
52  
53    private final HRegionInfo[] sourceRegions = new HRegionInfo[5];
54    private final HRegion[] regions = new HRegion[5];
55    private HTableDescriptor desc;
56    private byte [][][] rows;
57    private MiniDFSCluster dfsCluster = null;
58  
59    @Override
60    public void setUp() throws Exception {
61      this.conf.set("hbase.hstore.compactionThreshold", "2");
62  
63      // Create table description
64      this.desc = new HTableDescriptor("TestMergeTool");
65      this.desc.addFamily(new HColumnDescriptor(FAMILY));
66  
67      /*
68       * Create the HRegionInfos for the regions.
69       */
70      // Region 0 will contain the key range [row_0200,row_0300)
71      sourceRegions[0] = new HRegionInfo(this.desc, Bytes.toBytes("row_0200"),
72        Bytes.toBytes("row_0300"));
73  
74      // Region 1 will contain the key range [row_0250,row_0400) and overlaps
75      // with Region 0
76      sourceRegions[1] =
77        new HRegionInfo(this.desc, Bytes.toBytes("row_0250"),
78            Bytes.toBytes("row_0400"));
79  
80      // Region 2 will contain the key range [row_0100,row_0200) and is adjacent
81      // to Region 0 or the region resulting from the merge of Regions 0 and 1
82      sourceRegions[2] =
83        new HRegionInfo(this.desc, Bytes.toBytes("row_0100"),
84            Bytes.toBytes("row_0200"));
85  
86      // Region 3 will contain the key range [row_0500,row_0600) and is not
87      // adjacent to any of Regions 0, 1, 2 or the merged result of any or all
88      // of those regions
89      sourceRegions[3] =
90        new HRegionInfo(this.desc, Bytes.toBytes("row_0500"),
91            Bytes.toBytes("row_0600"));
92  
93      // Region 4 will have empty start and end keys and overlaps all regions.
94      sourceRegions[4] =
95        new HRegionInfo(this.desc, HConstants.EMPTY_BYTE_ARRAY,
96            HConstants.EMPTY_BYTE_ARRAY);
97  
98      /*
99       * Now create some row keys
100      */
101     this.rows = new byte [5][][];
102     this.rows[0] = Bytes.toByteArrays(new String[] { "row_0210", "row_0280" });
103     this.rows[1] = Bytes.toByteArrays(new String[] { "row_0260", "row_0350",
104         "row_035" });
105     this.rows[2] = Bytes.toByteArrays(new String[] { "row_0110", "row_0175",
106         "row_0175", "row_0175"});
107     this.rows[3] = Bytes.toByteArrays(new String[] { "row_0525", "row_0560",
108         "row_0560", "row_0560", "row_0560"});
109     this.rows[4] = Bytes.toByteArrays(new String[] { "row_0050", "row_1000",
110         "row_1000", "row_1000", "row_1000", "row_1000" });
111 
112     // Start up dfs
113     this.dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null);
114     this.fs = this.dfsCluster.getFileSystem();
115     System.out.println("fs=" + this.fs);
116     this.conf.set("fs.defaultFS", fs.getUri().toString());
117     Path parentdir = fs.getHomeDirectory();
118     conf.set(HConstants.HBASE_DIR, parentdir.toString());
119     fs.mkdirs(parentdir);
120     FSUtils.setVersion(fs, parentdir);
121 
122     // Note: we must call super.setUp after starting the mini cluster or
123     // we will end up with a local file system
124 
125     super.setUp();
126     try {
127       // Create root and meta regions
128       createRootAndMetaRegions();
129       /*
130        * Create the regions we will merge
131        */
132       for (int i = 0; i < sourceRegions.length; i++) {
133         regions[i] =
134           HRegion.createHRegion(this.sourceRegions[i], this.testDir, this.conf);
135         /*
136          * Insert data
137          */
138         for (int j = 0; j < rows[i].length; j++) {
139           byte [] row = rows[i][j];
140           Put put = new Put(row);
141           put.add(FAMILY, QUALIFIER, row);
142           regions[i].put(put);
143         }
144         HRegion.addRegionToMETA(meta, regions[i]);
145       }
146       // Close root and meta regions
147       closeRootAndMeta();
148 
149     } catch (Exception e) {
150       shutdownDfs(dfsCluster);
151       throw e;
152     }
153   }
154 
155   @Override
156   public void tearDown() throws Exception {
157     super.tearDown();
158     shutdownDfs(dfsCluster);
159   }
160 
161   /*
162    * @param msg Message that describes this merge
163    * @param regionName1
164    * @param regionName2
165    * @param log Log to use merging.
166    * @param upperbound Verifying, how high up in this.rows to go.
167    * @return Merged region.
168    * @throws Exception
169    */
170   private HRegion mergeAndVerify(final String msg, final String regionName1,
171     final String regionName2, final HLog log, final int upperbound)
172   throws Exception {
173     Merge merger = new Merge(this.conf);
174     LOG.info(msg);
175     System.out.println("fs2=" + this.conf.get("fs.defaultFS"));
176     int errCode = ToolRunner.run(this.conf, merger,
177       new String[] {this.desc.getNameAsString(), regionName1, regionName2}
178     );
179     assertTrue("'" + msg + "' failed", errCode == 0);
180     HRegionInfo mergedInfo = merger.getMergedHRegionInfo();
181 
182     // Now verify that we can read all the rows from regions 0, 1
183     // in the new merged region.
184     HRegion merged =
185       HRegion.openHRegion(mergedInfo, this.testDir, log, this.conf);
186     verifyMerge(merged, upperbound);
187     merged.close();
188     LOG.info("Verified " + msg);
189     return merged;
190   }
191 
192   private void verifyMerge(final HRegion merged, final int upperbound)
193   throws IOException {
194     //Test
195     Scan scan = new Scan();
196     scan.addFamily(FAMILY);
197     InternalScanner scanner = merged.getScanner(scan);
198     try {
199     List<KeyValue> testRes = null;
200       while (true) {
201         testRes = new ArrayList<KeyValue>();
202         boolean hasNext = scanner.next(testRes);
203         if (!hasNext) {
204           break;
205         }
206       }
207     } finally {
208       scanner.close();
209     }
210 
211     //!Test
212 
213     for (int i = 0; i < upperbound; i++) {
214       for (int j = 0; j < rows[i].length; j++) {
215         Get get = new Get(rows[i][j]);
216         get.addFamily(FAMILY);
217         Result result = merged.get(get, null);
218         assertEquals(1, result.size());
219         byte [] bytes = result.sorted()[0].getValue();
220         assertNotNull(Bytes.toStringBinary(rows[i][j]), bytes);
221         assertTrue(Bytes.equals(bytes, rows[i][j]));
222       }
223     }
224   }
225 
226   /**
227    * Test merge tool.
228    * @throws Exception
229    */
230   public void testMergeTool() throws Exception {
231     // First verify we can read the rows from the source regions and that they
232     // contain the right data.
233     for (int i = 0; i < regions.length; i++) {
234       for (int j = 0; j < rows[i].length; j++) {
235         Get get = new Get(rows[i][j]);
236         get.addFamily(FAMILY);
237         Result result = regions[i].get(get, null);
238         byte [] bytes = result.sorted()[0].getValue();
239         assertNotNull(bytes);
240         assertTrue(Bytes.equals(bytes, rows[i][j]));
241       }
242       // Close the region and delete the log
243       regions[i].close();
244       regions[i].getLog().closeAndDelete();
245     }
246 
247     // Create a log that we can reuse when we need to open regions
248     Path logPath = new Path("/tmp", HConstants.HREGION_LOGDIR_NAME + "_" +
249       System.currentTimeMillis());
250     LOG.info("Creating log " + logPath.toString());
251     Path oldLogDir = new Path("/tmp", HConstants.HREGION_OLDLOGDIR_NAME);
252     HLog log = new HLog(this.fs, logPath, oldLogDir, this.conf, null);
253     try {
254        // Merge Region 0 and Region 1
255       HRegion merged = mergeAndVerify("merging regions 0 and 1",
256         this.sourceRegions[0].getRegionNameAsString(),
257         this.sourceRegions[1].getRegionNameAsString(), log, 2);
258 
259       // Merge the result of merging regions 0 and 1 with region 2
260       merged = mergeAndVerify("merging regions 0+1 and 2",
261         merged.getRegionInfo().getRegionNameAsString(),
262         this.sourceRegions[2].getRegionNameAsString(), log, 3);
263 
264       // Merge the result of merging regions 0, 1 and 2 with region 3
265       merged = mergeAndVerify("merging regions 0+1+2 and 3",
266         merged.getRegionInfo().getRegionNameAsString(),
267         this.sourceRegions[3].getRegionNameAsString(), log, 4);
268 
269       // Merge the result of merging regions 0, 1, 2 and 3 with region 4
270       merged = mergeAndVerify("merging regions 0+1+2+3 and 4",
271         merged.getRegionInfo().getRegionNameAsString(),
272         this.sourceRegions[4].getRegionNameAsString(), log, rows.length);
273     } finally {
274       log.closeAndDelete();
275     }
276   }
277 }