1   /**
2    * Copyright 2007 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.util;
21  
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.List;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.fs.FileSystem;
31  import org.apache.hadoop.fs.Path;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HColumnDescriptor;
34  import org.apache.hadoop.hbase.HRegionInfo;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.catalog.CatalogTracker;
37  import org.apache.hadoop.hbase.catalog.MetaReader;
38  import org.apache.hadoop.hbase.client.HBaseAdmin;
39  import org.apache.hadoop.hbase.client.HConnection;
40  import org.apache.hadoop.hbase.client.HConnectionManager;
41  import org.apache.hadoop.hbase.client.Put;
42  import org.apache.hadoop.hbase.regionserver.HRegion;
43  import org.junit.Test;
44  
45  /**
46   * Tests merging a normal table's regions
47   */
48  public class TestMergeTable {
49    private static final Log LOG = LogFactory.getLog(TestMergeTable.class);
50    private final HBaseTestingUtility UTIL = new HBaseTestingUtility();
51    private static final byte [] COLUMN_NAME = Bytes.toBytes("contents");
52    private static final byte [] VALUE;
53    static {
54      // We will use the same value for the rows as that is not really important here
55      String partialValue = String.valueOf(System.currentTimeMillis());
56      StringBuilder val = new StringBuilder();
57      while (val.length() < 1024) {
58        val.append(partialValue);
59      }
60      VALUE = Bytes.toBytes(val.toString());
61    }
62  
63    /**
64     * Test merge.
65     * Hand-makes regions of a mergeable size and adds the hand-made regions to
66     * hand-made meta.  The hand-made regions are created offline.  We then start
67     * up mini cluster, disables the hand-made table and starts in on merging.
68     * @throws Exception 
69     */
70    @Test (timeout=300000) public void testMergeTable() throws Exception {
71      // Table we are manually creating offline.
72      HTableDescriptor desc = new HTableDescriptor(Bytes.toBytes("test"));
73      desc.addFamily(new HColumnDescriptor(COLUMN_NAME));
74  
75      // Set maximum regionsize down.
76      UTIL.getConfiguration().setLong("hbase.hregion.max.filesize", 64L * 1024L * 1024L);
77      // Make it so we don't split.
78      UTIL.getConfiguration().setInt("hbase.regionserver.regionSplitLimit", 0);
79      // Startup hdfs.  Its in here we'll be putting our manually made regions.
80      UTIL.startMiniDFSCluster(1);
81      // Create hdfs hbase rootdir.
82      Path rootdir = UTIL.createRootDir();
83      FileSystem fs = FileSystem.get(UTIL.getConfiguration());
84      if (fs.exists(rootdir)) {
85        if (fs.delete(rootdir, true)) {
86          LOG.info("Cleaned up existing " + rootdir);
87        }
88      }
89  
90      // Now create three data regions: The first is too large to merge since it
91      // will be > 64 MB in size. The second two will be smaller and will be
92      // selected for merging.
93  
94      // To ensure that the first region is larger than 64MB we need to write at
95      // least 65536 rows. We will make certain by writing 70000
96      byte [] row_70001 = Bytes.toBytes("row_70001");
97      byte [] row_80001 = Bytes.toBytes("row_80001");
98  
99      // Create regions and populate them at same time.
100     HRegion [] regions = {
101       createRegion(desc, null, row_70001, 1, 70000, rootdir),
102       createRegion(desc, row_70001, row_80001, 70001, 10000, rootdir),
103       createRegion(desc, row_80001, null, 80001, 11000, rootdir)
104     };
105 
106     // Now create the root and meta regions and insert the data regions
107     // created above into .META.
108     setupROOTAndMeta(rootdir, regions);
109     try {
110       LOG.info("Starting mini zk cluster");
111       UTIL.startMiniZKCluster();
112       LOG.info("Starting mini hbase cluster");
113       UTIL.startMiniHBaseCluster(1, 1);
114       Configuration c = new Configuration(UTIL.getConfiguration());
115       HConnection connection = HConnectionManager.getConnection(c);
116       CatalogTracker ct = new CatalogTracker(connection);
117       ct.start();
118       List<HRegionInfo> originalTableRegions =
119         MetaReader.getTableRegions(ct, desc.getName());
120       LOG.info("originalTableRegions size=" + originalTableRegions.size() +
121         "; " + originalTableRegions);
122       HBaseAdmin admin = new HBaseAdmin(new Configuration(c));
123       admin.disableTable(desc.getName());
124       HMerge.merge(c, FileSystem.get(c), desc.getName());
125       List<HRegionInfo> postMergeTableRegions =
126         MetaReader.getTableRegions(ct, desc.getName());
127       LOG.info("postMergeTableRegions size=" + postMergeTableRegions.size() +
128         "; " + postMergeTableRegions);
129       assertTrue("originalTableRegions=" + originalTableRegions.size() +
130         ", postMergeTableRegions=" + postMergeTableRegions.size(),
131         postMergeTableRegions.size() < originalTableRegions.size());
132     } finally {
133       UTIL.shutdownMiniCluster();
134     }
135   }
136 
137   private HRegion createRegion(final HTableDescriptor desc,
138       byte [] startKey, byte [] endKey, int firstRow, int nrows, Path rootdir)
139   throws IOException {
140     HRegionInfo hri = new HRegionInfo(desc, startKey, endKey);
141     HRegion region = HRegion.createHRegion(hri, rootdir, UTIL.getConfiguration());
142     LOG.info("Created region " + region.getRegionNameAsString());
143     for(int i = firstRow; i < firstRow + nrows; i++) {
144       Put put = new Put(Bytes.toBytes("row_" + String.format("%1$05d", i)));
145       put.add(COLUMN_NAME, null,  VALUE);
146       region.put(put);
147       if (i % 10000 == 0) {
148         LOG.info("Flushing write #" + i);
149         region.flushcache();
150       }
151     }
152     region.close();
153     region.getLog().closeAndDelete();
154     return region;
155   }
156 
157   protected void setupROOTAndMeta(Path rootdir, final HRegion [] regions)
158   throws IOException {
159     HRegion root =
160       HRegion.createHRegion(HRegionInfo.ROOT_REGIONINFO, rootdir, UTIL.getConfiguration());
161     HRegion meta =
162       HRegion.createHRegion(HRegionInfo.FIRST_META_REGIONINFO, rootdir,
163       UTIL.getConfiguration());
164     HRegion.addRegionToMETA(root, meta);
165     for (HRegion r: regions) {
166       HRegion.addRegionToMETA(meta, r);
167     }
168     meta.close();
169     meta.getLog().closeAndDelete();
170     root.close();
171     root.getLog().closeAndDelete();
172   }
173 }