View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.util;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.conf.Configured;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.HBaseConfiguration;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HRegionInfo;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.MasterNotRunningException;
34  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
35  import org.apache.hadoop.hbase.client.Delete;
36  import org.apache.hadoop.hbase.client.Get;
37  import org.apache.hadoop.hbase.client.HBaseAdmin;
38  import org.apache.hadoop.hbase.regionserver.HRegion;
39  import org.apache.hadoop.hbase.regionserver.wal.HLog;
40  import org.apache.hadoop.io.WritableComparator;
41  import org.apache.hadoop.util.GenericOptionsParser;
42  import org.apache.hadoop.util.Tool;
43  import org.apache.hadoop.util.ToolRunner;
44  
45  import java.io.IOException;
46  import java.util.List;
47  
48  /**
49   * Utility that can merge any two regions in the same table: adjacent,
50   * overlapping or disjoint.
51   */
52  public class Merge extends Configured implements Tool {
53    static final Log LOG = LogFactory.getLog(Merge.class);
54    private Path rootdir;
55    private volatile MetaUtils utils;
56    private byte [] tableName;               // Name of table
57    private volatile byte [] region1;        // Name of region 1
58    private volatile byte [] region2;        // Name of region 2
59    private volatile boolean isMetaTable;
60    private volatile HRegionInfo mergeInfo;
61  
62    /** default constructor */
63    public Merge() {
64      super();
65    }
66  
67    /**
68     * @param conf configuration
69     */
70    public Merge(Configuration conf) {
71      this.mergeInfo = null;
72      setConf(conf);
73    }
74  
75    public int run(String[] args) throws Exception {
76      if (parseArgs(args) != 0) {
77        return -1;
78      }
79  
80      // Verify file system is up.
81      FileSystem fs = FileSystem.get(getConf());              // get DFS handle
82      LOG.info("Verifying that file system is available...");
83      try {
84        FSUtils.checkFileSystemAvailable(fs);
85      } catch (IOException e) {
86        LOG.fatal("File system is not available", e);
87        return -1;
88      }
89  
90      // Verify HBase is down
91      LOG.info("Verifying that HBase is not running...");
92      try {
93        HBaseAdmin.checkHBaseAvailable(getConf());
94        LOG.fatal("HBase cluster must be off-line.");
95        return -1;
96      } catch (ZooKeeperConnectionException zkce) {
97        // If no zk, presume no master.
98      } catch (MasterNotRunningException e) {
99        // Expected. Ignore.
100     }
101 
102     // Initialize MetaUtils and and get the root of the HBase installation
103 
104     this.utils = new MetaUtils(getConf());
105     this.rootdir = FSUtils.getRootDir(getConf());
106     try {
107       if (isMetaTable) {
108         mergeTwoMetaRegions();
109       } else {
110         mergeTwoRegions();
111       }
112       return 0;
113     } catch (Exception e) {
114       LOG.fatal("Merge failed", e);
115       utils.scanMetaRegion(HRegionInfo.FIRST_META_REGIONINFO,
116           new MetaUtils.ScannerListener() {
117             public boolean processRow(HRegionInfo info) {
118               System.err.println(info.toString());
119               return true;
120             }
121           }
122       );
123 
124       return -1;
125 
126     } finally {
127       if (this.utils != null) {
128         this.utils.shutdown();
129       }
130     }
131   }
132 
133   /** @return HRegionInfo for merge result */
134   HRegionInfo getMergedHRegionInfo() {
135     return this.mergeInfo;
136   }
137 
138   /*
139    * Merge two meta regions. This is unlikely to be needed soon as we have only
140    * seend the meta table split once and that was with 64MB regions. With 256MB
141    * regions, it will be some time before someone has enough data in HBase to
142    * split the meta region and even less likely that a merge of two meta
143    * regions will be needed, but it is included for completeness.
144    */
145   private void mergeTwoMetaRegions() throws IOException {
146     HRegion rootRegion = utils.getRootRegion();
147     Get get = new Get(region1);
148     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
149     List<KeyValue> cells1 =  rootRegion.get(get, null).list();
150     HRegionInfo info1 = Writables.getHRegionInfo((cells1 == null)? null: cells1.get(0).getValue());
151 
152     get = new Get(region2);
153     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
154     List<KeyValue> cells2 =  rootRegion.get(get, null).list();
155     HRegionInfo info2 = Writables.getHRegionInfo((cells2 == null)? null: cells2.get(0).getValue());
156     HRegion merged = merge(info1, rootRegion, info2, rootRegion);
157     LOG.info("Adding " + merged.getRegionInfo() + " to " +
158         rootRegion.getRegionInfo());
159     HRegion.addRegionToMETA(rootRegion, merged);
160     merged.close();
161   }
162 
163   private static class MetaScannerListener
164   implements MetaUtils.ScannerListener {
165     private final byte [] region1;
166     private final byte [] region2;
167     private HRegionInfo meta1 = null;
168     private HRegionInfo meta2 = null;
169 
170     MetaScannerListener(final byte [] region1, final byte [] region2) {
171       this.region1 = region1;
172       this.region2 = region2;
173     }
174 
175     public boolean processRow(HRegionInfo info) {
176       if (meta1 == null && HRegion.rowIsInRange(info, region1)) {
177         meta1 = info;
178       }
179       if (region2 != null && meta2 == null &&
180           HRegion.rowIsInRange(info, region2)) {
181         meta2 = info;
182       }
183       return meta1 == null || (region2 != null && meta2 == null);
184     }
185 
186     HRegionInfo getMeta1() {
187       return meta1;
188     }
189 
190     HRegionInfo getMeta2() {
191       return meta2;
192     }
193   }
194 
195   /*
196    * Merges two regions from a user table.
197    */
198   private void mergeTwoRegions() throws IOException {
199     LOG.info("Merging regions " + Bytes.toStringBinary(this.region1) + " and " +
200         Bytes.toStringBinary(this.region2) + " in table " + Bytes.toString(this.tableName));
201     // Scan the root region for all the meta regions that contain the regions
202     // we're merging.
203     MetaScannerListener listener = new MetaScannerListener(region1, region2);
204     this.utils.scanRootRegion(listener);
205     HRegionInfo meta1 = listener.getMeta1();
206     if (meta1 == null) {
207       throw new IOException("Could not find meta region for " + Bytes.toStringBinary(region1));
208     }
209     HRegionInfo meta2 = listener.getMeta2();
210     if (meta2 == null) {
211       throw new IOException("Could not find meta region for " + Bytes.toStringBinary(region2));
212     }
213     LOG.info("Found meta for region1 " + Bytes.toStringBinary(meta1.getRegionName()) +
214       ", meta for region2 " + Bytes.toStringBinary(meta2.getRegionName()));
215     HRegion metaRegion1 = this.utils.getMetaRegion(meta1);
216     Get get = new Get(region1);
217     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
218     List<KeyValue> cells1 =  metaRegion1.get(get, null).list();
219     HRegionInfo info1 = Writables.getHRegionInfo((cells1 == null)? null: cells1.get(0).getValue());
220     if (info1== null) {
221       throw new NullPointerException("info1 is null using key " +
222           Bytes.toStringBinary(region1) + " in " + meta1);
223     }
224 
225     HRegion metaRegion2;
226     if (Bytes.equals(meta1.getRegionName(), meta2.getRegionName())) {
227       metaRegion2 = metaRegion1;
228     } else {
229       metaRegion2 = utils.getMetaRegion(meta2);
230     }
231     get = new Get(region2);
232     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
233     List<KeyValue> cells2 =  metaRegion2.get(get, null).list();
234     HRegionInfo info2 = Writables.getHRegionInfo((cells2 == null)? null: cells2.get(0).getValue());
235     if (info2 == null) {
236       throw new NullPointerException("info2 is null using key " + meta2);
237     }
238     HRegion merged = merge(info1, metaRegion1, info2, metaRegion2);
239 
240     // Now find the meta region which will contain the newly merged region
241 
242     listener = new MetaScannerListener(merged.getRegionName(), null);
243     utils.scanRootRegion(listener);
244     HRegionInfo mergedInfo = listener.getMeta1();
245     if (mergedInfo == null) {
246       throw new IOException("Could not find meta region for " +
247           Bytes.toStringBinary(merged.getRegionName()));
248     }
249     HRegion mergeMeta;
250     if (Bytes.equals(mergedInfo.getRegionName(), meta1.getRegionName())) {
251       mergeMeta = metaRegion1;
252     } else if (Bytes.equals(mergedInfo.getRegionName(), meta2.getRegionName())) {
253       mergeMeta = metaRegion2;
254     } else {
255       mergeMeta = utils.getMetaRegion(mergedInfo);
256     }
257     LOG.info("Adding " + merged.getRegionInfo() + " to " +
258         mergeMeta.getRegionInfo());
259 
260     HRegion.addRegionToMETA(mergeMeta, merged);
261     merged.close();
262   }
263 
264   /*
265    * Actually merge two regions and update their info in the meta region(s)
266    * If the meta is split, meta1 may be different from meta2. (and we may have
267    * to scan the meta if the resulting merged region does not go in either)
268    * Returns HRegion object for newly merged region
269    */
270   private HRegion merge(HRegionInfo info1, HRegion meta1, HRegionInfo info2,
271       HRegion meta2)
272   throws IOException {
273     if (info1 == null) {
274       throw new IOException("Could not find " + Bytes.toStringBinary(region1) + " in " +
275           Bytes.toStringBinary(meta1.getRegionName()));
276     }
277     if (info2 == null) {
278       throw new IOException("Cound not find " + Bytes.toStringBinary(region2) + " in " +
279           Bytes.toStringBinary(meta2.getRegionName()));
280     }
281     HRegion merged = null;
282     HLog log = utils.getLog();
283     HRegion r1 = HRegion.openHRegion(info1, log, getConf());
284     try {
285       HRegion r2 = HRegion.openHRegion(info2, log, getConf());
286       try {
287         merged = HRegion.merge(r1, r2);
288       } finally {
289         if (!r2.isClosed()) {
290           r2.close();
291         }
292       }
293     } finally {
294       if (!r1.isClosed()) {
295         r1.close();
296       }
297     }
298 
299     // Remove the old regions from meta.
300     // HRegion.merge has already deleted their files
301 
302     removeRegionFromMeta(meta1, info1);
303     removeRegionFromMeta(meta2, info2);
304 
305     this.mergeInfo = merged.getRegionInfo();
306     return merged;
307   }
308 
309   /*
310    * Removes a region's meta information from the passed <code>meta</code>
311    * region.
312    *
313    * @param meta META HRegion to be updated
314    * @param regioninfo HRegionInfo of region to remove from <code>meta</code>
315    *
316    * @throws IOException
317    */
318   private void removeRegionFromMeta(HRegion meta, HRegionInfo regioninfo)
319   throws IOException {
320     if (LOG.isDebugEnabled()) {
321       LOG.debug("Removing region: " + regioninfo + " from " + meta);
322     }
323 
324     Delete delete  = new Delete(regioninfo.getRegionName(),
325         System.currentTimeMillis(), null);
326     meta.delete(delete, null, true);
327   }
328 
329   /*
330    * Adds a region's meta information from the passed <code>meta</code>
331    * region.
332    *
333    * @param metainfo META HRegionInfo to be updated
334    * @param region HRegion to add to <code>meta</code>
335    *
336    * @throws IOException
337    */
338   private int parseArgs(String[] args) throws IOException {
339     GenericOptionsParser parser =
340       new GenericOptionsParser(getConf(), args);
341 
342     String[] remainingArgs = parser.getRemainingArgs();
343     if (remainingArgs.length != 3) {
344       usage();
345       return -1;
346     }
347     tableName = Bytes.toBytes(remainingArgs[0]);
348     isMetaTable = Bytes.compareTo(tableName, HConstants.META_TABLE_NAME) == 0;
349 
350     region1 = Bytes.toBytesBinary(remainingArgs[1]);
351     region2 = Bytes.toBytesBinary(remainingArgs[2]);
352     int status = 0;
353     if (notInTable(tableName, region1) || notInTable(tableName, region2)) {
354       status = -1;
355     } else if (Bytes.equals(region1, region2)) {
356       LOG.error("Can't merge a region with itself");
357       status = -1;
358     }
359     return status;
360   }
361 
362   private boolean notInTable(final byte [] tn, final byte [] rn) {
363     if (WritableComparator.compareBytes(tn, 0, tn.length, rn, 0, tn.length) != 0) {
364       LOG.error("Region " + Bytes.toStringBinary(rn) + " does not belong to table " +
365         Bytes.toString(tn));
366       return true;
367     }
368     return false;
369   }
370 
371   private void usage() {
372     System.err.println(
373         "Usage: bin/hbase merge <table-name> <region-1> <region-2>\n");
374   }
375 
376   public static void main(String[] args) {
377     int status;
378     try {
379       status = ToolRunner.run(HBaseConfiguration.create(), new Merge(), args);
380     } catch (Exception e) {
381       LOG.error("exiting due to error", e);
382       status = -1;
383     }
384     System.exit(status);
385   }
386 }