View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.util;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.conf.Configured;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.HBaseConfiguration;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HRegionInfo;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.MasterNotRunningException;
34  import org.apache.hadoop.hbase.client.Delete;
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.HBaseAdmin;
37  import org.apache.hadoop.hbase.regionserver.HRegion;
38  import org.apache.hadoop.hbase.regionserver.wal.HLog;
39  import org.apache.hadoop.io.WritableComparator;
40  import org.apache.hadoop.util.GenericOptionsParser;
41  import org.apache.hadoop.util.Tool;
42  import org.apache.hadoop.util.ToolRunner;
43  
44  import java.io.IOException;
45  import java.util.List;
46  
47  /**
48   * Utility that can merge any two regions in the same table: adjacent,
49   * overlapping or disjoint.
50   */
51  public class Merge extends Configured implements Tool {
52    static final Log LOG = LogFactory.getLog(Merge.class);
53    private Path rootdir;
54    private volatile MetaUtils utils;
55    private byte [] tableName;               // Name of table
56    private volatile byte [] region1;        // Name of region 1
57    private volatile byte [] region2;        // Name of region 2
58    private volatile boolean isMetaTable;
59    private volatile HRegionInfo mergeInfo;
60  
61    /** default constructor */
62    public Merge() {
63      super();
64    }
65  
66    /**
67     * @param conf configuration
68     */
69    public Merge(Configuration conf) {
70      this.mergeInfo = null;
71      setConf(conf);
72    }
73  
74    public int run(String[] args) throws Exception {
75      if (parseArgs(args) != 0) {
76        return -1;
77      }
78  
79      // Verify file system is up.
80      FileSystem fs = FileSystem.get(getConf());              // get DFS handle
81      LOG.info("Verifying that file system is available...");
82      try {
83        FSUtils.checkFileSystemAvailable(fs);
84      } catch (IOException e) {
85        LOG.fatal("File system is not available", e);
86        return -1;
87      }
88  
89      // Verify HBase is down
90      LOG.info("Verifying that HBase is not running...");
91      try {
92        HBaseAdmin.checkHBaseAvailable(getConf());
93        LOG.fatal("HBase cluster must be off-line.");
94        return -1;
95      } catch (MasterNotRunningException e) {
96        // Expected. Ignore.
97      }
98  
99      // Initialize MetaUtils and and get the root of the HBase installation
100 
101     this.utils = new MetaUtils(getConf());
102     this.rootdir = FSUtils.getRootDir(getConf());
103     try {
104       if (isMetaTable) {
105         mergeTwoMetaRegions();
106       } else {
107         mergeTwoRegions();
108       }
109       return 0;
110     } catch (Exception e) {
111       LOG.fatal("Merge failed", e);
112       utils.scanMetaRegion(HRegionInfo.FIRST_META_REGIONINFO,
113           new MetaUtils.ScannerListener() {
114             public boolean processRow(HRegionInfo info) {
115               System.err.println(info.toString());
116               return true;
117             }
118           }
119       );
120 
121       return -1;
122 
123     } finally {
124       if (this.utils != null) {
125         this.utils.shutdown();
126       }
127     }
128   }
129 
130   /** @return HRegionInfo for merge result */
131   HRegionInfo getMergedHRegionInfo() {
132     return this.mergeInfo;
133   }
134 
135   /*
136    * Merge two meta regions. This is unlikely to be needed soon as we have only
137    * seend the meta table split once and that was with 64MB regions. With 256MB
138    * regions, it will be some time before someone has enough data in HBase to
139    * split the meta region and even less likely that a merge of two meta
140    * regions will be needed, but it is included for completeness.
141    */
142   private void mergeTwoMetaRegions() throws IOException {
143     HRegion rootRegion = utils.getRootRegion();
144     Get get = new Get(region1);
145     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
146     List<KeyValue> cells1 =  rootRegion.get(get, null).list();
147     HRegionInfo info1 = Writables.getHRegionInfo((cells1 == null)? null: cells1.get(0).getValue());
148 
149     get = new Get(region2);
150     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
151     List<KeyValue> cells2 =  rootRegion.get(get, null).list();
152     HRegionInfo info2 = Writables.getHRegionInfo((cells2 == null)? null: cells2.get(0).getValue());
153     HRegion merged = merge(info1, rootRegion, info2, rootRegion);
154     LOG.info("Adding " + merged.getRegionInfo() + " to " +
155         rootRegion.getRegionInfo());
156     HRegion.addRegionToMETA(rootRegion, merged);
157     merged.close();
158   }
159 
160   private static class MetaScannerListener
161   implements MetaUtils.ScannerListener {
162     private final byte [] region1;
163     private final byte [] region2;
164     private HRegionInfo meta1 = null;
165     private HRegionInfo meta2 = null;
166 
167     MetaScannerListener(final byte [] region1, final byte [] region2) {
168       this.region1 = region1;
169       this.region2 = region2;
170     }
171 
172     public boolean processRow(HRegionInfo info) {
173       if (meta1 == null && HRegion.rowIsInRange(info, region1)) {
174         meta1 = info;
175       }
176       if (region2 != null && meta2 == null &&
177           HRegion.rowIsInRange(info, region2)) {
178         meta2 = info;
179       }
180       return meta1 == null || (region2 != null && meta2 == null);
181     }
182 
183     HRegionInfo getMeta1() {
184       return meta1;
185     }
186 
187     HRegionInfo getMeta2() {
188       return meta2;
189     }
190   }
191 
192   /*
193    * Merges two regions from a user table.
194    */
195   private void mergeTwoRegions() throws IOException {
196     LOG.info("Merging regions " + Bytes.toString(this.region1) + " and " +
197         Bytes.toString(this.region2) + " in table " + Bytes.toString(this.tableName));
198     // Scan the root region for all the meta regions that contain the regions
199     // we're merging.
200     MetaScannerListener listener = new MetaScannerListener(region1, region2);
201     this.utils.scanRootRegion(listener);
202     HRegionInfo meta1 = listener.getMeta1();
203     if (meta1 == null) {
204       throw new IOException("Could not find meta region for " + Bytes.toString(region1));
205     }
206     HRegionInfo meta2 = listener.getMeta2();
207     if (meta2 == null) {
208       throw new IOException("Could not find meta region for " + Bytes.toString(region2));
209     }
210     LOG.info("Found meta for region1 " + Bytes.toString(meta1.getRegionName()) +
211       ", meta for region2 " + Bytes.toString(meta2.getRegionName()));
212     HRegion metaRegion1 = this.utils.getMetaRegion(meta1);
213     Get get = new Get(region1);
214     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
215     List<KeyValue> cells1 =  metaRegion1.get(get, null).list();
216     HRegionInfo info1 = Writables.getHRegionInfo((cells1 == null)? null: cells1.get(0).getValue());
217     if (info1== null) {
218       throw new NullPointerException("info1 is null using key " +
219           Bytes.toString(region1) + " in " + meta1);
220     }
221 
222     HRegion metaRegion2;
223     if (Bytes.equals(meta1.getRegionName(), meta2.getRegionName())) {
224       metaRegion2 = metaRegion1;
225     } else {
226       metaRegion2 = utils.getMetaRegion(meta2);
227     }
228     get = new Get(region2);
229     get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
230     List<KeyValue> cells2 =  metaRegion2.get(get, null).list();
231     HRegionInfo info2 = Writables.getHRegionInfo((cells2 == null)? null: cells2.get(0).getValue());
232     if (info2 == null) {
233       throw new NullPointerException("info2 is null using key " + meta2);
234     }
235     HRegion merged = merge(info1, metaRegion1, info2, metaRegion2);
236 
237     // Now find the meta region which will contain the newly merged region
238 
239     listener = new MetaScannerListener(merged.getRegionName(), null);
240     utils.scanRootRegion(listener);
241     HRegionInfo mergedInfo = listener.getMeta1();
242     if (mergedInfo == null) {
243       throw new IOException("Could not find meta region for " +
244           Bytes.toString(merged.getRegionName()));
245     }
246     HRegion mergeMeta;
247     if (Bytes.equals(mergedInfo.getRegionName(), meta1.getRegionName())) {
248       mergeMeta = metaRegion1;
249     } else if (Bytes.equals(mergedInfo.getRegionName(), meta2.getRegionName())) {
250       mergeMeta = metaRegion2;
251     } else {
252       mergeMeta = utils.getMetaRegion(mergedInfo);
253     }
254     LOG.info("Adding " + merged.getRegionInfo() + " to " +
255         mergeMeta.getRegionInfo());
256 
257     HRegion.addRegionToMETA(mergeMeta, merged);
258     merged.close();
259   }
260 
261   /*
262    * Actually merge two regions and update their info in the meta region(s)
263    * If the meta is split, meta1 may be different from meta2. (and we may have
264    * to scan the meta if the resulting merged region does not go in either)
265    * Returns HRegion object for newly merged region
266    */
267   private HRegion merge(HRegionInfo info1, HRegion meta1, HRegionInfo info2,
268       HRegion meta2)
269   throws IOException {
270     if (info1 == null) {
271       throw new IOException("Could not find " + Bytes.toString(region1) + " in " +
272           Bytes.toString(meta1.getRegionName()));
273     }
274     if (info2 == null) {
275       throw new IOException("Cound not find " + Bytes.toString(region2) + " in " +
276           Bytes.toString(meta2.getRegionName()));
277     }
278     HRegion merged = null;
279     HLog log = utils.getLog();
280     HRegion r1 = HRegion.openHRegion(info1, this.rootdir, log, getConf());
281     try {
282       HRegion r2 = HRegion.openHRegion(info2, this.rootdir, log, getConf());
283       try {
284         merged = HRegion.merge(r1, r2);
285       } finally {
286         if (!r2.isClosed()) {
287           r2.close();
288         }
289       }
290     } finally {
291       if (!r1.isClosed()) {
292         r1.close();
293       }
294     }
295 
296     // Remove the old regions from meta.
297     // HRegion.merge has already deleted their files
298 
299     removeRegionFromMeta(meta1, info1);
300     removeRegionFromMeta(meta2, info2);
301 
302     this.mergeInfo = merged.getRegionInfo();
303     return merged;
304   }
305 
306   /*
307    * Removes a region's meta information from the passed <code>meta</code>
308    * region.
309    *
310    * @param meta META HRegion to be updated
311    * @param regioninfo HRegionInfo of region to remove from <code>meta</code>
312    *
313    * @throws IOException
314    */
315   private void removeRegionFromMeta(HRegion meta, HRegionInfo regioninfo)
316   throws IOException {
317     if (LOG.isDebugEnabled()) {
318       LOG.debug("Removing region: " + regioninfo + " from " + meta);
319     }
320 
321     Delete delete  = new Delete(regioninfo.getRegionName(),
322         System.currentTimeMillis(), null);
323     meta.delete(delete, null, true);
324   }
325 
326   /*
327    * Adds a region's meta information from the passed <code>meta</code>
328    * region.
329    *
330    * @param metainfo META HRegionInfo to be updated
331    * @param region HRegion to add to <code>meta</code>
332    *
333    * @throws IOException
334    */
335   private int parseArgs(String[] args) throws IOException {
336     GenericOptionsParser parser =
337       new GenericOptionsParser(getConf(), args);
338 
339     String[] remainingArgs = parser.getRemainingArgs();
340     if (remainingArgs.length != 3) {
341       usage();
342       return -1;
343     }
344     tableName = Bytes.toBytes(remainingArgs[0]);
345     isMetaTable = Bytes.compareTo(tableName, HConstants.META_TABLE_NAME) == 0;
346 
347     region1 = Bytes.toBytesBinary(remainingArgs[1]);
348     region2 = Bytes.toBytesBinary(remainingArgs[2]);
349     int status = 0;
350     if (notInTable(tableName, region1) || notInTable(tableName, region2)) {
351       status = -1;
352     } else if (Bytes.equals(region1, region2)) {
353       LOG.error("Can't merge a region with itself");
354       status = -1;
355     }
356     return status;
357   }
358 
359   private boolean notInTable(final byte [] tn, final byte [] rn) {
360     if (WritableComparator.compareBytes(tn, 0, tn.length, rn, 0, tn.length) != 0) {
361       LOG.error("Region " + Bytes.toString(rn) + " does not belong to table " +
362         Bytes.toString(tn));
363       return true;
364     }
365     return false;
366   }
367 
368   private void usage() {
369     System.err.println(
370         "Usage: bin/hbase merge <table-name> <region-1> <region-2>\n");
371   }
372 
373   public static void main(String[] args) {
374     int status;
375     try {
376       status = ToolRunner.run(HBaseConfiguration.create(), new Merge(), args);
377     } catch (Exception e) {
378       LOG.error("exiting due to error", e);
379       status = -1;
380     }
381     System.exit(status);
382   }
383 }