1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.util;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.conf.Configured;
27 import org.apache.hadoop.fs.FileSystem;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.HBaseConfiguration;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HRegionInfo;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.MasterNotRunningException;
34 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
35 import org.apache.hadoop.hbase.client.Delete;
36 import org.apache.hadoop.hbase.client.Get;
37 import org.apache.hadoop.hbase.client.HBaseAdmin;
38 import org.apache.hadoop.hbase.regionserver.HRegion;
39 import org.apache.hadoop.hbase.regionserver.wal.HLog;
40 import org.apache.hadoop.io.WritableComparator;
41 import org.apache.hadoop.util.GenericOptionsParser;
42 import org.apache.hadoop.util.Tool;
43 import org.apache.hadoop.util.ToolRunner;
44
45 import java.io.IOException;
46 import java.util.List;
47
48
49
50
51
52 public class Merge extends Configured implements Tool {
53 static final Log LOG = LogFactory.getLog(Merge.class);
54 private Path rootdir;
55 private volatile MetaUtils utils;
56 private byte [] tableName;
57 private volatile byte [] region1;
58 private volatile byte [] region2;
59 private volatile boolean isMetaTable;
60 private volatile HRegionInfo mergeInfo;
61
62
63 public Merge() {
64 super();
65 }
66
67
68
69
70 public Merge(Configuration conf) {
71 this.mergeInfo = null;
72 setConf(conf);
73 }
74
75 public int run(String[] args) throws Exception {
76 if (parseArgs(args) != 0) {
77 return -1;
78 }
79
80
81 FileSystem fs = FileSystem.get(getConf());
82 LOG.info("Verifying that file system is available...");
83 try {
84 FSUtils.checkFileSystemAvailable(fs);
85 } catch (IOException e) {
86 LOG.fatal("File system is not available", e);
87 return -1;
88 }
89
90
91 LOG.info("Verifying that HBase is not running...");
92 try {
93 HBaseAdmin.checkHBaseAvailable(getConf());
94 LOG.fatal("HBase cluster must be off-line.");
95 return -1;
96 } catch (ZooKeeperConnectionException zkce) {
97
98 } catch (MasterNotRunningException e) {
99
100 }
101
102
103
104 this.utils = new MetaUtils(getConf());
105 this.rootdir = FSUtils.getRootDir(getConf());
106 try {
107 if (isMetaTable) {
108 mergeTwoMetaRegions();
109 } else {
110 mergeTwoRegions();
111 }
112 return 0;
113 } catch (Exception e) {
114 LOG.fatal("Merge failed", e);
115 utils.scanMetaRegion(HRegionInfo.FIRST_META_REGIONINFO,
116 new MetaUtils.ScannerListener() {
117 public boolean processRow(HRegionInfo info) {
118 System.err.println(info.toString());
119 return true;
120 }
121 }
122 );
123
124 return -1;
125
126 } finally {
127 if (this.utils != null) {
128 this.utils.shutdown();
129 }
130 }
131 }
132
133
134 HRegionInfo getMergedHRegionInfo() {
135 return this.mergeInfo;
136 }
137
138
139
140
141
142
143
144
145 private void mergeTwoMetaRegions() throws IOException {
146 HRegion rootRegion = utils.getRootRegion();
147 Get get = new Get(region1);
148 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
149 List<KeyValue> cells1 = rootRegion.get(get, null).list();
150 HRegionInfo info1 = Writables.getHRegionInfo((cells1 == null)? null: cells1.get(0).getValue());
151
152 get = new Get(region2);
153 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
154 List<KeyValue> cells2 = rootRegion.get(get, null).list();
155 HRegionInfo info2 = Writables.getHRegionInfo((cells2 == null)? null: cells2.get(0).getValue());
156 HRegion merged = merge(info1, rootRegion, info2, rootRegion);
157 LOG.info("Adding " + merged.getRegionInfo() + " to " +
158 rootRegion.getRegionInfo());
159 HRegion.addRegionToMETA(rootRegion, merged);
160 merged.close();
161 }
162
163 private static class MetaScannerListener
164 implements MetaUtils.ScannerListener {
165 private final byte [] region1;
166 private final byte [] region2;
167 private HRegionInfo meta1 = null;
168 private HRegionInfo meta2 = null;
169
170 MetaScannerListener(final byte [] region1, final byte [] region2) {
171 this.region1 = region1;
172 this.region2 = region2;
173 }
174
175 public boolean processRow(HRegionInfo info) {
176 if (meta1 == null && HRegion.rowIsInRange(info, region1)) {
177 meta1 = info;
178 }
179 if (region2 != null && meta2 == null &&
180 HRegion.rowIsInRange(info, region2)) {
181 meta2 = info;
182 }
183 return meta1 == null || (region2 != null && meta2 == null);
184 }
185
186 HRegionInfo getMeta1() {
187 return meta1;
188 }
189
190 HRegionInfo getMeta2() {
191 return meta2;
192 }
193 }
194
195
196
197
198 private void mergeTwoRegions() throws IOException {
199 LOG.info("Merging regions " + Bytes.toStringBinary(this.region1) + " and " +
200 Bytes.toStringBinary(this.region2) + " in table " + Bytes.toString(this.tableName));
201
202
203 MetaScannerListener listener = new MetaScannerListener(region1, region2);
204 this.utils.scanRootRegion(listener);
205 HRegionInfo meta1 = listener.getMeta1();
206 if (meta1 == null) {
207 throw new IOException("Could not find meta region for " + Bytes.toStringBinary(region1));
208 }
209 HRegionInfo meta2 = listener.getMeta2();
210 if (meta2 == null) {
211 throw new IOException("Could not find meta region for " + Bytes.toStringBinary(region2));
212 }
213 LOG.info("Found meta for region1 " + Bytes.toStringBinary(meta1.getRegionName()) +
214 ", meta for region2 " + Bytes.toStringBinary(meta2.getRegionName()));
215 HRegion metaRegion1 = this.utils.getMetaRegion(meta1);
216 Get get = new Get(region1);
217 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
218 List<KeyValue> cells1 = metaRegion1.get(get, null).list();
219 HRegionInfo info1 = Writables.getHRegionInfo((cells1 == null)? null: cells1.get(0).getValue());
220 if (info1== null) {
221 throw new NullPointerException("info1 is null using key " +
222 Bytes.toStringBinary(region1) + " in " + meta1);
223 }
224
225 HRegion metaRegion2;
226 if (Bytes.equals(meta1.getRegionName(), meta2.getRegionName())) {
227 metaRegion2 = metaRegion1;
228 } else {
229 metaRegion2 = utils.getMetaRegion(meta2);
230 }
231 get = new Get(region2);
232 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
233 List<KeyValue> cells2 = metaRegion2.get(get, null).list();
234 HRegionInfo info2 = Writables.getHRegionInfo((cells2 == null)? null: cells2.get(0).getValue());
235 if (info2 == null) {
236 throw new NullPointerException("info2 is null using key " + meta2);
237 }
238 HRegion merged = merge(info1, metaRegion1, info2, metaRegion2);
239
240
241
242 listener = new MetaScannerListener(merged.getRegionName(), null);
243 utils.scanRootRegion(listener);
244 HRegionInfo mergedInfo = listener.getMeta1();
245 if (mergedInfo == null) {
246 throw new IOException("Could not find meta region for " +
247 Bytes.toStringBinary(merged.getRegionName()));
248 }
249 HRegion mergeMeta;
250 if (Bytes.equals(mergedInfo.getRegionName(), meta1.getRegionName())) {
251 mergeMeta = metaRegion1;
252 } else if (Bytes.equals(mergedInfo.getRegionName(), meta2.getRegionName())) {
253 mergeMeta = metaRegion2;
254 } else {
255 mergeMeta = utils.getMetaRegion(mergedInfo);
256 }
257 LOG.info("Adding " + merged.getRegionInfo() + " to " +
258 mergeMeta.getRegionInfo());
259
260 HRegion.addRegionToMETA(mergeMeta, merged);
261 merged.close();
262 }
263
264
265
266
267
268
269
270 private HRegion merge(HRegionInfo info1, HRegion meta1, HRegionInfo info2,
271 HRegion meta2)
272 throws IOException {
273 if (info1 == null) {
274 throw new IOException("Could not find " + Bytes.toStringBinary(region1) + " in " +
275 Bytes.toStringBinary(meta1.getRegionName()));
276 }
277 if (info2 == null) {
278 throw new IOException("Cound not find " + Bytes.toStringBinary(region2) + " in " +
279 Bytes.toStringBinary(meta2.getRegionName()));
280 }
281 HRegion merged = null;
282 HLog log = utils.getLog();
283 HRegion r1 = HRegion.openHRegion(info1, log, getConf());
284 try {
285 HRegion r2 = HRegion.openHRegion(info2, log, getConf());
286 try {
287 merged = HRegion.merge(r1, r2);
288 } finally {
289 if (!r2.isClosed()) {
290 r2.close();
291 }
292 }
293 } finally {
294 if (!r1.isClosed()) {
295 r1.close();
296 }
297 }
298
299
300
301
302 removeRegionFromMeta(meta1, info1);
303 removeRegionFromMeta(meta2, info2);
304
305 this.mergeInfo = merged.getRegionInfo();
306 return merged;
307 }
308
309
310
311
312
313
314
315
316
317
318 private void removeRegionFromMeta(HRegion meta, HRegionInfo regioninfo)
319 throws IOException {
320 if (LOG.isDebugEnabled()) {
321 LOG.debug("Removing region: " + regioninfo + " from " + meta);
322 }
323
324 Delete delete = new Delete(regioninfo.getRegionName(),
325 System.currentTimeMillis(), null);
326 meta.delete(delete, null, true);
327 }
328
329
330
331
332
333
334
335
336
337
338 private int parseArgs(String[] args) throws IOException {
339 GenericOptionsParser parser =
340 new GenericOptionsParser(getConf(), args);
341
342 String[] remainingArgs = parser.getRemainingArgs();
343 if (remainingArgs.length != 3) {
344 usage();
345 return -1;
346 }
347 tableName = Bytes.toBytes(remainingArgs[0]);
348 isMetaTable = Bytes.compareTo(tableName, HConstants.META_TABLE_NAME) == 0;
349
350 region1 = Bytes.toBytesBinary(remainingArgs[1]);
351 region2 = Bytes.toBytesBinary(remainingArgs[2]);
352 int status = 0;
353 if (notInTable(tableName, region1) || notInTable(tableName, region2)) {
354 status = -1;
355 } else if (Bytes.equals(region1, region2)) {
356 LOG.error("Can't merge a region with itself");
357 status = -1;
358 }
359 return status;
360 }
361
362 private boolean notInTable(final byte [] tn, final byte [] rn) {
363 if (WritableComparator.compareBytes(tn, 0, tn.length, rn, 0, tn.length) != 0) {
364 LOG.error("Region " + Bytes.toStringBinary(rn) + " does not belong to table " +
365 Bytes.toString(tn));
366 return true;
367 }
368 return false;
369 }
370
371 private void usage() {
372 System.err.println(
373 "Usage: bin/hbase merge <table-name> <region-1> <region-2>\n");
374 }
375
376 public static void main(String[] args) {
377 int status;
378 try {
379 status = ToolRunner.run(HBaseConfiguration.create(), new Merge(), args);
380 } catch (Exception e) {
381 LOG.error("exiting due to error", e);
382 status = -1;
383 }
384 System.exit(status);
385 }
386 }