1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22 import org.apache.commons.logging.Log;
23 import org.apache.commons.logging.LogFactory;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.FileSystem;
26 import org.apache.hadoop.fs.Path;
27 import org.apache.hadoop.hbase.client.Delete;
28 import org.apache.hadoop.hbase.client.HConnection;
29 import org.apache.hadoop.hbase.client.HConnectionManager;
30 import org.apache.hadoop.hbase.client.HTable;
31 import org.apache.hadoop.hbase.client.Put;
32 import org.apache.hadoop.hbase.client.Result;
33 import org.apache.hadoop.hbase.client.ResultScanner;
34 import org.apache.hadoop.hbase.client.Scan;
35 import org.apache.hadoop.hbase.regionserver.HRegion;
36 import org.apache.hadoop.hbase.regionserver.InternalScanner;
37 import org.apache.hadoop.hbase.regionserver.wal.HLog;
38 import org.apache.hadoop.hbase.util.Bytes;
39 import org.apache.hadoop.hbase.util.Writables;
40
41 import java.io.IOException;
42 import java.util.ArrayList;
43 import java.util.List;
44 import java.util.NoSuchElementException;
45 import java.util.Random;
46
47
48
49
50
51 class HMerge {
52 static final Log LOG = LogFactory.getLog(HMerge.class);
53 static final Random rand = new Random();
54
55
56
57
58 private HMerge() {
59 super();
60 }
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75 public static void merge(Configuration conf, FileSystem fs,
76 final byte [] tableName)
77 throws IOException {
78 HConnection connection = HConnectionManager.getConnection(conf);
79 boolean masterIsRunning = connection.isMasterRunning();
80 HConnectionManager.deleteConnectionInfo(conf, false);
81 if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
82 if (masterIsRunning) {
83 throw new IllegalStateException(
84 "Can not compact META table if instance is on-line");
85 }
86 new OfflineMerger(conf, fs).process();
87 } else {
88 if(!masterIsRunning) {
89 throw new IllegalStateException(
90 "HBase instance must be running to merge a normal table");
91 }
92 new OnlineMerger(conf, fs, tableName).process();
93 }
94 }
95
96 private static abstract class Merger {
97 protected final Configuration conf;
98 protected final FileSystem fs;
99 protected final Path tabledir;
100 protected final HLog hlog;
101 private final long maxFilesize;
102
103
104 protected Merger(Configuration conf, FileSystem fs,
105 final byte [] tableName)
106 throws IOException {
107 this.conf = conf;
108 this.fs = fs;
109 this.maxFilesize = conf.getLong("hbase.hregion.max.filesize",
110 HConstants.DEFAULT_MAX_FILE_SIZE);
111
112 this.tabledir = new Path(
113 fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR))),
114 Bytes.toString(tableName)
115 );
116 Path logdir = new Path(tabledir, "merge_" + System.currentTimeMillis() +
117 HConstants.HREGION_LOGDIR_NAME);
118 Path oldLogDir = new Path(tabledir, HConstants.HREGION_OLDLOGDIR_NAME);
119 this.hlog =
120 new HLog(fs, logdir, oldLogDir, conf, null);
121 }
122
123 void process() throws IOException {
124 try {
125 for(HRegionInfo[] regionsToMerge = next();
126 regionsToMerge != null;
127 regionsToMerge = next()) {
128 if (!merge(regionsToMerge)) {
129 return;
130 }
131 }
132 } finally {
133 try {
134 hlog.closeAndDelete();
135
136 } catch(IOException e) {
137 LOG.error(e);
138 }
139 }
140 }
141
142 protected boolean merge(final HRegionInfo[] info) throws IOException {
143 if(info.length < 2) {
144 LOG.info("only one region - nothing to merge");
145 return false;
146 }
147
148 HRegion currentRegion = null;
149 long currentSize = 0;
150 HRegion nextRegion = null;
151 long nextSize = 0;
152 for (int i = 0; i < info.length - 1; i++) {
153 if (currentRegion == null) {
154 currentRegion =
155 HRegion.newHRegion(tabledir, hlog, fs, conf, info[i], null);
156 currentRegion.initialize();
157 currentSize = currentRegion.getLargestHStoreSize();
158 }
159 nextRegion =
160 HRegion.newHRegion(tabledir, hlog, fs, conf, info[i + 1], null);
161 nextRegion.initialize();
162 nextSize = nextRegion.getLargestHStoreSize();
163
164 if ((currentSize + nextSize) <= (maxFilesize / 2)) {
165
166
167 LOG.info("merging regions " + Bytes.toString(currentRegion.getRegionName())
168 + " and " + Bytes.toString(nextRegion.getRegionName()));
169 HRegion mergedRegion =
170 HRegion.mergeAdjacent(currentRegion, nextRegion);
171 updateMeta(currentRegion.getRegionName(), nextRegion.getRegionName(),
172 mergedRegion);
173 break;
174 }
175 LOG.info("not merging regions " + Bytes.toString(currentRegion.getRegionName())
176 + " and " + Bytes.toString(nextRegion.getRegionName()));
177 currentRegion.close();
178 currentRegion = nextRegion;
179 currentSize = nextSize;
180 }
181 if(currentRegion != null) {
182 currentRegion.close();
183 }
184 return true;
185 }
186
187 protected abstract HRegionInfo[] next() throws IOException;
188
189 protected abstract void updateMeta(final byte [] oldRegion1,
190 final byte [] oldRegion2, HRegion newRegion)
191 throws IOException;
192
193 }
194
195
196 private static class OnlineMerger extends Merger {
197 private final byte [] tableName;
198 private final HTable table;
199 private final ResultScanner metaScanner;
200 private HRegionInfo latestRegion;
201
202 OnlineMerger(Configuration conf, FileSystem fs,
203 final byte [] tableName)
204 throws IOException {
205 super(conf, fs, tableName);
206 this.tableName = tableName;
207 this.table = new HTable(conf, HConstants.META_TABLE_NAME);
208 this.metaScanner = table.getScanner(HConstants.CATALOG_FAMILY,
209 HConstants.REGIONINFO_QUALIFIER);
210 this.latestRegion = null;
211 }
212
213 private HRegionInfo nextRegion() throws IOException {
214 try {
215 Result results = getMetaRow();
216 if (results == null) {
217 return null;
218 }
219 byte[] regionInfoValue = results.getValue(HConstants.CATALOG_FAMILY,
220 HConstants.REGIONINFO_QUALIFIER);
221 if (regionInfoValue == null || regionInfoValue.length == 0) {
222 throw new NoSuchElementException("meta region entry missing " +
223 Bytes.toString(HConstants.CATALOG_FAMILY) + ":" +
224 Bytes.toString(HConstants.REGIONINFO_QUALIFIER));
225 }
226 HRegionInfo region = Writables.getHRegionInfo(regionInfoValue);
227 if (!Bytes.equals(region.getTableDesc().getName(), this.tableName)) {
228 return null;
229 }
230 checkOfflined(region);
231 return region;
232 } catch (IOException e) {
233 e = RemoteExceptionHandler.checkIOException(e);
234 LOG.error("meta scanner error", e);
235 metaScanner.close();
236 throw e;
237 }
238 }
239
240 protected void checkOfflined(final HRegionInfo hri)
241 throws TableNotDisabledException {
242 if (!hri.isOffline()) {
243 throw new TableNotDisabledException("Region " +
244 hri.getRegionNameAsString() + " is not disabled");
245 }
246 }
247
248
249
250
251
252
253 private Result getMetaRow() throws IOException {
254 Result currentRow = metaScanner.next();
255 boolean foundResult = false;
256 while (currentRow != null) {
257 LOG.info("Row: <" + Bytes.toString(currentRow.getRow()) + ">");
258 byte[] regionInfoValue = currentRow.getValue(HConstants.CATALOG_FAMILY,
259 HConstants.REGIONINFO_QUALIFIER);
260 if (regionInfoValue == null || regionInfoValue.length == 0) {
261 currentRow = metaScanner.next();
262 continue;
263 }
264 foundResult = true;
265 break;
266 }
267 return foundResult ? currentRow : null;
268 }
269
270 @Override
271 protected HRegionInfo[] next() throws IOException {
272 List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
273 if(latestRegion == null) {
274 latestRegion = nextRegion();
275 }
276 if(latestRegion != null) {
277 regions.add(latestRegion);
278 }
279 latestRegion = nextRegion();
280 if(latestRegion != null) {
281 regions.add(latestRegion);
282 }
283 return regions.toArray(new HRegionInfo[regions.size()]);
284 }
285
286 @Override
287 protected void updateMeta(final byte [] oldRegion1,
288 final byte [] oldRegion2,
289 HRegion newRegion)
290 throws IOException {
291 byte[][] regionsToDelete = {oldRegion1, oldRegion2};
292 for (int r = 0; r < regionsToDelete.length; r++) {
293 if(Bytes.equals(regionsToDelete[r], latestRegion.getRegionName())) {
294 latestRegion = null;
295 }
296 Delete delete = new Delete(regionsToDelete[r]);
297 table.delete(delete);
298 if(LOG.isDebugEnabled()) {
299 LOG.debug("updated columns in row: " + Bytes.toString(regionsToDelete[r]));
300 }
301 }
302 newRegion.getRegionInfo().setOffline(true);
303
304 Put put = new Put(newRegion.getRegionName());
305 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
306 Writables.getBytes(newRegion.getRegionInfo()));
307 table.put(put);
308
309 if(LOG.isDebugEnabled()) {
310 LOG.debug("updated columns in row: "
311 + Bytes.toString(newRegion.getRegionName()));
312 }
313 }
314 }
315
316
317 private static class OfflineMerger extends Merger {
318 private final List<HRegionInfo> metaRegions = new ArrayList<HRegionInfo>();
319 private final HRegion root;
320
321 OfflineMerger(Configuration conf, FileSystem fs)
322 throws IOException {
323 super(conf, fs, HConstants.META_TABLE_NAME);
324
325 Path rootTableDir = HTableDescriptor.getTableDir(
326 fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR))),
327 HConstants.ROOT_TABLE_NAME);
328
329
330
331 root = HRegion.newHRegion(rootTableDir, hlog, fs, conf,
332 HRegionInfo.ROOT_REGIONINFO, null);
333 root.initialize();
334
335 Scan scan = new Scan();
336 scan.addColumn(HConstants.CATALOG_FAMILY,
337 HConstants.REGIONINFO_QUALIFIER);
338 InternalScanner rootScanner =
339 root.getScanner(scan);
340
341 try {
342 List<KeyValue> results = new ArrayList<KeyValue>();
343 while(rootScanner.next(results)) {
344 for(KeyValue kv: results) {
345 HRegionInfo info = Writables.getHRegionInfoOrNull(kv.getValue());
346 if (info != null) {
347 metaRegions.add(info);
348 }
349 }
350 }
351 } finally {
352 rootScanner.close();
353 try {
354 root.close();
355
356 } catch(IOException e) {
357 LOG.error(e);
358 }
359 }
360 }
361
362 @Override
363 protected HRegionInfo[] next() {
364 HRegionInfo[] results = null;
365 if (metaRegions.size() > 0) {
366 results = metaRegions.toArray(new HRegionInfo[metaRegions.size()]);
367 metaRegions.clear();
368 }
369 return results;
370 }
371
372 @Override
373 protected void updateMeta(final byte [] oldRegion1,
374 final byte [] oldRegion2, HRegion newRegion)
375 throws IOException {
376 byte[][] regionsToDelete = {oldRegion1, oldRegion2};
377 for(int r = 0; r < regionsToDelete.length; r++) {
378 Delete delete = new Delete(regionsToDelete[r]);
379 delete.deleteColumns(HConstants.CATALOG_FAMILY,
380 HConstants.REGIONINFO_QUALIFIER);
381 delete.deleteColumns(HConstants.CATALOG_FAMILY,
382 HConstants.SERVER_QUALIFIER);
383 delete.deleteColumns(HConstants.CATALOG_FAMILY,
384 HConstants.STARTCODE_QUALIFIER);
385 delete.deleteColumns(HConstants.CATALOG_FAMILY,
386 HConstants.SPLITA_QUALIFIER);
387 delete.deleteColumns(HConstants.CATALOG_FAMILY,
388 HConstants.SPLITB_QUALIFIER);
389 root.delete(delete, null, true);
390
391 if(LOG.isDebugEnabled()) {
392 LOG.debug("updated columns in row: " + Bytes.toString(regionsToDelete[r]));
393 }
394 }
395 HRegionInfo newInfo = newRegion.getRegionInfo();
396 newInfo.setOffline(true);
397 Put put = new Put(newRegion.getRegionName());
398 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
399 Writables.getBytes(newInfo));
400 root.put(put);
401 if(LOG.isDebugEnabled()) {
402 LOG.debug("updated columns in row: " + Bytes.toString(newRegion.getRegionName()));
403 }
404 }
405 }
406 }