1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.List;
24 import java.util.NoSuchElementException;
25 import java.util.Random;
26
27 import org.apache.commons.logging.Log;
28 import org.apache.commons.logging.LogFactory;
29 import org.apache.hadoop.classification.InterfaceAudience;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HConstants;
34 import org.apache.hadoop.hbase.HRegionInfo;
35 import org.apache.hadoop.hbase.HTableDescriptor;
36 import org.apache.hadoop.hbase.KeyValue;
37 import org.apache.hadoop.hbase.RemoteExceptionHandler;
38 import org.apache.hadoop.hbase.exceptions.TableNotDisabledException;
39 import org.apache.hadoop.hbase.catalog.MetaEditor;
40 import org.apache.hadoop.hbase.client.Delete;
41 import org.apache.hadoop.hbase.client.HBaseAdmin;
42 import org.apache.hadoop.hbase.client.HConnection;
43 import org.apache.hadoop.hbase.client.HConnectionManager;
44 import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
45 import org.apache.hadoop.hbase.client.HTable;
46 import org.apache.hadoop.hbase.client.Put;
47 import org.apache.hadoop.hbase.client.Result;
48 import org.apache.hadoop.hbase.client.ResultScanner;
49 import org.apache.hadoop.hbase.client.Scan;
50 import org.apache.hadoop.hbase.regionserver.HRegion;
51 import org.apache.hadoop.hbase.regionserver.InternalScanner;
52 import org.apache.hadoop.hbase.regionserver.wal.HLog;
53 import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
54
55
56
57
58
59 @InterfaceAudience.Private
60 class HMerge {
61
62 static final Log LOG = LogFactory.getLog(HMerge.class);
63 static final Random rand = new Random();
64
65
66
67
68 private HMerge() {
69 super();
70 }
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85 public static void merge(Configuration conf, FileSystem fs,
86 final byte [] tableName)
87 throws IOException {
88 merge(conf, fs, tableName, true);
89 }
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106 public static void merge(Configuration conf, FileSystem fs,
107 final byte [] tableName, final boolean testMasterRunning)
108 throws IOException {
109 boolean masterIsRunning = false;
110 if (testMasterRunning) {
111 masterIsRunning = HConnectionManager
112 .execute(new HConnectable<Boolean>(conf) {
113 @Override
114 public Boolean connect(HConnection connection) throws IOException {
115 return connection.isMasterRunning();
116 }
117 });
118 }
119 if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
120 if (masterIsRunning) {
121 throw new IllegalStateException(
122 "Can not compact META table if instance is on-line");
123 }
124
125 } else {
126 if(!masterIsRunning) {
127 throw new IllegalStateException(
128 "HBase instance must be running to merge a normal table");
129 }
130 HBaseAdmin admin = new HBaseAdmin(conf);
131 if (!admin.isTableDisabled(tableName)) {
132 throw new TableNotDisabledException(tableName);
133 }
134 new OnlineMerger(conf, fs, tableName).process();
135 }
136 }
137
138 private static abstract class Merger {
139 protected final Configuration conf;
140 protected final FileSystem fs;
141 protected final Path rootDir;
142 protected final HTableDescriptor htd;
143 protected final HLog hlog;
144 private final long maxFilesize;
145
146
147 protected Merger(Configuration conf, FileSystem fs, final byte [] tableName)
148 throws IOException {
149 this.conf = conf;
150 this.fs = fs;
151 this.maxFilesize = conf.getLong(HConstants.HREGION_MAX_FILESIZE,
152 HConstants.DEFAULT_MAX_FILE_SIZE);
153
154 this.rootDir = FSUtils.getRootDir(conf);
155 Path tabledir = HTableDescriptor.getTableDir(this.rootDir, tableName);
156 this.htd = FSTableDescriptors.getTableDescriptor(this.fs, tabledir);
157 String logname = "merge_" + System.currentTimeMillis() + HConstants.HREGION_LOGDIR_NAME;
158
159 this.hlog = HLogFactory.createHLog(fs, tabledir, logname, conf);
160 }
161
162 void process() throws IOException {
163 try {
164 for (HRegionInfo[] regionsToMerge = next();
165 regionsToMerge != null;
166 regionsToMerge = next()) {
167 if (!merge(regionsToMerge)) {
168 return;
169 }
170 }
171 } finally {
172 try {
173 hlog.closeAndDelete();
174
175 } catch(IOException e) {
176 LOG.error(e);
177 }
178 }
179 }
180
181 protected boolean merge(final HRegionInfo[] info) throws IOException {
182 if (info.length < 2) {
183 LOG.info("only one region - nothing to merge");
184 return false;
185 }
186
187 HRegion currentRegion = null;
188 long currentSize = 0;
189 HRegion nextRegion = null;
190 long nextSize = 0;
191 for (int i = 0; i < info.length - 1; i++) {
192 if (currentRegion == null) {
193 currentRegion = HRegion.openHRegion(conf, fs, this.rootDir, info[i], this.htd, hlog);
194 currentSize = currentRegion.getLargestHStoreSize();
195 }
196 nextRegion = HRegion.openHRegion(conf, fs, this.rootDir, info[i + 1], this.htd, hlog);
197 nextSize = nextRegion.getLargestHStoreSize();
198
199 if ((currentSize + nextSize) <= (maxFilesize / 2)) {
200
201
202 LOG.info("Merging regions " + currentRegion.getRegionNameAsString() +
203 " and " + nextRegion.getRegionNameAsString());
204 HRegion mergedRegion =
205 HRegion.mergeAdjacent(currentRegion, nextRegion);
206 updateMeta(currentRegion.getRegionName(), nextRegion.getRegionName(),
207 mergedRegion);
208 break;
209 }
210 LOG.info("not merging regions " + Bytes.toStringBinary(currentRegion.getRegionName())
211 + " and " + Bytes.toStringBinary(nextRegion.getRegionName()));
212 currentRegion.close();
213 currentRegion = nextRegion;
214 currentSize = nextSize;
215 }
216 if(currentRegion != null) {
217 currentRegion.close();
218 }
219 return true;
220 }
221
222 protected abstract HRegionInfo[] next() throws IOException;
223
224 protected abstract void updateMeta(final byte [] oldRegion1,
225 final byte [] oldRegion2, HRegion newRegion)
226 throws IOException;
227
228 }
229
230
231 private static class OnlineMerger extends Merger {
232 private final byte [] tableName;
233 private final HTable table;
234 private final ResultScanner metaScanner;
235 private HRegionInfo latestRegion;
236
237 OnlineMerger(Configuration conf, FileSystem fs,
238 final byte [] tableName)
239 throws IOException {
240 super(conf, fs, tableName);
241 this.tableName = tableName;
242 this.table = new HTable(conf, HConstants.META_TABLE_NAME);
243 this.metaScanner = table.getScanner(HConstants.CATALOG_FAMILY,
244 HConstants.REGIONINFO_QUALIFIER);
245 this.latestRegion = null;
246 }
247
248 private HRegionInfo nextRegion() throws IOException {
249 try {
250 Result results = getMetaRow();
251 if (results == null) {
252 return null;
253 }
254 HRegionInfo region = HRegionInfo.getHRegionInfo(results);
255 if (region == null) {
256 throw new NoSuchElementException("meta region entry missing " +
257 Bytes.toString(HConstants.CATALOG_FAMILY) + ":" +
258 Bytes.toString(HConstants.REGIONINFO_QUALIFIER));
259 }
260 if (!Bytes.equals(region.getTableName(), this.tableName)) {
261 return null;
262 }
263 return region;
264 } catch (IOException e) {
265 e = RemoteExceptionHandler.checkIOException(e);
266 LOG.error("meta scanner error", e);
267 metaScanner.close();
268 throw e;
269 }
270 }
271
272
273
274
275
276
277 private Result getMetaRow() throws IOException {
278 Result currentRow = metaScanner.next();
279 boolean foundResult = false;
280 while (currentRow != null) {
281 LOG.info("Row: <" + Bytes.toStringBinary(currentRow.getRow()) + ">");
282 byte[] regionInfoValue = currentRow.getValue(HConstants.CATALOG_FAMILY,
283 HConstants.REGIONINFO_QUALIFIER);
284 if (regionInfoValue == null || regionInfoValue.length == 0) {
285 currentRow = metaScanner.next();
286 continue;
287 }
288 foundResult = true;
289 break;
290 }
291 return foundResult ? currentRow : null;
292 }
293
294 @Override
295 protected HRegionInfo[] next() throws IOException {
296 List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
297 if(latestRegion == null) {
298 latestRegion = nextRegion();
299 }
300 if(latestRegion != null) {
301 regions.add(latestRegion);
302 }
303 latestRegion = nextRegion();
304 if(latestRegion != null) {
305 regions.add(latestRegion);
306 }
307 return regions.toArray(new HRegionInfo[regions.size()]);
308 }
309
310 @Override
311 protected void updateMeta(final byte [] oldRegion1,
312 final byte [] oldRegion2,
313 HRegion newRegion)
314 throws IOException {
315 byte[][] regionsToDelete = {oldRegion1, oldRegion2};
316 for (int r = 0; r < regionsToDelete.length; r++) {
317 if(Bytes.equals(regionsToDelete[r], latestRegion.getRegionName())) {
318 latestRegion = null;
319 }
320 Delete delete = new Delete(regionsToDelete[r]);
321 table.delete(delete);
322 if(LOG.isDebugEnabled()) {
323 LOG.debug("updated columns in row: " + Bytes.toStringBinary(regionsToDelete[r]));
324 }
325 }
326 newRegion.getRegionInfo().setOffline(true);
327
328 MetaEditor.addRegionToMeta(table, newRegion.getRegionInfo());
329
330 if(LOG.isDebugEnabled()) {
331 LOG.debug("updated columns in row: "
332 + Bytes.toStringBinary(newRegion.getRegionName()));
333 }
334 }
335 }
336 }