1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.util;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.conf.Configuration;
26 import org.apache.hadoop.fs.FileSystem;
27 import org.apache.hadoop.fs.Path;
28 import org.apache.hadoop.hbase.HBaseConfiguration;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.client.Delete;
32 import org.apache.hadoop.hbase.client.Get;
33 import org.apache.hadoop.hbase.client.HTable;
34 import org.apache.hadoop.hbase.client.Put;
35 import org.apache.hadoop.hbase.client.Result;
36 import org.apache.hadoop.hbase.client.Scan;
37 import org.apache.hadoop.hbase.regionserver.HRegion;
38 import org.apache.hadoop.hbase.regionserver.InternalScanner;
39 import org.apache.hadoop.hbase.regionserver.Store;
40 import org.apache.hadoop.hbase.regionserver.wal.HLog;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.KeyValue;
43
44 import java.io.IOException;
45 import java.util.ArrayList;
46 import java.util.Collections;
47 import java.util.List;
48 import java.util.Map;
49 import java.util.TreeMap;
50
51
52
53
54
55
56
57
58 public class MetaUtils {
59 private static final Log LOG = LogFactory.getLog(MetaUtils.class);
60 private final Configuration conf;
61 private FileSystem fs;
62 private Path rootdir;
63 private HLog log;
64 private HRegion rootRegion;
65 private Map<byte [], HRegion> metaRegions = Collections.synchronizedSortedMap(
66 new TreeMap<byte [], HRegion>(Bytes.BYTES_COMPARATOR));
67
68
69
70
71 public MetaUtils() throws IOException {
72 this(HBaseConfiguration.create());
73 }
74
75
76
77
78
79 public MetaUtils(Configuration conf) throws IOException {
80 this.conf = conf;
81 conf.setInt("hbase.client.retries.number", 1);
82 this.rootRegion = null;
83 initialize();
84 }
85
86
87
88
89
90 private void initialize() throws IOException {
91 this.fs = FileSystem.get(this.conf);
92
93 this.rootdir = FSUtils.getRootDir(this.conf);
94 }
95
96
97
98
99
100 public synchronized HLog getLog() throws IOException {
101 if (this.log == null) {
102 Path logdir = new Path(this.fs.getHomeDirectory(),
103 HConstants.HREGION_LOGDIR_NAME + "_" + System.currentTimeMillis());
104 Path oldLogDir = new Path(this.fs.getHomeDirectory(),
105 HConstants.HREGION_OLDLOGDIR_NAME);
106 this.log = new HLog(this.fs, logdir, oldLogDir, this.conf, null);
107 }
108 return this.log;
109 }
110
111
112
113
114
115 public HRegion getRootRegion() throws IOException {
116 if (this.rootRegion == null) {
117 openRootRegion();
118 }
119 return this.rootRegion;
120 }
121
122
123
124
125
126
127
128
129 public HRegion getMetaRegion(HRegionInfo metaInfo) throws IOException {
130 HRegion meta = metaRegions.get(metaInfo.getRegionName());
131 if (meta == null) {
132 meta = openMetaRegion(metaInfo);
133 LOG.info("OPENING META " + meta.toString());
134 this.metaRegions.put(metaInfo.getRegionName(), meta);
135 }
136 return meta;
137 }
138
139
140
141
142
143
144 public void shutdown() {
145 if (this.rootRegion != null) {
146 try {
147 this.rootRegion.close();
148 } catch (IOException e) {
149 LOG.error("closing root region", e);
150 } finally {
151 this.rootRegion = null;
152 }
153 }
154 try {
155 for (HRegion r: metaRegions.values()) {
156 LOG.info("CLOSING META " + r.toString());
157 r.close();
158 }
159 } catch (IOException e) {
160 LOG.error("closing meta region", e);
161 } finally {
162 metaRegions.clear();
163 }
164 try {
165 if (this.log != null) {
166 this.log.rollWriter();
167 this.log.closeAndDelete();
168 }
169 } catch (IOException e) {
170 LOG.error("closing HLog", e);
171 } finally {
172 this.log = null;
173 }
174 }
175
176
177
178
179
180 public interface ScannerListener {
181
182
183
184
185
186
187
188 public boolean processRow(HRegionInfo info) throws IOException;
189 }
190
191
192
193
194
195
196
197
198 public void scanRootRegion(ScannerListener listener) throws IOException {
199
200 if (this.rootRegion == null) {
201 openRootRegion();
202 }
203 scanMetaRegion(this.rootRegion, listener);
204 }
205
206
207
208
209
210
211
212
213 public void scanMetaRegion(final HRegion r, final ScannerListener listener)
214 throws IOException {
215 Scan scan = new Scan();
216 scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
217 InternalScanner s = r.getScanner(scan);
218 try {
219 List<KeyValue> results = new ArrayList<KeyValue>();
220 boolean hasNext = true;
221 do {
222 hasNext = s.next(results);
223 HRegionInfo info = null;
224 for (KeyValue kv: results) {
225 info = Writables.getHRegionInfoOrNull(kv.getValue());
226 if (info == null) {
227 LOG.warn("Region info is null for row " +
228 Bytes.toString(kv.getRow()) + " in table " +
229 r.getTableDesc().getNameAsString());
230 }
231 continue;
232 }
233 if (!listener.processRow(info)) {
234 break;
235 }
236 results.clear();
237 } while (hasNext);
238 } finally {
239 s.close();
240 }
241 }
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257 public void scanMetaRegion(HRegionInfo metaRegionInfo,
258 ScannerListener listener)
259 throws IOException {
260
261 HRegion metaRegion = openMetaRegion(metaRegionInfo);
262 scanMetaRegion(metaRegion, listener);
263 }
264
265 private synchronized HRegion openRootRegion() throws IOException {
266 if (this.rootRegion != null) {
267 return this.rootRegion;
268 }
269 this.rootRegion = HRegion.openHRegion(HRegionInfo.ROOT_REGIONINFO,
270 this.rootdir, getLog(), this.conf);
271 this.rootRegion.compactStores();
272 return this.rootRegion;
273 }
274
275 private HRegion openMetaRegion(HRegionInfo metaInfo) throws IOException {
276 HRegion meta =
277 HRegion.openHRegion(metaInfo, this.rootdir, getLog(), this.conf);
278 meta.compactStores();
279 return meta;
280 }
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295 public static void changeOnlineStatus (final Configuration c,
296 final byte [] row, final boolean onlineOffline)
297 throws IOException {
298 HTable t = new HTable(c, HConstants.META_TABLE_NAME);
299 Get get = new Get(row);
300 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
301 Result res = t.get(get);
302 KeyValue [] kvs = res.raw();
303 if(kvs.length <= 0) {
304 throw new IOException("no information for row " + Bytes.toString(row));
305 }
306 byte [] value = kvs[0].getValue();
307 if (value == null) {
308 throw new IOException("no information for row " + Bytes.toString(row));
309 }
310 HRegionInfo info = Writables.getHRegionInfo(value);
311 Put put = new Put(row);
312 info.setOffline(onlineOffline);
313 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
314 Writables.getBytes(info));
315 t.put(put);
316
317 Delete delete = new Delete(row);
318 delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
319 delete.deleteColumns(HConstants.CATALOG_FAMILY,
320 HConstants.STARTCODE_QUALIFIER);
321
322 t.delete(delete);
323 }
324
325
326
327
328
329
330
331
332 public void addColumn(final byte [] tableName,
333 final HColumnDescriptor hcd)
334 throws IOException {
335 List<HRegionInfo> metas = getMETARows(tableName);
336 for (HRegionInfo hri: metas) {
337 final HRegion m = getMetaRegion(hri);
338 scanMetaRegion(m, new ScannerListener() {
339 private boolean inTable = true;
340
341 @SuppressWarnings("synthetic-access")
342 public boolean processRow(HRegionInfo info) throws IOException {
343 LOG.debug("Testing " + Bytes.toString(tableName) + " against " +
344 Bytes.toString(info.getTableDesc().getName()));
345 if (Bytes.equals(info.getTableDesc().getName(), tableName)) {
346 this.inTable = false;
347 info.getTableDesc().addFamily(hcd);
348 updateMETARegionInfo(m, info);
349 return true;
350 }
351
352
353
354 return this.inTable;
355 }});
356 }
357 }
358
359
360
361
362
363
364
365
366 public void deleteColumn(final byte [] tableName,
367 final byte [] columnFamily) throws IOException {
368 List<HRegionInfo> metas = getMETARows(tableName);
369 for (HRegionInfo hri: metas) {
370 final HRegion m = getMetaRegion(hri);
371 scanMetaRegion(m, new ScannerListener() {
372 private boolean inTable = true;
373
374 @SuppressWarnings("synthetic-access")
375 public boolean processRow(HRegionInfo info) throws IOException {
376 if (Bytes.equals(info.getTableDesc().getName(), tableName)) {
377 this.inTable = false;
378 info.getTableDesc().removeFamily(columnFamily);
379 updateMETARegionInfo(m, info);
380 Path tabledir = new Path(rootdir,
381 info.getTableDesc().getNameAsString());
382 Path p = Store.getStoreHomedir(tabledir, info.getEncodedName(),
383 columnFamily);
384 if (!fs.delete(p, true)) {
385 LOG.warn("Failed delete of " + p);
386 }
387 return false;
388 }
389
390
391
392 return this.inTable;
393 }});
394 }
395 }
396
397
398
399
400
401
402
403
404 public void updateMETARegionInfo(HRegion r, final HRegionInfo hri)
405 throws IOException {
406 if (LOG.isDebugEnabled()) {
407 Get get = new Get(hri.getRegionName());
408 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
409 Result res = r.get(get, null);
410 KeyValue [] kvs = res.raw();
411 if(kvs.length <= 0) {
412 return;
413 }
414 byte [] value = kvs[0].getValue();
415 if (value == null) {
416 return;
417 }
418 HRegionInfo h = Writables.getHRegionInfoOrNull(value);
419
420 LOG.debug("Old " + Bytes.toString(HConstants.CATALOG_FAMILY) + ":" +
421 Bytes.toString(HConstants.REGIONINFO_QUALIFIER) + " for " +
422 hri.toString() + " in " + r.toString() + " is: " + h.toString());
423 }
424
425 Put put = new Put(hri.getRegionName());
426 put.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
427 Writables.getBytes(hri));
428 r.put(put);
429
430 if (LOG.isDebugEnabled()) {
431 Get get = new Get(hri.getRegionName());
432 get.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
433 Result res = r.get(get, null);
434 KeyValue [] kvs = res.raw();
435 if(kvs.length <= 0) {
436 return;
437 }
438 byte [] value = kvs[0].getValue();
439 if (value == null) {
440 return;
441 }
442 HRegionInfo h = Writables.getHRegionInfoOrNull(value);
443 LOG.debug("New " + Bytes.toString(HConstants.CATALOG_FAMILY) + ":" +
444 Bytes.toString(HConstants.REGIONINFO_QUALIFIER) + " for " +
445 hri.toString() + " in " + r.toString() + " is: " + h.toString());
446 }
447 }
448
449
450
451
452
453
454
455
456 public List<HRegionInfo> getMETARows(final byte [] tableName)
457 throws IOException {
458 final List<HRegionInfo> result = new ArrayList<HRegionInfo>();
459
460 if (Bytes.equals(HConstants.META_TABLE_NAME, tableName)) {
461 result.add(openRootRegion().getRegionInfo());
462 return result;
463 }
464
465 scanRootRegion(new ScannerListener() {
466 private final Log SL_LOG = LogFactory.getLog(this.getClass());
467
468 public boolean processRow(HRegionInfo info) throws IOException {
469 SL_LOG.debug("Testing " + info);
470 if (Bytes.equals(info.getTableDesc().getName(),
471 HConstants.META_TABLE_NAME)) {
472 result.add(info);
473 return false;
474 }
475 return true;
476 }});
477 return result;
478 }
479
480
481
482
483
484 public static boolean isMetaTableName(final byte [] n) {
485 return Bytes.equals(n, HConstants.ROOT_TABLE_NAME) ||
486 Bytes.equals(n, HConstants.META_TABLE_NAME);
487 }
488 }