1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.client;
22
23 import java.io.Closeable;
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.NavigableMap;
28 import java.util.TreeMap;
29 import java.util.TreeSet;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.ServerName;
37 import org.apache.hadoop.hbase.TableNotFoundException;
38 import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
39 import org.apache.hadoop.hbase.errorhandling.TimeoutException;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.apache.hadoop.hbase.util.Writables;
42
43
44
45
46
47
48
49
50
51
52
53
54
55 public class MetaScanner {
56 private static final Log LOG = LogFactory.getLog(MetaScanner.class);
57
58
59
60
61
62
63
64
65 public static void metaScan(Configuration configuration,
66 MetaScannerVisitor visitor)
67 throws IOException {
68 metaScan(configuration, visitor, null);
69 }
70
71
72
73
74
75
76
77
78
79
80
81 public static void metaScan(Configuration configuration,
82 MetaScannerVisitor visitor, byte [] userTableName)
83 throws IOException {
84 metaScan(configuration, visitor, userTableName, null, Integer.MAX_VALUE);
85 }
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102 public static void metaScan(Configuration configuration,
103 MetaScannerVisitor visitor, byte [] userTableName, byte[] row,
104 int rowLimit)
105 throws IOException {
106 metaScan(configuration, visitor, userTableName, row, rowLimit,
107 HConstants.META_TABLE_NAME);
108 }
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126 public static void metaScan(Configuration configuration,
127 final MetaScannerVisitor visitor, final byte[] tableName,
128 final byte[] row, final int rowLimit, final byte[] metaTableName)
129 throws IOException {
130 try {
131 HConnectionManager.execute(new HConnectable<Void>(configuration) {
132 @Override
133 public Void connect(HConnection connection) throws IOException {
134 metaScan(conf, connection, visitor, tableName, row, rowLimit,
135 metaTableName);
136 return null;
137 }
138 });
139 } finally {
140 visitor.close();
141 }
142 }
143
144 private static void metaScan(Configuration configuration, HConnection connection,
145 MetaScannerVisitor visitor, byte [] tableName, byte[] row,
146 int rowLimit, final byte [] metaTableName)
147 throws IOException {
148 int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
149
150
151
152 byte[] startRow;
153 if (row != null) {
154
155 assert tableName != null;
156 byte[] searchRow =
157 HRegionInfo.createRegionName(tableName, row, HConstants.NINES,
158 false);
159 HTable metaTable = null;
160 try {
161 metaTable = new HTable(configuration, HConstants.META_TABLE_NAME);
162 Result startRowResult = metaTable.getRowOrBefore(searchRow,
163 HConstants.CATALOG_FAMILY);
164 if (startRowResult == null) {
165 throw new TableNotFoundException("Cannot find row in .META. for table: "
166 + Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow));
167 }
168 byte[] value = startRowResult.getValue(HConstants.CATALOG_FAMILY,
169 HConstants.REGIONINFO_QUALIFIER);
170 if (value == null || value.length == 0) {
171 throw new IOException("HRegionInfo was null or empty in Meta for " +
172 Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow));
173 }
174 HRegionInfo regionInfo = Writables.getHRegionInfo(value);
175
176 byte[] rowBefore = regionInfo.getStartKey();
177 startRow = HRegionInfo.createRegionName(tableName, rowBefore,
178 HConstants.ZEROES, false);
179 } finally {
180 if (metaTable != null) {
181 metaTable.close();
182 }
183 }
184 } else if (tableName == null || tableName.length == 0) {
185
186 startRow = HConstants.EMPTY_START_ROW;
187 } else {
188
189 startRow = HRegionInfo.createRegionName(
190 tableName, HConstants.EMPTY_START_ROW, HConstants.ZEROES, false);
191 }
192
193
194 ScannerCallable callable;
195 int rows = Math.min(rowLimit, configuration.getInt(
196 HConstants.HBASE_META_SCANNER_CACHING,
197 HConstants.DEFAULT_HBASE_META_SCANNER_CACHING));
198 do {
199 final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
200 if (LOG.isDebugEnabled()) {
201 LOG.debug("Scanning " + Bytes.toString(metaTableName) +
202 " starting at row=" + Bytes.toStringBinary(startRow) + " for max=" +
203 rowUpperLimit + " rows using " + connection.toString());
204 }
205 callable = new ScannerCallable(connection, metaTableName, scan, null);
206
207 callable.withRetries();
208
209 int processedRows = 0;
210 try {
211 callable.setCaching(rows);
212 done: do {
213 if (processedRows >= rowUpperLimit) {
214 break;
215 }
216
217 Result [] rrs = callable.withRetries();
218 if (rrs == null || rrs.length == 0 || rrs[0].size() == 0) {
219 break;
220 }
221 for (Result rr : rrs) {
222 if (processedRows >= rowUpperLimit) {
223 break done;
224 }
225 if (!visitor.processRow(rr))
226 break done;
227 processedRows++;
228 }
229
230 } while(true);
231
232 startRow = callable.getHRegionInfo().getEndKey();
233 } finally {
234
235 callable.setClose();
236 callable.withRetries();
237 }
238 } while (Bytes.compareTo(startRow, HConstants.LAST_ROW) != 0);
239 }
240
241
242
243
244
245
246
247 public static List<HRegionInfo> listAllRegions(Configuration conf)
248 throws IOException {
249 return listAllRegions(conf, true);
250 }
251
252
253
254
255
256
257
258
259
260 public static List<HRegionInfo> listAllRegions(Configuration conf, final boolean offlined)
261 throws IOException {
262 final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
263 MetaScannerVisitor visitor = new BlockingMetaScannerVisitor(conf) {
264 @Override
265 public boolean processRowInternal(Result result) throws IOException {
266 if (result == null || result.isEmpty()) {
267 return true;
268 }
269 byte [] bytes = result.getValue(HConstants.CATALOG_FAMILY,
270 HConstants.REGIONINFO_QUALIFIER);
271 if (bytes == null) {
272 LOG.warn("Null REGIONINFO_QUALIFIER: " + result);
273 return true;
274 }
275 HRegionInfo regionInfo = Writables.getHRegionInfo(bytes);
276
277 if (regionInfo.isOffline() && !offlined) return true;
278 regions.add(regionInfo);
279 return true;
280 }
281 };
282 metaScan(conf, visitor);
283 return regions;
284 }
285
286
287
288
289
290
291
292
293
294 public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
295 final byte [] tablename, final boolean offlined) throws IOException {
296 final NavigableMap<HRegionInfo, ServerName> regions =
297 new TreeMap<HRegionInfo, ServerName>();
298 MetaScannerVisitor visitor = new TableMetaScannerVisitor(conf, tablename) {
299 @Override
300 public boolean processRowInternal(Result rowResult) throws IOException {
301 HRegionInfo info = Writables.getHRegionInfo(
302 rowResult.getValue(HConstants.CATALOG_FAMILY,
303 HConstants.REGIONINFO_QUALIFIER));
304 byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
305 HConstants.SERVER_QUALIFIER);
306 String hostAndPort = null;
307 if (value != null && value.length > 0) {
308 hostAndPort = Bytes.toString(value);
309 }
310 value = rowResult.getValue(HConstants.CATALOG_FAMILY,
311 HConstants.STARTCODE_QUALIFIER);
312 long startcode = -1L;
313 if (value != null && value.length > 0) startcode = Bytes.toLong(value);
314 if (!(info.isOffline() || info.isSplit())) {
315 ServerName sn = null;
316 if (hostAndPort != null && hostAndPort.length() > 0) {
317 sn = new ServerName(hostAndPort, startcode);
318 }
319 regions.put(new UnmodifyableHRegionInfo(info), sn);
320 }
321 return true;
322 }
323 };
324 metaScan(conf, visitor, tablename);
325 return regions;
326 }
327
328
329
330
331 public interface MetaScannerVisitor extends Closeable {
332
333
334
335
336
337
338
339
340
341 public boolean processRow(Result rowResult) throws IOException;
342 }
343
344 public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor {
345 @Override
346 public void close() throws IOException {
347 }
348 }
349
350
351
352
353
354
355
356 public static abstract class BlockingMetaScannerVisitor
357 extends MetaScannerVisitorBase {
358
359 private static final int DEFAULT_BLOCKING_TIMEOUT = 10000;
360 private Configuration conf;
361 private TreeSet<byte[]> daughterRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
362 private int blockingTimeout;
363 private HTable metaTable;
364
365 public BlockingMetaScannerVisitor(Configuration conf) {
366 this.conf = conf;
367 this.blockingTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
368 DEFAULT_BLOCKING_TIMEOUT);
369 }
370
371 public abstract boolean processRowInternal(Result rowResult) throws IOException;
372
373 @Override
374 public void close() throws IOException {
375 super.close();
376 if (metaTable != null) {
377 metaTable.close();
378 metaTable = null;
379 }
380 }
381
382 public HTable getMetaTable() throws IOException {
383 if (metaTable == null) {
384 metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
385 }
386 return metaTable;
387 }
388
389 @Override
390 public boolean processRow(Result rowResult) throws IOException {
391 HRegionInfo info = Writables.getHRegionInfoOrNull(
392 rowResult.getValue(HConstants.CATALOG_FAMILY,
393 HConstants.REGIONINFO_QUALIFIER));
394 if (info == null) {
395 return true;
396 }
397
398 if (daughterRegions.remove(info.getRegionName())) {
399 return true;
400 }
401
402 if (info.isSplitParent()) {
403
404
405
406
407 HRegionInfo splitA = Writables.getHRegionInfoOrNull(rowResult.getValue(HConstants.CATALOG_FAMILY,
408 HConstants.SPLITA_QUALIFIER));
409 HRegionInfo splitB = Writables.getHRegionInfoOrNull(rowResult.getValue(HConstants.CATALOG_FAMILY,
410 HConstants.SPLITB_QUALIFIER));
411
412 HTable metaTable = getMetaTable();
413 long start = System.currentTimeMillis();
414 if (splitA != null) {
415 try {
416 Result resultA = getRegionResultBlocking(metaTable, blockingTimeout,
417 info.getRegionName(), splitA.getRegionName());
418 if (resultA != null) {
419 processRow(resultA);
420 daughterRegions.add(splitA.getRegionName());
421 }
422
423 } catch (TimeoutException e) {
424 throw new RegionOfflineException("Split daughter region " +
425 splitA.getRegionNameAsString() + " cannot be found in META. Parent:" +
426 info.getRegionNameAsString());
427 }
428 }
429 long rem = blockingTimeout - (System.currentTimeMillis() - start);
430
431 if (splitB != null) {
432 try {
433 Result resultB = getRegionResultBlocking(metaTable, rem,
434 info.getRegionName(), splitB.getRegionName());
435 if (resultB != null) {
436 processRow(resultB);
437 daughterRegions.add(splitB.getRegionName());
438 }
439
440 } catch (TimeoutException e) {
441 throw new RegionOfflineException("Split daughter region " +
442 splitB.getRegionNameAsString() + " cannot be found in META. Parent:" +
443 info.getRegionNameAsString());
444 }
445 }
446 }
447
448 return processRowInternal(rowResult);
449 }
450
451
452
453
454
455
456
457
458 private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] parentRegionName, byte[] regionName)
459 throws IOException, TimeoutException {
460 boolean logged = false;
461 long start = System.currentTimeMillis();
462 while (System.currentTimeMillis() - start < timeout) {
463 Get get = new Get(regionName);
464 Result result = metaTable.get(get);
465 HRegionInfo info = Writables.getHRegionInfoOrNull(
466 result.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
467 if (info != null) {
468 return result;
469 }
470
471
472 Get parentGet = new Get(parentRegionName);
473 Result parentResult = metaTable.get(parentGet);
474 HRegionInfo parentInfo = Writables.getHRegionInfoOrNull(
475 parentResult.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
476 if (parentInfo == null) {
477
478 return null;
479 }
480
481 try {
482 if (!logged) {
483 if (LOG.isDebugEnabled()) {
484 LOG.debug("blocking until region is in META: " + Bytes.toStringBinary(regionName));
485 }
486 logged = true;
487 }
488 Thread.sleep(10);
489 } catch (InterruptedException ex) {
490 Thread.currentThread().interrupt();
491 break;
492 }
493 }
494 throw new TimeoutException("getRegionResultBlocking", start, System.currentTimeMillis(),
495 timeout);
496 }
497 }
498
499
500
501
502
503
504
505 public static abstract class TableMetaScannerVisitor extends BlockingMetaScannerVisitor {
506 private byte[] tableName;
507
508 public TableMetaScannerVisitor(Configuration conf, byte[] tableName) {
509 super(conf);
510 this.tableName = tableName;
511 }
512
513 @Override
514 public final boolean processRow(Result rowResult) throws IOException {
515 HRegionInfo info = Writables.getHRegionInfoOrNull(
516 rowResult.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
517 if (info == null) {
518 return true;
519 }
520 if (!(Bytes.equals(info.getTableName(), tableName))) {
521 return false;
522 }
523 return super.processRow(rowResult);
524 }
525
526 }
527 }