1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.client;
22
23 import java.io.Closeable;
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.NavigableMap;
28 import java.util.TreeMap;
29 import java.util.TreeSet;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.ServerName;
37 import org.apache.hadoop.hbase.TableNotFoundException;
38 import org.apache.hadoop.hbase.errorhandling.TimeoutException;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.apache.hadoop.hbase.util.Writables;
41
42
43
44
45
46
47
48
49
50
51
52
53
54 public class MetaScanner {
55 private static final Log LOG = LogFactory.getLog(MetaScanner.class);
56
57
58
59
60
61
62
63
64 public static void metaScan(Configuration configuration,
65 MetaScannerVisitor visitor)
66 throws IOException {
67 metaScan(configuration, null, visitor, null);
68 }
69
70
71
72
73
74
75
76
77
78
79 public static void metaScan(Configuration configuration, HConnection connection,
80 MetaScannerVisitor visitor, byte [] userTableName)
81 throws IOException {
82 metaScan(configuration, connection, visitor, userTableName, null, Integer.MAX_VALUE,
83 HConstants.META_TABLE_NAME);
84 }
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101 public static void metaScan(Configuration configuration,
102 MetaScannerVisitor visitor, byte [] userTableName, byte[] row,
103 int rowLimit)
104 throws IOException {
105 metaScan(configuration, null, visitor, userTableName, row, rowLimit,
106 HConstants.META_TABLE_NAME);
107 }
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126 public static void metaScan(Configuration configuration, HConnection connection,
127 MetaScannerVisitor visitor, byte [] tableName, byte[] row,
128 int rowLimit, final byte [] metaTableName)
129 throws IOException {
130 HTable metaTable = null;
131 try {
132 if (connection == null) {
133 metaTable = new HTable(configuration, HConstants.META_TABLE_NAME, null);
134 } else {
135 metaTable = new HTable(HConstants.META_TABLE_NAME, connection, null);
136 }
137 int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
138
139
140
141 byte[] startRow;
142 if (row != null) {
143
144 assert tableName != null;
145 byte[] searchRow =
146 HRegionInfo.createRegionName(tableName, row, HConstants.NINES,
147 false);
148 Result startRowResult = metaTable.getRowOrBefore(searchRow,
149 HConstants.CATALOG_FAMILY);
150 if (startRowResult == null) {
151 throw new TableNotFoundException("Cannot find row in .META. for table: "
152 + Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow));
153 }
154 byte[] value = startRowResult.getValue(HConstants.CATALOG_FAMILY,
155 HConstants.REGIONINFO_QUALIFIER);
156 if (value == null || value.length == 0) {
157 throw new IOException("HRegionInfo was null or empty in Meta for " +
158 Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow));
159 }
160 HRegionInfo regionInfo = Writables.getHRegionInfo(value);
161
162 byte[] rowBefore = regionInfo.getStartKey();
163 startRow = HRegionInfo.createRegionName(tableName, rowBefore,
164 HConstants.ZEROES, false);
165 } else if (tableName == null || tableName.length == 0) {
166
167 startRow = HConstants.EMPTY_START_ROW;
168 } else {
169
170 startRow = HRegionInfo.createRegionName(
171 tableName, HConstants.EMPTY_START_ROW, HConstants.ZEROES, false);
172 }
173
174
175 ScannerCallable callable;
176 int rows = Math.min(rowLimit, configuration.getInt(
177 HConstants.HBASE_META_SCANNER_CACHING,
178 HConstants.DEFAULT_HBASE_META_SCANNER_CACHING));
179 do {
180 final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
181 if (LOG.isDebugEnabled()) {
182 LOG.debug("Scanning " + Bytes.toString(metaTableName) +
183 " starting at row=" + Bytes.toStringBinary(startRow) + " for max=" +
184 rowUpperLimit + " rows using " + metaTable.getConnection().toString());
185 }
186 callable = new ScannerCallable(metaTable.getConnection(), metaTableName, scan, null);
187
188 callable.withRetries();
189
190 int processedRows = 0;
191 try {
192 callable.setCaching(rows);
193 done: do {
194 if (processedRows >= rowUpperLimit) {
195 break;
196 }
197
198 Result [] rrs = callable.withRetries();
199 if (rrs == null || rrs.length == 0 || rrs[0].size() == 0) {
200 break;
201 }
202 for (Result rr : rrs) {
203 if (processedRows >= rowUpperLimit) {
204 break done;
205 }
206 if (!visitor.processRow(rr))
207 break done;
208 processedRows++;
209 }
210
211 } while(true);
212
213 startRow = callable.getHRegionInfo().getEndKey();
214 } finally {
215
216 callable.setClose();
217 callable.withRetries();
218 }
219 } while (Bytes.compareTo(startRow, HConstants.LAST_ROW) != 0);
220 } finally {
221 visitor.close();
222 if (metaTable != null) {
223 metaTable.close();
224 }
225 }
226 }
227
228
229
230
231
232
233
234
235
236
237
238 public static List<HRegionInfo> listAllRegions(Configuration conf, final boolean offlined)
239 throws IOException {
240 final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
241 MetaScannerVisitor visitor = new BlockingMetaScannerVisitor(conf) {
242 @Override
243 public boolean processRowInternal(Result result) throws IOException {
244 if (result == null || result.isEmpty()) {
245 return true;
246 }
247 byte [] bytes = result.getValue(HConstants.CATALOG_FAMILY,
248 HConstants.REGIONINFO_QUALIFIER);
249 if (bytes == null) {
250 LOG.warn("Null REGIONINFO_QUALIFIER: " + result);
251 return true;
252 }
253 HRegionInfo regionInfo = Writables.getHRegionInfo(bytes);
254
255 if (regionInfo.isOffline() && !offlined) return true;
256 regions.add(regionInfo);
257 return true;
258 }
259 };
260 metaScan(conf, visitor);
261 return regions;
262 }
263
264
265
266
267
268
269
270
271
272
273
274
275 @Deprecated
276 public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
277 final byte[] tablename, final boolean offlined) throws IOException {
278 return allTableRegions(conf, null, tablename, offlined);
279 }
280
281
282
283
284
285
286
287
288
289 public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
290 HConnection connection, final byte[] tablename, final boolean offlined) throws IOException {
291 final NavigableMap<HRegionInfo, ServerName> regions =
292 new TreeMap<HRegionInfo, ServerName>();
293 MetaScannerVisitor visitor = new TableMetaScannerVisitor(conf, tablename) {
294 @Override
295 public boolean processRowInternal(Result rowResult) throws IOException {
296 HRegionInfo info = Writables.getHRegionInfo(
297 rowResult.getValue(HConstants.CATALOG_FAMILY,
298 HConstants.REGIONINFO_QUALIFIER));
299 byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
300 HConstants.SERVER_QUALIFIER);
301 String hostAndPort = null;
302 if (value != null && value.length > 0) {
303 hostAndPort = Bytes.toString(value);
304 }
305 value = rowResult.getValue(HConstants.CATALOG_FAMILY,
306 HConstants.STARTCODE_QUALIFIER);
307 long startcode = -1L;
308 if (value != null && value.length > 0) startcode = Bytes.toLong(value);
309 if (!(info.isOffline() || info.isSplit())) {
310 ServerName sn = null;
311 if (hostAndPort != null && hostAndPort.length() > 0) {
312 sn = new ServerName(hostAndPort, startcode);
313 }
314 regions.put(new UnmodifyableHRegionInfo(info), sn);
315 }
316 return true;
317 }
318 };
319 metaScan(conf, connection, visitor, tablename);
320 return regions;
321 }
322
323
324
325
326 public interface MetaScannerVisitor extends Closeable {
327
328
329
330
331
332
333
334
335
336 public boolean processRow(Result rowResult) throws IOException;
337 }
338
339 public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor {
340 @Override
341 public void close() throws IOException {
342 }
343 }
344
345
346
347
348
349
350
351 public static abstract class BlockingMetaScannerVisitor
352 extends MetaScannerVisitorBase {
353
354 private static final int DEFAULT_BLOCKING_TIMEOUT = 10000;
355 private Configuration conf;
356 private TreeSet<byte[]> daughterRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
357 private int blockingTimeout;
358 private HTable metaTable;
359
360 public BlockingMetaScannerVisitor(Configuration conf) {
361 this.conf = conf;
362 this.blockingTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
363 DEFAULT_BLOCKING_TIMEOUT);
364 }
365
366 public abstract boolean processRowInternal(Result rowResult) throws IOException;
367
368 @Override
369 public void close() throws IOException {
370 super.close();
371 if (metaTable != null) {
372 metaTable.close();
373 metaTable = null;
374 }
375 }
376
377 public HTable getMetaTable() throws IOException {
378 if (metaTable == null) {
379 metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
380 }
381 return metaTable;
382 }
383
384 @Override
385 public boolean processRow(Result rowResult) throws IOException {
386 HRegionInfo info = Writables.getHRegionInfoOrNull(
387 rowResult.getValue(HConstants.CATALOG_FAMILY,
388 HConstants.REGIONINFO_QUALIFIER));
389 if (info == null) {
390 return true;
391 }
392
393 if (daughterRegions.remove(info.getRegionName())) {
394 return true;
395 }
396
397 if (info.isSplitParent()) {
398
399
400
401
402 HRegionInfo splitA = Writables.getHRegionInfoOrNull(rowResult.getValue(HConstants.CATALOG_FAMILY,
403 HConstants.SPLITA_QUALIFIER));
404 HRegionInfo splitB = Writables.getHRegionInfoOrNull(rowResult.getValue(HConstants.CATALOG_FAMILY,
405 HConstants.SPLITB_QUALIFIER));
406
407 HTable metaTable = getMetaTable();
408 long start = System.currentTimeMillis();
409 if (splitA != null) {
410 try {
411 Result resultA = getRegionResultBlocking(metaTable, blockingTimeout,
412 info.getRegionName(), splitA.getRegionName());
413 if (resultA != null) {
414 processRow(resultA);
415 daughterRegions.add(splitA.getRegionName());
416 }
417
418 } catch (TimeoutException e) {
419 throw new RegionOfflineException("Split daughter region " +
420 splitA.getRegionNameAsString() + " cannot be found in META. Parent:" +
421 info.getRegionNameAsString());
422 }
423 }
424 long rem = blockingTimeout - (System.currentTimeMillis() - start);
425
426 if (splitB != null) {
427 try {
428 Result resultB = getRegionResultBlocking(metaTable, rem,
429 info.getRegionName(), splitB.getRegionName());
430 if (resultB != null) {
431 processRow(resultB);
432 daughterRegions.add(splitB.getRegionName());
433 }
434
435 } catch (TimeoutException e) {
436 throw new RegionOfflineException("Split daughter region " +
437 splitB.getRegionNameAsString() + " cannot be found in META. Parent:" +
438 info.getRegionNameAsString());
439 }
440 }
441 }
442
443 return processRowInternal(rowResult);
444 }
445
446
447
448
449
450
451
452
453 private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] parentRegionName, byte[] regionName)
454 throws IOException, TimeoutException {
455 boolean logged = false;
456 long start = System.currentTimeMillis();
457 while (System.currentTimeMillis() - start < timeout) {
458 Get get = new Get(regionName);
459 Result result = metaTable.get(get);
460 HRegionInfo info = Writables.getHRegionInfoOrNull(
461 result.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
462 if (info != null) {
463 return result;
464 }
465
466
467 Get parentGet = new Get(parentRegionName);
468 Result parentResult = metaTable.get(parentGet);
469 HRegionInfo parentInfo = Writables.getHRegionInfoOrNull(
470 parentResult.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
471 if (parentInfo == null) {
472
473 return null;
474 }
475
476 try {
477 if (!logged) {
478 if (LOG.isDebugEnabled()) {
479 LOG.debug("blocking until region is in META: " + Bytes.toStringBinary(regionName));
480 }
481 logged = true;
482 }
483 Thread.sleep(10);
484 } catch (InterruptedException ex) {
485 Thread.currentThread().interrupt();
486 break;
487 }
488 }
489 throw new TimeoutException("getRegionResultBlocking", start, System.currentTimeMillis(),
490 timeout);
491 }
492 }
493
494
495
496
497
498
499
500 public static abstract class TableMetaScannerVisitor extends BlockingMetaScannerVisitor {
501 private byte[] tableName;
502
503 public TableMetaScannerVisitor(Configuration conf, byte[] tableName) {
504 super(conf);
505 this.tableName = tableName;
506 }
507
508 @Override
509 public final boolean processRow(Result rowResult) throws IOException {
510 HRegionInfo info = Writables.getHRegionInfoOrNull(
511 rowResult.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
512 if (info == null) {
513 return true;
514 }
515 if (!(Bytes.equals(info.getTableName(), tableName))) {
516 return false;
517 }
518 return super.processRow(rowResult);
519 }
520
521 }
522 }