1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21 package org.apache.hadoop.hbase.client;
22
23 import java.io.Closeable;
24 import java.io.IOException;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.NavigableMap;
28 import java.util.TreeMap;
29 import java.util.TreeSet;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.hbase.HConstants;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.ServerName;
37 import org.apache.hadoop.hbase.TableNotFoundException;
38 import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.apache.hadoop.hbase.util.Writables;
41
42
43
44
45
46
47
48
49
50
51
52
53
54 public class MetaScanner {
55 private static final Log LOG = LogFactory.getLog(MetaScanner.class);
56
57
58
59
60
61
62
63
64 public static void metaScan(Configuration configuration,
65 MetaScannerVisitor visitor)
66 throws IOException {
67 metaScan(configuration, visitor, null);
68 }
69
70
71
72
73
74
75
76
77
78
79
80 public static void metaScan(Configuration configuration,
81 MetaScannerVisitor visitor, byte [] userTableName)
82 throws IOException {
83 metaScan(configuration, visitor, userTableName, null, Integer.MAX_VALUE);
84 }
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101 public static void metaScan(Configuration configuration,
102 MetaScannerVisitor visitor, byte [] userTableName, byte[] row,
103 int rowLimit)
104 throws IOException {
105 metaScan(configuration, visitor, userTableName, row, rowLimit,
106 HConstants.META_TABLE_NAME);
107 }
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125 public static void metaScan(Configuration configuration,
126 final MetaScannerVisitor visitor, final byte[] tableName,
127 final byte[] row, final int rowLimit, final byte[] metaTableName)
128 throws IOException {
129 try {
130 HConnectionManager.execute(new HConnectable<Void>(configuration) {
131 @Override
132 public Void connect(HConnection connection) throws IOException {
133 metaScan(conf, connection, visitor, tableName, row, rowLimit,
134 metaTableName);
135 return null;
136 }
137 });
138 } finally {
139 visitor.close();
140 }
141 }
142
143 private static void metaScan(Configuration configuration, HConnection connection,
144 MetaScannerVisitor visitor, byte [] tableName, byte[] row,
145 int rowLimit, final byte [] metaTableName)
146 throws IOException {
147 int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
148
149
150
151 byte[] startRow;
152 if (row != null) {
153
154 assert tableName != null;
155 byte[] searchRow =
156 HRegionInfo.createRegionName(tableName, row, HConstants.NINES,
157 false);
158 HTable metaTable = null;
159 try {
160 metaTable = new HTable(configuration, HConstants.META_TABLE_NAME);
161 Result startRowResult = metaTable.getRowOrBefore(searchRow,
162 HConstants.CATALOG_FAMILY);
163 if (startRowResult == null) {
164 throw new TableNotFoundException("Cannot find row in .META. for table: "
165 + Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow));
166 }
167 byte[] value = startRowResult.getValue(HConstants.CATALOG_FAMILY,
168 HConstants.REGIONINFO_QUALIFIER);
169 if (value == null || value.length == 0) {
170 throw new IOException("HRegionInfo was null or empty in Meta for " +
171 Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow));
172 }
173 HRegionInfo regionInfo = Writables.getHRegionInfo(value);
174
175 byte[] rowBefore = regionInfo.getStartKey();
176 startRow = HRegionInfo.createRegionName(tableName, rowBefore,
177 HConstants.ZEROES, false);
178 } finally {
179 if (metaTable != null) {
180 metaTable.close();
181 }
182 }
183 } else if (tableName == null || tableName.length == 0) {
184
185 startRow = HConstants.EMPTY_START_ROW;
186 } else {
187
188 startRow = HRegionInfo.createRegionName(
189 tableName, HConstants.EMPTY_START_ROW, HConstants.ZEROES, false);
190 }
191
192
193 ScannerCallable callable;
194 int rows = Math.min(rowLimit, configuration.getInt(
195 HConstants.HBASE_META_SCANNER_CACHING,
196 HConstants.DEFAULT_HBASE_META_SCANNER_CACHING));
197 do {
198 final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
199 if (LOG.isDebugEnabled()) {
200 LOG.debug("Scanning " + Bytes.toString(metaTableName) +
201 " starting at row=" + Bytes.toStringBinary(startRow) + " for max=" +
202 rowUpperLimit + " rows using " + connection.toString());
203 }
204 callable = new ScannerCallable(connection, metaTableName, scan, null);
205
206 callable.withRetries();
207
208 int processedRows = 0;
209 try {
210 callable.setCaching(rows);
211 done: do {
212 if (processedRows >= rowUpperLimit) {
213 break;
214 }
215
216 Result [] rrs = callable.withRetries();
217 if (rrs == null || rrs.length == 0 || rrs[0].size() == 0) {
218 break;
219 }
220 for (Result rr : rrs) {
221 if (processedRows >= rowUpperLimit) {
222 break done;
223 }
224 if (!visitor.processRow(rr))
225 break done;
226 processedRows++;
227 }
228
229 } while(true);
230
231 startRow = callable.getHRegionInfo().getEndKey();
232 } finally {
233
234 callable.setClose();
235 callable.withRetries();
236 }
237 } while (Bytes.compareTo(startRow, HConstants.LAST_ROW) != 0);
238 }
239
240
241
242
243
244
245
246 public static List<HRegionInfo> listAllRegions(Configuration conf)
247 throws IOException {
248 return listAllRegions(conf, true);
249 }
250
251
252
253
254
255
256
257
258
259 public static List<HRegionInfo> listAllRegions(Configuration conf, final boolean offlined)
260 throws IOException {
261 final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
262 MetaScannerVisitor visitor = new BlockingMetaScannerVisitor(conf) {
263 @Override
264 public boolean processRowInternal(Result result) throws IOException {
265 if (result == null || result.isEmpty()) {
266 return true;
267 }
268 byte [] bytes = result.getValue(HConstants.CATALOG_FAMILY,
269 HConstants.REGIONINFO_QUALIFIER);
270 if (bytes == null) {
271 LOG.warn("Null REGIONINFO_QUALIFIER: " + result);
272 return true;
273 }
274 HRegionInfo regionInfo = Writables.getHRegionInfo(bytes);
275
276 if (regionInfo.isOffline() && !offlined) return true;
277 regions.add(regionInfo);
278 return true;
279 }
280 };
281 metaScan(conf, visitor);
282 return regions;
283 }
284
285
286
287
288
289
290
291
292
293 public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
294 final byte [] tablename, final boolean offlined) throws IOException {
295 final NavigableMap<HRegionInfo, ServerName> regions =
296 new TreeMap<HRegionInfo, ServerName>();
297 MetaScannerVisitor visitor = new TableMetaScannerVisitor(conf, tablename) {
298 @Override
299 public boolean processRowInternal(Result rowResult) throws IOException {
300 HRegionInfo info = Writables.getHRegionInfo(
301 rowResult.getValue(HConstants.CATALOG_FAMILY,
302 HConstants.REGIONINFO_QUALIFIER));
303 byte [] value = rowResult.getValue(HConstants.CATALOG_FAMILY,
304 HConstants.SERVER_QUALIFIER);
305 String hostAndPort = null;
306 if (value != null && value.length > 0) {
307 hostAndPort = Bytes.toString(value);
308 }
309 value = rowResult.getValue(HConstants.CATALOG_FAMILY,
310 HConstants.STARTCODE_QUALIFIER);
311 long startcode = -1L;
312 if (value != null && value.length > 0) startcode = Bytes.toLong(value);
313 if (!(info.isOffline() || info.isSplit())) {
314 ServerName sn = null;
315 if (hostAndPort != null && hostAndPort.length() > 0) {
316 sn = new ServerName(hostAndPort, startcode);
317 }
318 regions.put(new UnmodifyableHRegionInfo(info), sn);
319 }
320 return true;
321 }
322 };
323 metaScan(conf, visitor, tablename);
324 return regions;
325 }
326
327
328
329
330 public interface MetaScannerVisitor extends Closeable {
331
332
333
334
335
336
337
338
339
340 public boolean processRow(Result rowResult) throws IOException;
341 }
342
343 public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor {
344 @Override
345 public void close() throws IOException {
346 }
347 }
348
349
350
351
352
353
354
355 public static abstract class BlockingMetaScannerVisitor
356 extends MetaScannerVisitorBase {
357
358 private static final int DEFAULT_BLOCKING_TIMEOUT = 10000;
359 private Configuration conf;
360 private TreeSet<byte[]> daughterRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
361 private int blockingTimeout;
362 private HTable metaTable;
363
364 public BlockingMetaScannerVisitor(Configuration conf) {
365 this.conf = conf;
366 this.blockingTimeout = conf.getInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT,
367 DEFAULT_BLOCKING_TIMEOUT);
368 }
369
370 public abstract boolean processRowInternal(Result rowResult) throws IOException;
371
372 @Override
373 public void close() throws IOException {
374 super.close();
375 if (metaTable != null) {
376 metaTable.close();
377 metaTable = null;
378 }
379 }
380
381 public HTable getMetaTable() throws IOException {
382 if (metaTable == null) {
383 metaTable = new HTable(conf, HConstants.META_TABLE_NAME);
384 }
385 return metaTable;
386 }
387
388 @Override
389 public boolean processRow(Result rowResult) throws IOException {
390 HRegionInfo info = Writables.getHRegionInfoOrNull(
391 rowResult.getValue(HConstants.CATALOG_FAMILY,
392 HConstants.REGIONINFO_QUALIFIER));
393 if (info == null) {
394 return true;
395 }
396
397 if (daughterRegions.remove(info.getRegionName())) {
398 return true;
399 }
400
401 if (info.isSplitParent()) {
402
403
404
405
406 HRegionInfo splitA = Writables.getHRegionInfo(rowResult.getValue(HConstants.CATALOG_FAMILY,
407 HConstants.SPLITA_QUALIFIER));
408 HRegionInfo splitB = Writables.getHRegionInfo(rowResult.getValue(HConstants.CATALOG_FAMILY,
409 HConstants.SPLITB_QUALIFIER));
410
411 HTable metaTable = getMetaTable();
412 long start = System.currentTimeMillis();
413 Result resultA = getRegionResultBlocking(metaTable, blockingTimeout,
414 splitA.getRegionName());
415 if (resultA != null) {
416 processRow(resultA);
417 daughterRegions.add(splitA.getRegionName());
418 } else {
419 throw new RegionOfflineException("Split daughter region " +
420 splitA.getRegionNameAsString() + " cannot be found in META.");
421 }
422 long rem = blockingTimeout - (System.currentTimeMillis() - start);
423
424 Result resultB = getRegionResultBlocking(metaTable, rem,
425 splitB.getRegionName());
426 if (resultB != null) {
427 processRow(resultB);
428 daughterRegions.add(splitB.getRegionName());
429 } else {
430 throw new RegionOfflineException("Split daughter region " +
431 splitB.getRegionNameAsString() + " cannot be found in META.");
432 }
433 }
434
435 return processRowInternal(rowResult);
436 }
437
438 private Result getRegionResultBlocking(HTable metaTable, long timeout, byte[] regionName)
439 throws IOException {
440 if (LOG.isDebugEnabled()) {
441 LOG.debug("blocking until region is in META: " + Bytes.toStringBinary(regionName));
442 }
443 long start = System.currentTimeMillis();
444 while (System.currentTimeMillis() - start < timeout) {
445 Get get = new Get(regionName);
446 Result result = metaTable.get(get);
447 HRegionInfo info = Writables.getHRegionInfoOrNull(
448 result.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
449 if (info != null) {
450 return result;
451 }
452 try {
453 Thread.sleep(10);
454 } catch (InterruptedException ex) {
455 Thread.currentThread().interrupt();
456 break;
457 }
458 }
459 return null;
460 }
461 }
462
463
464
465
466
467
468
469 public static abstract class TableMetaScannerVisitor extends BlockingMetaScannerVisitor {
470 private byte[] tableName;
471
472 public TableMetaScannerVisitor(Configuration conf, byte[] tableName) {
473 super(conf);
474 this.tableName = tableName;
475 }
476
477 @Override
478 public final boolean processRow(Result rowResult) throws IOException {
479 HRegionInfo info = Writables.getHRegionInfoOrNull(
480 rowResult.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
481 if (info == null) {
482 return true;
483 }
484 if (!(Bytes.equals(info.getTableName(), tableName))) {
485 return false;
486 }
487 return super.processRow(rowResult);
488 }
489
490 }
491 }