1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.catalog;
19
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.NavigableMap;
25 import java.util.Set;
26 import java.util.TreeMap;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HRegionInfo;
32 import org.apache.hadoop.hbase.HTableDescriptor;
33 import org.apache.hadoop.hbase.NotServingRegionException;
34 import org.apache.hadoop.hbase.ServerName;
35 import org.apache.hadoop.hbase.client.Get;
36 import org.apache.hadoop.hbase.client.HTable;
37 import org.apache.hadoop.hbase.client.Result;
38 import org.apache.hadoop.hbase.client.ResultScanner;
39 import org.apache.hadoop.hbase.client.Scan;
40 import org.apache.hadoop.hbase.ipc.HRegionInterface;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.apache.hadoop.hbase.util.Pair;
43 import org.apache.hadoop.hbase.util.PairOfSameType;
44 import org.apache.hadoop.hbase.util.Writables;
45 import org.apache.hadoop.ipc.RemoteException;
46
47
48
49
50 public class MetaReader {
51
52
53
54 private static final Log LOG = LogFactory.getLog(MetaReader.class);
55
56 static final byte [] META_REGION_PREFIX;
57 static {
58
59
60 int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2;
61 META_REGION_PREFIX = new byte [len];
62 System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0,
63 META_REGION_PREFIX, 0, len);
64 }
65
66
67
68
69
70 private static boolean isRootTableRow(final byte [] row) {
71 if (row.length < META_REGION_PREFIX.length + 2
72
73 return false;
74 }
75
76
77 return Bytes.equals(row, 0, META_REGION_PREFIX.length,
78 META_REGION_PREFIX, 0, META_REGION_PREFIX.length);
79 }
80
81
82
83
84
85
86
87
88
89
90
91 public static Map<HRegionInfo, ServerName> fullScan(
92 CatalogTracker catalogTracker, final Set<String> disabledTables)
93 throws IOException {
94 return fullScan(catalogTracker, disabledTables, false);
95 }
96
97
98
99
100
101
102
103
104
105
106
107
108
109 public static Map<HRegionInfo, ServerName> fullScan(
110 CatalogTracker catalogTracker, final Set<String> disabledTables,
111 final boolean excludeOfflinedSplitParents)
112 throws IOException {
113 final Map<HRegionInfo, ServerName> regions =
114 new TreeMap<HRegionInfo, ServerName>();
115 Visitor v = new Visitor() {
116 @Override
117 public boolean visit(Result r) throws IOException {
118 if (r == null || r.isEmpty()) return true;
119 Pair<HRegionInfo, ServerName> region = parseCatalogResult(r);
120 if (region == null) return true;
121 HRegionInfo hri = region.getFirst();
122 if (hri == null) return true;
123 if (hri.getTableNameAsString() == null) return true;
124 if (disabledTables.contains(
125 hri.getTableNameAsString())) return true;
126
127 if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
128 regions.put(hri, region.getSecond());
129 return true;
130 }
131 };
132 fullScan(catalogTracker, v);
133 return regions;
134 }
135
136
137
138
139
140
141 public static List<Result> fullScan(CatalogTracker catalogTracker)
142 throws IOException {
143 CollectAllVisitor v = new CollectAllVisitor();
144 fullScan(catalogTracker, v, null);
145 return v.getResults();
146 }
147
148
149
150
151
152
153 public static List<Result> fullScanOfRoot(CatalogTracker catalogTracker)
154 throws IOException {
155 CollectAllVisitor v = new CollectAllVisitor();
156 fullScan(catalogTracker, v, null, true);
157 return v.getResults();
158 }
159
160
161
162
163
164
165
166 public static void fullScan(CatalogTracker catalogTracker,
167 final Visitor visitor)
168 throws IOException {
169 fullScan(catalogTracker, visitor, null);
170 }
171
172
173
174
175
176
177
178
179
180
181 public static void fullScan(CatalogTracker catalogTracker,
182 final Visitor visitor, final byte [] startrow)
183 throws IOException {
184 fullScan(catalogTracker, visitor, startrow, false);
185 }
186
187
188
189
190
191
192
193
194 private static HTable getHTable(final CatalogTracker catalogTracker,
195 final byte [] tableName)
196 throws IOException {
197
198
199 org.apache.hadoop.hbase.client.HConnection c = catalogTracker.getConnection();
200 if (c == null) throw new NullPointerException("No connection");
201 return new HTable(catalogTracker.getConnection().getConfiguration(), tableName);
202 }
203
204
205
206
207
208
209
210
211 static HTable getCatalogHTable(final CatalogTracker catalogTracker,
212 final byte [] row)
213 throws IOException {
214 return isRootTableRow(row)?
215 getRootHTable(catalogTracker):
216 getMetaHTable(catalogTracker);
217 }
218
219
220
221
222
223
224
225 static HTable getMetaHTable(final CatalogTracker ct)
226 throws IOException {
227 return getHTable(ct, HConstants.META_TABLE_NAME);
228 }
229
230
231
232
233
234
235
236 static HTable getRootHTable(final CatalogTracker ct)
237 throws IOException {
238 return getHTable(ct, HConstants.ROOT_TABLE_NAME);
239 }
240
241
242
243
244
245
246 private static Result get(final HTable t, final Get g) throws IOException {
247 try {
248 return t.get(g);
249 } finally {
250 t.close();
251 }
252 }
253
254
255
256
257
258
259
260
261 public static ServerName readMetaLocation(HRegionInterface metaServer)
262 throws IOException {
263 return readLocation(metaServer, CatalogTracker.ROOT_REGION_NAME,
264 CatalogTracker.META_REGION_NAME);
265 }
266
267
268
269
270
271
272
273
274
275 static ServerName getMetaRegionLocation(final CatalogTracker ct)
276 throws IOException {
277 return MetaReader.readRegionLocation(ct, CatalogTracker.META_REGION_NAME);
278 }
279
280
281
282
283
284
285
286
287 static ServerName readRegionLocation(CatalogTracker catalogTracker,
288 byte [] regionName)
289 throws IOException {
290 Pair<HRegionInfo, ServerName> pair = getRegion(catalogTracker, regionName);
291 return (pair == null || pair.getSecond() == null)? null: pair.getSecond();
292 }
293
294
295 private static ServerName readLocation(HRegionInterface metaServer,
296 byte [] catalogRegionName, byte [] regionName)
297 throws IOException {
298 Result r = null;
299 try {
300 r = metaServer.get(catalogRegionName,
301 new Get(regionName).
302 addColumn(HConstants.CATALOG_FAMILY,
303 HConstants.SERVER_QUALIFIER).
304 addColumn(HConstants.CATALOG_FAMILY,
305 HConstants.STARTCODE_QUALIFIER));
306 } catch (java.net.SocketTimeoutException e) {
307
308
309 } catch (java.net.SocketException e) {
310
311
312 } catch (RemoteException re) {
313 IOException ioe = re.unwrapRemoteException();
314 if (ioe instanceof NotServingRegionException) {
315
316
317 } else if (ioe.getMessage().contains("Server not running")) {
318
319 } else {
320 throw re;
321 }
322 } catch (IOException e) {
323 if (e.getCause() != null && e.getCause() instanceof IOException &&
324 e.getCause().getMessage() != null &&
325 e.getCause().getMessage().contains("Connection reset by peer")) {
326
327
328 } else {
329 throw e;
330 }
331 }
332 if (r == null || r.isEmpty()) {
333 return null;
334 }
335 return getServerNameFromCatalogResult(r);
336 }
337
338
339
340
341
342
343
344
345 public static Pair<HRegionInfo, ServerName> getRegion(
346 CatalogTracker catalogTracker, byte [] regionName)
347 throws IOException {
348 Get get = new Get(regionName);
349 get.addFamily(HConstants.CATALOG_FAMILY);
350 Result r = get(getCatalogHTable(catalogTracker, regionName), get);
351 return (r == null || r.isEmpty())? null: parseCatalogResult(r);
352 }
353
354
355
356
357
358
359
360 public static ServerName getServerNameFromCatalogResult(final Result r) {
361 byte[] value = r.getValue(HConstants.CATALOG_FAMILY,
362 HConstants.SERVER_QUALIFIER);
363 if (value == null || value.length == 0) return null;
364 String hostAndPort = Bytes.toString(value);
365 value = r.getValue(HConstants.CATALOG_FAMILY,
366 HConstants.STARTCODE_QUALIFIER);
367 if (value == null || value.length == 0) return null;
368 return new ServerName(hostAndPort, Bytes.toLong(value));
369 }
370
371
372
373
374
375
376
377
378
379 public static Pair<HRegionInfo, ServerName> parseCatalogResult(final Result r)
380 throws IOException {
381 HRegionInfo info =
382 parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
383 ServerName sn = getServerNameFromCatalogResult(r);
384 return new Pair<HRegionInfo, ServerName>(info, sn);
385 }
386
387
388
389
390
391
392
393
394
395
396
397
398 public static HRegionInfo parseHRegionInfoFromCatalogResult(final Result r,
399 byte [] qualifier)
400 throws IOException {
401 byte [] bytes = r.getValue(HConstants.CATALOG_FAMILY, qualifier);
402 if (bytes == null || bytes.length <= 0) return null;
403 return Writables.getHRegionInfoOrNull(bytes);
404 }
405
406
407
408
409
410
411
412
413
414
415
416 public static boolean tableExists(CatalogTracker catalogTracker,
417 String tableName)
418 throws IOException {
419 if (tableName.equals(HTableDescriptor.ROOT_TABLEDESC.getNameAsString()) ||
420 tableName.equals(HTableDescriptor.META_TABLEDESC.getNameAsString())) {
421
422 return true;
423 }
424 final byte [] tableNameBytes = Bytes.toBytes(tableName);
425
426 CollectingVisitor<HRegionInfo> visitor = new CollectingVisitor<HRegionInfo>() {
427 private HRegionInfo current = null;
428
429 @Override
430 public boolean visit(Result r) throws IOException {
431 this.current =
432 parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
433 if (this.current == null) {
434 LOG.warn("No serialized HRegionInfo in " + r);
435 return true;
436 }
437 if (!isInsideTable(this.current, tableNameBytes)) return false;
438
439 super.visit(r);
440
441 return false;
442 }
443
444 @Override
445 void add(Result r) {
446
447 this.results.add(this.current);
448 }
449 };
450 fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableNameBytes));
451
452 return visitor.getResults().size() >= 1;
453 }
454
455
456
457
458
459
460
461
462 public static PairOfSameType<HRegionInfo> getDaughterRegions(Result data) throws IOException {
463 HRegionInfo splitA = Writables.getHRegionInfoOrNull(data.getValue(HConstants.CATALOG_FAMILY,
464 HConstants.SPLITA_QUALIFIER));
465 HRegionInfo splitB = Writables.getHRegionInfoOrNull(data.getValue(HConstants.CATALOG_FAMILY,
466 HConstants.SPLITB_QUALIFIER));
467 return new PairOfSameType<HRegionInfo>(splitA, splitB);
468 }
469
470
471
472
473
474
475
476
477 public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
478 byte [] tableName)
479 throws IOException {
480 return getTableRegions(catalogTracker, tableName, false);
481 }
482
483
484
485
486
487
488
489
490
491
492 public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
493 byte [] tableName, final boolean excludeOfflinedSplitParents)
494 throws IOException {
495 List<Pair<HRegionInfo, ServerName>> result = null;
496 try {
497 result = getTableRegionsAndLocations(catalogTracker, tableName,
498 excludeOfflinedSplitParents);
499 } catch (InterruptedException e) {
500 throw new RuntimeException(e);
501 }
502 return getListOfHRegionInfos(result);
503 }
504
505 static List<HRegionInfo> getListOfHRegionInfos(final List<Pair<HRegionInfo, ServerName>> pairs) {
506 if (pairs == null || pairs.isEmpty()) return null;
507 List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
508 for (Pair<HRegionInfo, ServerName> pair: pairs) {
509 result.add(pair.getFirst());
510 }
511 return result;
512 }
513
514
515
516
517
518
519
520 static boolean isInsideTable(final HRegionInfo current, final byte [] tableName) {
521 return Bytes.equals(tableName, current.getTableName());
522 }
523
524
525
526
527
528
529 static byte [] getTableStartRowForMeta(final byte [] tableName) {
530 byte [] startRow = new byte[tableName.length + 2];
531 System.arraycopy(tableName, 0, startRow, 0, tableName.length);
532 startRow[startRow.length - 2] = HRegionInfo.DELIMITER;
533 startRow[startRow.length - 1] = HRegionInfo.DELIMITER;
534 return startRow;
535 }
536
537
538
539
540
541
542
543
544
545
546 public static Scan getScanForTableName(byte[] tableName) {
547 String strName = Bytes.toString(tableName);
548
549 byte[] startKey = Bytes.toBytes(strName + ",,");
550
551 byte[] stopKey = Bytes.toBytes(strName + " ,,");
552
553 Scan scan = new Scan(startKey);
554 scan.setStopRow(stopKey);
555 return scan;
556 }
557
558
559
560
561
562
563
564
565 public static List<Pair<HRegionInfo, ServerName>>
566 getTableRegionsAndLocations(CatalogTracker catalogTracker, String tableName)
567 throws IOException, InterruptedException {
568 return getTableRegionsAndLocations(catalogTracker, Bytes.toBytes(tableName),
569 true);
570 }
571
572
573
574
575
576
577
578
579 public static List<Pair<HRegionInfo, ServerName>>
580 getTableRegionsAndLocations(final CatalogTracker catalogTracker,
581 final byte [] tableName, final boolean excludeOfflinedSplitParents)
582 throws IOException, InterruptedException {
583 if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
584
585 ServerName serverName = catalogTracker.getRootLocation();
586 List<Pair<HRegionInfo, ServerName>> list =
587 new ArrayList<Pair<HRegionInfo, ServerName>>();
588 list.add(new Pair<HRegionInfo, ServerName>(HRegionInfo.ROOT_REGIONINFO,
589 serverName));
590 return list;
591 }
592
593 CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
594 new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
595 private Pair<HRegionInfo, ServerName> current = null;
596
597 @Override
598 public boolean visit(Result r) throws IOException {
599 HRegionInfo hri =
600 parseHRegionInfoFromCatalogResult(r, HConstants.REGIONINFO_QUALIFIER);
601 if (hri == null) {
602 LOG.warn("No serialized HRegionInfo in " + r);
603 return true;
604 }
605 if (!isInsideTable(hri, tableName)) return false;
606 if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
607 ServerName sn = getServerNameFromCatalogResult(r);
608
609 this.current = new Pair<HRegionInfo, ServerName>(hri, sn);
610
611 return super.visit(r);
612 }
613
614 @Override
615 void add(Result r) {
616 this.results.add(this.current);
617 }
618 };
619 fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName),
620 Bytes.equals(tableName, HConstants.META_TABLE_NAME));
621 return visitor.getResults();
622 }
623
624
625
626
627
628
629
630
631 public static NavigableMap<HRegionInfo, Result>
632 getServerUserRegions(CatalogTracker catalogTracker, final ServerName serverName)
633 throws IOException {
634 final NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
635
636
637 CollectingVisitor<Result> v = new CollectingVisitor<Result>() {
638 @Override
639 void add(Result r) {
640 if (r == null || r.isEmpty()) return;
641 ServerName sn = getServerNameFromCatalogResult(r);
642 if (sn != null && sn.equals(serverName)) this.results.add(r);
643 }
644 };
645 fullScan(catalogTracker, v);
646 List<Result> results = v.getResults();
647 if (results != null && !results.isEmpty()) {
648
649 for (Result r: results) {
650 Pair<HRegionInfo, ServerName> p = parseCatalogResult(r);
651 if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r);
652 }
653 }
654 return hris;
655 }
656
657 public static void fullScanMetaAndPrint(final CatalogTracker catalogTracker)
658 throws IOException {
659 Visitor v = new Visitor() {
660 @Override
661 public boolean visit(Result r) throws IOException {
662 if (r == null || r.isEmpty()) return true;
663 LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r);
664 HRegionInfo hrim = MetaEditor.getHRegionInfo(r);
665 LOG.info("fullScanMetaAndPrint.HRI Print= " + hrim);
666 return true;
667 }
668 };
669 fullScan(catalogTracker, v);
670 }
671
672
673
674
675
676
677
678
679
680
681
682 public static void fullScan(HRegionInterface hRegionInterface,
683 Visitor visitor, final byte[] regionName,
684 byte[] startrow) throws IOException {
685 if (hRegionInterface == null) return;
686 Scan scan = new Scan();
687 if (startrow != null) scan.setStartRow(startrow);
688 scan.addFamily(HConstants.CATALOG_FAMILY);
689 long scannerid = hRegionInterface.openScanner(regionName, scan);
690 try {
691 Result data;
692 while((data = hRegionInterface.next(scannerid)) != null) {
693 if (!data.isEmpty()) visitor.visit(data);
694 }
695 } finally {
696 hRegionInterface.close(scannerid);
697 }
698 return;
699 }
700
701
702
703
704
705
706
707
708
709
710
711 static void fullScan(CatalogTracker catalogTracker,
712 final Visitor visitor, final byte [] startrow, final boolean scanRoot)
713 throws IOException {
714 Scan scan = new Scan();
715 if (startrow != null) scan.setStartRow(startrow);
716 if (startrow == null && !scanRoot) {
717 int caching = catalogTracker.getConnection().getConfiguration()
718 .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
719 scan.setCaching(caching);
720 }
721 scan.addFamily(HConstants.CATALOG_FAMILY);
722 HTable metaTable = scanRoot?
723 getRootHTable(catalogTracker): getMetaHTable(catalogTracker);
724 ResultScanner scanner = metaTable.getScanner(scan);
725 try {
726 Result data;
727 while((data = scanner.next()) != null) {
728 if (data.isEmpty()) continue;
729
730 if (!visitor.visit(data)) break;
731 }
732 } finally {
733 scanner.close();
734 metaTable.close();
735 }
736 return;
737 }
738
739
740
741
742 public interface Visitor {
743
744
745
746
747
748
749 public boolean visit(final Result r) throws IOException;
750 }
751
752
753
754
755 static abstract class CollectingVisitor<T> implements Visitor {
756 final List<T> results = new ArrayList<T>();
757 @Override
758 public boolean visit(Result r) throws IOException {
759 if (r == null || r.isEmpty()) return true;
760 add(r);
761 return true;
762 }
763
764 abstract void add(Result r);
765
766
767
768
769
770 List<T> getResults() {
771 return this.results;
772 }
773 }
774
775
776
777
778 static class CollectAllVisitor extends CollectingVisitor<Result> {
779 @Override
780 void add(Result r) {
781 this.results.add(r);
782 }
783 }
784 }