View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.catalog;
19  
20  import org.apache.commons.logging.Log;
21  import org.apache.commons.logging.LogFactory;
22  import org.apache.hadoop.classification.InterfaceAudience;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.TableName;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.HRegionInfo;
27  import org.apache.hadoop.hbase.HTableDescriptor;
28  import org.apache.hadoop.hbase.ServerName;
29  import org.apache.hadoop.hbase.client.Get;
30  import org.apache.hadoop.hbase.client.HTable;
31  import org.apache.hadoop.hbase.client.Result;
32  import org.apache.hadoop.hbase.client.ResultScanner;
33  import org.apache.hadoop.hbase.client.Scan;
34  import org.apache.hadoop.hbase.util.Bytes;
35  import org.apache.hadoop.hbase.util.Pair;
36  
37  import java.io.IOException;
38  import java.util.ArrayList;
39  import java.util.List;
40  import java.util.Map;
41  import java.util.NavigableMap;
42  import java.util.Set;
43  import java.util.TreeMap;
44  
45  /**
46   * Reads region and assignment information from <code>hbase:meta</code>.
47   */
48  @InterfaceAudience.Private
49  public class MetaReader {
50    // TODO: Strip CatalogTracker from this class.  Its all over and in the end
51    // its only used to get its Configuration so we can get associated
52    // Connection.
53    private static final Log LOG = LogFactory.getLog(MetaReader.class);
54  
55    static final byte [] META_REGION_PREFIX;
56    static {
57      // Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX.
58      // FIRST_META_REGIONINFO == 'hbase:meta,,1'.  META_REGION_PREFIX == 'hbase:meta,'
59      int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2;
60      META_REGION_PREFIX = new byte [len];
61      System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0,
62        META_REGION_PREFIX, 0, len);
63    }
64  
65    /**
66     * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
67     * tables in the specified set of disabled tables.
68     * @param catalogTracker
69     * @param disabledTables set of disabled tables that will not be returned
70     * @return Returns a map of every region to it's currently assigned server,
71     * according to META.  If the region does not have an assignment it will have
72     * a null value in the map.
73     * @throws IOException
74     */
75    public static Map<HRegionInfo, ServerName> fullScan(
76        CatalogTracker catalogTracker, final Set<TableName> disabledTables)
77    throws IOException {
78      return fullScan(catalogTracker, disabledTables, false);
79    }
80  
81    /**
82     * Performs a full scan of <code>hbase:meta</code>, skipping regions from any
83     * tables in the specified set of disabled tables.
84     * @param catalogTracker
85     * @param disabledTables set of disabled tables that will not be returned
86     * @param excludeOfflinedSplitParents If true, do not include offlined split
87     * parents in the return.
88     * @return Returns a map of every region to it's currently assigned server,
89     * according to META.  If the region does not have an assignment it will have
90     * a null value in the map.
91     * @throws IOException
92     */
93    public static Map<HRegionInfo, ServerName> fullScan(
94        CatalogTracker catalogTracker, final Set<TableName> disabledTables,
95        final boolean excludeOfflinedSplitParents)
96    throws IOException {
97      final Map<HRegionInfo, ServerName> regions =
98        new TreeMap<HRegionInfo, ServerName>();
99      Visitor v = new Visitor() {
100       @Override
101       public boolean visit(Result r) throws IOException {
102         if (r ==  null || r.isEmpty()) return true;
103         Pair<HRegionInfo, ServerName> region = HRegionInfo.getHRegionInfoAndServerName(r);
104         HRegionInfo hri = region.getFirst();
105         if (hri  == null) return true;
106         if (hri.getTable() == null) return true;
107         if (disabledTables.contains(
108             hri.getTable())) return true;
109         // Are we to include split parents in the list?
110         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
111         regions.put(hri, region.getSecond());
112         return true;
113       }
114     };
115     fullScan(catalogTracker, v);
116     return regions;
117   }
118 
119   /**
120    * Performs a full scan of <code>hbase:meta</code>.
121    * @return List of {@link Result}
122    * @throws IOException
123    */
124   public static List<Result> fullScan(CatalogTracker catalogTracker)
125   throws IOException {
126     CollectAllVisitor v = new CollectAllVisitor();
127     fullScan(catalogTracker, v, null);
128     return v.getResults();
129   }
130 
131   /**
132    * Performs a full scan of a <code>hbase:meta</code> table.
133    * @return List of {@link Result}
134    * @throws IOException
135    */
136   public static List<Result> fullScanOfMeta(CatalogTracker catalogTracker)
137   throws IOException {
138     CollectAllVisitor v = new CollectAllVisitor();
139     fullScan(catalogTracker, v, null);
140     return v.getResults();
141   }
142 
143   /**
144    * Performs a full scan of <code>hbase:meta</code>.
145    * @param catalogTracker
146    * @param visitor Visitor invoked against each row.
147    * @throws IOException
148    */
149   public static void fullScan(CatalogTracker catalogTracker,
150       final Visitor visitor)
151   throws IOException {
152     fullScan(catalogTracker, visitor, null);
153   }
154 
155   /**
156    * Callers should call close on the returned {@link HTable} instance.
157    * @param catalogTracker We'll use this catalogtracker's connection
158    * @param tableName Table to get an {@link HTable} against.
159    * @return An {@link HTable} for <code>tableName</code>
160    * @throws IOException
161    */
162   private static HTable getHTable(final CatalogTracker catalogTracker,
163       final TableName tableName)
164   throws IOException {
165     // Passing the CatalogTracker's connection ensures this
166     // HTable instance uses the CatalogTracker's connection.
167     org.apache.hadoop.hbase.client.HConnection c = catalogTracker.getConnection();
168     if (c == null) throw new NullPointerException("No connection");
169     return new HTable(tableName, c);
170   }
171 
172   /**
173    * Callers should call close on the returned {@link HTable} instance.
174    * @param catalogTracker
175    * @return An {@link HTable} for <code>hbase:meta</code>
176    * @throws IOException
177    */
178   static HTable getCatalogHTable(final CatalogTracker catalogTracker)
179   throws IOException {
180     return getMetaHTable(catalogTracker);
181   }
182 
183   /**
184    * Callers should call close on the returned {@link HTable} instance.
185    * @param ct
186    * @return An {@link HTable} for <code>hbase:meta</code>
187    * @throws IOException
188    */
189   static HTable getMetaHTable(final CatalogTracker ct)
190   throws IOException {
191     return getHTable(ct, TableName.META_TABLE_NAME);
192   }
193 
194   /**
195    * @param t Table to use (will be closed when done).
196    * @param g Get to run
197    * @throws IOException
198    */
199   private static Result get(final HTable t, final Get g) throws IOException {
200     try {
201       return t.get(g);
202     } finally {
203       t.close();
204     }
205   }
206 
207   /**
208    * Reads the location of the specified region
209    * @param catalogTracker
210    * @param regionName region whose location we are after
211    * @return location of region as a {@link ServerName} or null if not found
212    * @throws IOException
213    */
214   static ServerName readRegionLocation(CatalogTracker catalogTracker,
215       byte [] regionName)
216   throws IOException {
217     Pair<HRegionInfo, ServerName> pair = getRegion(catalogTracker, regionName);
218     return (pair == null || pair.getSecond() == null)? null: pair.getSecond();
219   }
220 
221   /**
222    * Gets the region info and assignment for the specified region.
223    * @param catalogTracker
224    * @param regionName Region to lookup.
225    * @return Location and HRegionInfo for <code>regionName</code>
226    * @throws IOException
227    */
228   public static Pair<HRegionInfo, ServerName> getRegion(
229       CatalogTracker catalogTracker, byte [] regionName)
230   throws IOException {
231     Get get = new Get(regionName);
232     get.addFamily(HConstants.CATALOG_FAMILY);
233     Result r = get(getCatalogHTable(catalogTracker), get);
234     return (r == null || r.isEmpty())? null: HRegionInfo.getHRegionInfoAndServerName(r);
235   }
236 
237   /**
238    * Gets the result in hbase:meta for the specified region.
239    * @param catalogTracker
240    * @param regionName
241    * @return result of the specified region
242    * @throws IOException
243    */
244   public static Result getRegionResult(CatalogTracker catalogTracker,
245       byte[] regionName) throws IOException {
246     Get get = new Get(regionName);
247     get.addFamily(HConstants.CATALOG_FAMILY);
248     return get(getCatalogHTable(catalogTracker), get);
249   }
250 
251   /**
252    * Get regions from the merge qualifier of the specified merged region
253    * @return null if it doesn't contain merge qualifier, else two merge regions
254    * @throws IOException
255    */
256   public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
257       CatalogTracker catalogTracker, byte[] regionName) throws IOException {
258     Result result = getRegionResult(catalogTracker, regionName);
259     HRegionInfo mergeA = HRegionInfo.getHRegionInfo(result,
260         HConstants.MERGEA_QUALIFIER);
261     HRegionInfo mergeB = HRegionInfo.getHRegionInfo(result,
262         HConstants.MERGEB_QUALIFIER);
263     if (mergeA == null && mergeB == null) {
264       return null;
265     }
266     return new Pair<HRegionInfo, HRegionInfo>(mergeA, mergeB);
267  }
268 
269   /**
270    * Checks if the specified table exists.  Looks at the hbase:meta table hosted on
271    * the specified server.
272    * @param catalogTracker
273    * @param tableName table to check
274    * @return true if the table exists in meta, false if not
275    * @throws IOException
276    */
277   public static boolean tableExists(CatalogTracker catalogTracker,
278       final TableName tableName)
279   throws IOException {
280     if (tableName.equals(HTableDescriptor.META_TABLEDESC.getTableName())) {
281       // Catalog tables always exist.
282       return true;
283     }
284     // Make a version of ResultCollectingVisitor that only collects the first
285     CollectingVisitor<HRegionInfo> visitor = new CollectingVisitor<HRegionInfo>() {
286       private HRegionInfo current = null;
287 
288       @Override
289       public boolean visit(Result r) throws IOException {
290         this.current =
291           HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
292         if (this.current == null) {
293           LOG.warn("No serialized HRegionInfo in " + r);
294           return true;
295         }
296         if (!isInsideTable(this.current, tableName)) return false;
297         // Else call super and add this Result to the collection.
298         super.visit(r);
299         // Stop collecting regions from table after we get one.
300         return false;
301       }
302 
303       @Override
304       void add(Result r) {
305         // Add the current HRI.
306         this.results.add(this.current);
307       }
308     };
309     fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
310     // If visitor has results >= 1 then table exists.
311     return visitor.getResults().size() >= 1;
312   }
313 
314   /**
315    * Gets all of the regions of the specified table.
316    * @param catalogTracker
317    * @param tableName
318    * @return Ordered list of {@link HRegionInfo}.
319    * @throws IOException
320    */
321   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
322       TableName tableName)
323   throws IOException {
324     return getTableRegions(catalogTracker, tableName, false);
325   }
326 
327   /**
328    * Gets all of the regions of the specified table.
329    * @param catalogTracker
330    * @param tableName
331    * @param excludeOfflinedSplitParents If true, do not include offlined split
332    * parents in the return.
333    * @return Ordered list of {@link HRegionInfo}.
334    * @throws IOException
335    */
336   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
337       TableName tableName, final boolean excludeOfflinedSplitParents)
338   throws IOException {
339     List<Pair<HRegionInfo, ServerName>> result = null;
340     try {
341       result = getTableRegionsAndLocations(catalogTracker, tableName,
342         excludeOfflinedSplitParents);
343     } catch (InterruptedException e) {
344       throw new RuntimeException(e);
345     }
346     return getListOfHRegionInfos(result);
347   }
348 
349   static List<HRegionInfo> getListOfHRegionInfos(final List<Pair<HRegionInfo, ServerName>> pairs) {
350     if (pairs == null || pairs.isEmpty()) return null;
351     List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
352     for (Pair<HRegionInfo, ServerName> pair: pairs) {
353       result.add(pair.getFirst());
354     }
355     return result;
356   }
357 
358   /**
359    * @param current
360    * @param tableName
361    * @return True if <code>current</code> tablename is equal to
362    * <code>tableName</code>
363    */
364   static boolean isInsideTable(final HRegionInfo current, final TableName tableName) {
365     return tableName.equals(current.getTable());
366   }
367 
368   /**
369    * @param tableName
370    * @return Place to start Scan in <code>hbase:meta</code> when passed a
371    * <code>tableName</code>; returns &lt;tableName&rt; &lt;,&rt; &lt;,&rt;
372    */
373   static byte [] getTableStartRowForMeta(TableName tableName) {
374     byte [] startRow = new byte[tableName.getName().length + 2];
375     System.arraycopy(tableName.getName(), 0, startRow, 0, tableName.getName().length);
376     startRow[startRow.length - 2] = HConstants.DELIMITER;
377     startRow[startRow.length - 1] = HConstants.DELIMITER;
378     return startRow;
379   }
380 
381   /**
382    * This method creates a Scan object that will only scan catalog rows that
383    * belong to the specified table. It doesn't specify any columns.
384    * This is a better alternative to just using a start row and scan until
385    * it hits a new table since that requires parsing the HRI to get the table
386    * name.
387    * @param tableName bytes of table's name
388    * @return configured Scan object
389    */
390   public static Scan getScanForTableName(TableName tableName) {
391     String strName = tableName.getNameAsString();
392     // Start key is just the table name with delimiters
393     byte[] startKey = Bytes.toBytes(strName + ",,");
394     // Stop key appends the smallest possible char to the table name
395     byte[] stopKey = Bytes.toBytes(strName + " ,,");
396 
397     Scan scan = new Scan(startKey);
398     scan.setStopRow(stopKey);
399     return scan;
400   }
401 
402   /**
403    * @param catalogTracker
404    * @param tableName
405    * @return Return list of regioninfos and server.
406    * @throws IOException
407    * @throws InterruptedException
408    */
409   public static List<Pair<HRegionInfo, ServerName>>
410   getTableRegionsAndLocations(CatalogTracker catalogTracker, TableName tableName)
411   throws IOException, InterruptedException {
412     return getTableRegionsAndLocations(catalogTracker, tableName,
413       true);
414   }
415 
416   /**
417    * @param catalogTracker
418    * @param tableName
419    * @return Return list of regioninfos and server addresses.
420    * @throws IOException
421    * @throws InterruptedException
422    */
423   public static List<Pair<HRegionInfo, ServerName>>
424   getTableRegionsAndLocations(final CatalogTracker catalogTracker,
425       final TableName tableName, final boolean excludeOfflinedSplitParents)
426   throws IOException, InterruptedException {
427     if (tableName.equals(TableName.META_TABLE_NAME)) {
428       // If meta, do a bit of special handling.
429       ServerName serverName = catalogTracker.getMetaLocation();
430       List<Pair<HRegionInfo, ServerName>> list =
431           new ArrayList<Pair<HRegionInfo, ServerName>>();
432       list.add(new Pair<HRegionInfo, ServerName>(HRegionInfo.FIRST_META_REGIONINFO,
433           serverName));
434       return list;
435     }
436     // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress
437     CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
438         new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
439       private Pair<HRegionInfo, ServerName> current = null;
440 
441       @Override
442       public boolean visit(Result r) throws IOException {
443         HRegionInfo hri =
444           HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
445         if (hri == null) {
446           LOG.warn("No serialized HRegionInfo in " + r);
447           return true;
448         }
449         if (!isInsideTable(hri, tableName)) return false;
450         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
451         ServerName sn = HRegionInfo.getServerName(r);
452         // Populate this.current so available when we call #add
453         this.current = new Pair<HRegionInfo, ServerName>(hri, sn);
454         // Else call super and add this Result to the collection.
455         return super.visit(r);
456       }
457 
458       @Override
459       void add(Result r) {
460         this.results.add(this.current);
461       }
462     };
463     fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
464     return visitor.getResults();
465   }
466 
467   /**
468    * @param catalogTracker
469    * @param serverName
470    * @return List of user regions installed on this server (does not include
471    * catalog regions).
472    * @throws IOException
473    */
474   public static NavigableMap<HRegionInfo, Result>
475   getServerUserRegions(CatalogTracker catalogTracker, final ServerName serverName)
476   throws IOException {
477     final NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
478     // Fill the above hris map with entries from hbase:meta that have the passed
479     // servername.
480     CollectingVisitor<Result> v = new CollectingVisitor<Result>() {
481       @Override
482       void add(Result r) {
483         if (r == null || r.isEmpty()) return;
484         ServerName sn = HRegionInfo.getServerName(r);
485         if (sn != null && sn.equals(serverName)) this.results.add(r);
486       }
487     };
488     fullScan(catalogTracker, v);
489     List<Result> results = v.getResults();
490     if (results != null && !results.isEmpty()) {
491       // Convert results to Map keyed by HRI
492       for (Result r: results) {
493         Pair<HRegionInfo, ServerName> p = HRegionInfo.getHRegionInfoAndServerName(r);
494         if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r);
495       }
496     }
497     return hris;
498   }
499 
500   public static void fullScanMetaAndPrint(final CatalogTracker catalogTracker)
501   throws IOException {
502     Visitor v = new Visitor() {
503       @Override
504       public boolean visit(Result r) throws IOException {
505         if (r ==  null || r.isEmpty()) return true;
506         LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r);
507         HRegionInfo hrim = HRegionInfo.getHRegionInfo(r);
508         LOG.info("fullScanMetaAndPrint.HRI Print= " + hrim);
509         return true;
510       }
511     };
512     fullScan(catalogTracker, v);
513   }
514 
515   /**
516    * Performs a full scan of a catalog table.
517    * @param catalogTracker
518    * @param visitor Visitor invoked against each row.
519    * @param startrow Where to start the scan. Pass null if want to begin scan
520    * at first row.
521    * <code>hbase:meta</code>, the default (pass false to scan hbase:meta)
522    * @throws IOException
523    */
524   public static void fullScan(CatalogTracker catalogTracker,
525     final Visitor visitor, final byte [] startrow)
526   throws IOException {
527     Scan scan = new Scan();
528     if (startrow != null) scan.setStartRow(startrow);
529     if (startrow == null) {
530       int caching = catalogTracker.getConnection().getConfiguration()
531           .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
532       scan.setCaching(caching);
533     }
534     scan.addFamily(HConstants.CATALOG_FAMILY);
535     HTable metaTable = getMetaHTable(catalogTracker);
536     ResultScanner scanner = null;
537     try {
538       scanner = metaTable.getScanner(scan);
539       Result data;
540       while((data = scanner.next()) != null) {
541         if (data.isEmpty()) continue;
542         // Break if visit returns false.
543         if (!visitor.visit(data)) break;
544       }
545     } finally {
546       if (scanner != null) scanner.close();
547       metaTable.close();
548     }
549     return;
550   }
551 
552   /**
553    * Implementations 'visit' a catalog table row.
554    */
555   public interface Visitor {
556     /**
557      * Visit the catalog table row.
558      * @param r A row from catalog table
559      * @return True if we are to proceed scanning the table, else false if
560      * we are to stop now.
561      */
562     boolean visit(final Result r) throws IOException;
563   }
564 
565   /**
566    * A {@link Visitor} that collects content out of passed {@link Result}.
567    */
568   static abstract class CollectingVisitor<T> implements Visitor {
569     final List<T> results = new ArrayList<T>();
570     @Override
571     public boolean visit(Result r) throws IOException {
572       if (r ==  null || r.isEmpty()) return true;
573       add(r);
574       return true;
575     }
576 
577     abstract void add(Result r);
578 
579     /**
580      * @return Collected results; wait till visits complete to collect all
581      * possible results
582      */
583     List<T> getResults() {
584       return this.results;
585     }
586   }
587 
588   /**
589    * Collects all returned.
590    */
591   static class CollectAllVisitor extends CollectingVisitor<Result> {
592     @Override
593     void add(Result r) {
594       this.results.add(r);
595     }
596   }
597 
598   /**
599    * Count regions in <code>hbase:meta</code> for passed table.
600    * @param c
601    * @param tableName
602    * @return Count or regions in table <code>tableName</code>
603    * @throws IOException
604    */
605   public static int getRegionCount(final Configuration c, final String tableName) throws IOException {
606     HTable t = new HTable(c, tableName);
607     try {
608       return t.getRegionLocations().size();
609     } finally {
610       t.close();
611     }
612   }
613 }