View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.catalog;
19  
20  import org.apache.commons.logging.Log;
21  import org.apache.commons.logging.LogFactory;
22  import org.apache.hadoop.classification.InterfaceAudience;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.HConstants;
25  import org.apache.hadoop.hbase.HRegionInfo;
26  import org.apache.hadoop.hbase.HTableDescriptor;
27  import org.apache.hadoop.hbase.ServerName;
28  import org.apache.hadoop.hbase.client.Get;
29  import org.apache.hadoop.hbase.client.HTable;
30  import org.apache.hadoop.hbase.client.Result;
31  import org.apache.hadoop.hbase.client.ResultScanner;
32  import org.apache.hadoop.hbase.client.Scan;
33  import org.apache.hadoop.hbase.util.Bytes;
34  import org.apache.hadoop.hbase.util.Pair;
35  
36  import java.io.IOException;
37  import java.util.ArrayList;
38  import java.util.List;
39  import java.util.Map;
40  import java.util.NavigableMap;
41  import java.util.Set;
42  import java.util.TreeMap;
43  
44  /**
45   * Reads region and assignment information from <code>.META.</code>.
46   */
47  @InterfaceAudience.Private
48  public class MetaReader {
49    // TODO: Strip CatalogTracker from this class.  Its all over and in the end
50    // its only used to get its Configuration so we can get associated
51    // Connection.
52    private static final Log LOG = LogFactory.getLog(MetaReader.class);
53  
54    static final byte [] META_REGION_PREFIX;
55    static {
56      // Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX.
57      // FIRST_META_REGIONINFO == '.META.,,1'.  META_REGION_PREFIX == '.META.,'
58      int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2;
59      META_REGION_PREFIX = new byte [len];
60      System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0,
61        META_REGION_PREFIX, 0, len);
62    }
63  
64    /**
65     * Performs a full scan of <code>.META.</code>, skipping regions from any
66     * tables in the specified set of disabled tables.
67     * @param catalogTracker
68     * @param disabledTables set of disabled tables that will not be returned
69     * @return Returns a map of every region to it's currently assigned server,
70     * according to META.  If the region does not have an assignment it will have
71     * a null value in the map.
72     * @throws IOException
73     */
74    public static Map<HRegionInfo, ServerName> fullScan(
75        CatalogTracker catalogTracker, final Set<String> disabledTables)
76    throws IOException {
77      return fullScan(catalogTracker, disabledTables, false);
78    }
79  
80    /**
81     * Performs a full scan of <code>.META.</code>, skipping regions from any
82     * tables in the specified set of disabled tables.
83     * @param catalogTracker
84     * @param disabledTables set of disabled tables that will not be returned
85     * @param excludeOfflinedSplitParents If true, do not include offlined split
86     * parents in the return.
87     * @return Returns a map of every region to it's currently assigned server,
88     * according to META.  If the region does not have an assignment it will have
89     * a null value in the map.
90     * @throws IOException
91     */
92    public static Map<HRegionInfo, ServerName> fullScan(
93        CatalogTracker catalogTracker, final Set<String> disabledTables,
94        final boolean excludeOfflinedSplitParents)
95    throws IOException {
96      final Map<HRegionInfo, ServerName> regions =
97        new TreeMap<HRegionInfo, ServerName>();
98      Visitor v = new Visitor() {
99        @Override
100       public boolean visit(Result r) throws IOException {
101         if (r ==  null || r.isEmpty()) return true;
102         Pair<HRegionInfo, ServerName> region = HRegionInfo.getHRegionInfoAndServerName(r);
103         HRegionInfo hri = region.getFirst();
104         if (hri  == null) return true;
105         if (hri.getTableNameAsString() == null) return true;
106         if (disabledTables.contains(
107             hri.getTableNameAsString())) return true;
108         // Are we to include split parents in the list?
109         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
110         regions.put(hri, region.getSecond());
111         return true;
112       }
113     };
114     fullScan(catalogTracker, v);
115     return regions;
116   }
117 
118   /**
119    * Performs a full scan of <code>.META.</code>.
120    * @return List of {@link Result}
121    * @throws IOException
122    */
123   public static List<Result> fullScan(CatalogTracker catalogTracker)
124   throws IOException {
125     CollectAllVisitor v = new CollectAllVisitor();
126     fullScan(catalogTracker, v, null);
127     return v.getResults();
128   }
129 
130   /**
131    * Performs a full scan of a <code>.META.</code> table.
132    * @return List of {@link Result}
133    * @throws IOException
134    */
135   public static List<Result> fullScanOfMeta(CatalogTracker catalogTracker)
136   throws IOException {
137     CollectAllVisitor v = new CollectAllVisitor();
138     fullScan(catalogTracker, v, null);
139     return v.getResults();
140   }
141 
142   /**
143    * Performs a full scan of <code>.META.</code>.
144    * @param catalogTracker
145    * @param visitor Visitor invoked against each row.
146    * @throws IOException
147    */
148   public static void fullScan(CatalogTracker catalogTracker,
149       final Visitor visitor)
150   throws IOException {
151     fullScan(catalogTracker, visitor, null);
152   }
153 
154   /**
155    * Callers should call close on the returned {@link HTable} instance.
156    * @param catalogTracker We'll use this catalogtracker's connection
157    * @param tableName Table to get an {@link HTable} against.
158    * @return An {@link HTable} for <code>tableName</code>
159    * @throws IOException
160    */
161   private static HTable getHTable(final CatalogTracker catalogTracker,
162       final byte [] tableName)
163   throws IOException {
164     // Passing the CatalogTracker's connection configuration ensures this
165     // HTable instance uses the CatalogTracker's connection.
166     org.apache.hadoop.hbase.client.HConnection c = catalogTracker.getConnection();
167     if (c == null) throw new NullPointerException("No connection");
168     return new HTable(catalogTracker.getConnection().getConfiguration(), tableName);
169   }
170 
171   /**
172    * Callers should call close on the returned {@link HTable} instance.
173    * @param catalogTracker
174    * @return
175    * @throws IOException
176    */
177   static HTable getCatalogHTable(final CatalogTracker catalogTracker)
178   throws IOException {
179     return getMetaHTable(catalogTracker);
180   }
181 
182   /**
183    * Callers should call close on the returned {@link HTable} instance.
184    * @param ct
185    * @return An {@link HTable} for <code>.META.</code>
186    * @throws IOException
187    */
188   static HTable getMetaHTable(final CatalogTracker ct)
189   throws IOException {
190     return getHTable(ct, HConstants.META_TABLE_NAME);
191   }
192 
193   /**
194    * @param t Table to use (will be closed when done).
195    * @param g Get to run
196    * @throws IOException
197    */
198   private static Result get(final HTable t, final Get g) throws IOException {
199     try {
200       return t.get(g);
201     } finally {
202       t.close();
203     }
204   }
205 
206   /**
207    * Reads the location of the specified region
208    * @param catalogTracker
209    * @param regionName region whose location we are after
210    * @return location of region as a {@link ServerName} or null if not found
211    * @throws IOException
212    */
213   static ServerName readRegionLocation(CatalogTracker catalogTracker,
214       byte [] regionName)
215   throws IOException {
216     Pair<HRegionInfo, ServerName> pair = getRegion(catalogTracker, regionName);
217     return (pair == null || pair.getSecond() == null)? null: pair.getSecond();
218   }
219 
220   /**
221    * Gets the region info and assignment for the specified region.
222    * @param catalogTracker
223    * @param regionName Region to lookup.
224    * @return Location and HRegionInfo for <code>regionName</code>
225    * @throws IOException
226    */
227   public static Pair<HRegionInfo, ServerName> getRegion(
228       CatalogTracker catalogTracker, byte [] regionName)
229   throws IOException {
230     Get get = new Get(regionName);
231     get.addFamily(HConstants.CATALOG_FAMILY);
232     Result r = get(getCatalogHTable(catalogTracker), get);
233     return (r == null || r.isEmpty())? null: HRegionInfo.getHRegionInfoAndServerName(r);
234   }
235 
236   /**
237    * Gets the result in META for the specified region.
238    * @param catalogTracker
239    * @param regionName
240    * @return result of the specified region
241    * @throws IOException
242    */
243   public static Result getRegionResult(CatalogTracker catalogTracker,
244       byte[] regionName) throws IOException {
245     Get get = new Get(regionName);
246     get.addFamily(HConstants.CATALOG_FAMILY);
247     return get(getCatalogHTable(catalogTracker), get);
248   }
249 
250   /**
251    * Get regions from the merge qualifier of the specified merged region
252    * @return null if it doesn't contain merge qualifier, else two merge regions
253    * @throws IOException
254    */
255   public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
256       CatalogTracker catalogTracker, byte[] regionName) throws IOException {
257     Result result = getRegionResult(catalogTracker, regionName);
258     HRegionInfo mergeA = HRegionInfo.getHRegionInfo(result,
259         HConstants.MERGEA_QUALIFIER);
260     HRegionInfo mergeB = HRegionInfo.getHRegionInfo(result,
261         HConstants.MERGEB_QUALIFIER);
262     if (mergeA == null && mergeB == null) {
263       return null;
264     }
265     return new Pair<HRegionInfo, HRegionInfo>(mergeA, mergeB);
266  }
267 
268   /**
269    * Checks if the specified table exists.  Looks at the META table hosted on
270    * the specified server.
271    * @param catalogTracker
272    * @param tableName table to check
273    * @return true if the table exists in meta, false if not
274    * @throws IOException
275    */
276   public static boolean tableExists(CatalogTracker catalogTracker,
277       String tableName)
278   throws IOException {
279     if (tableName.equals(HTableDescriptor.META_TABLEDESC.getNameAsString())) {
280       // Catalog tables always exist.
281       return true;
282     }
283     final byte [] tableNameBytes = Bytes.toBytes(tableName);
284     // Make a version of ResultCollectingVisitor that only collects the first
285     CollectingVisitor<HRegionInfo> visitor = new CollectingVisitor<HRegionInfo>() {
286       private HRegionInfo current = null;
287 
288       @Override
289       public boolean visit(Result r) throws IOException {
290         this.current =
291           HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
292         if (this.current == null) {
293           LOG.warn("No serialized HRegionInfo in " + r);
294           return true;
295         }
296         if (!isInsideTable(this.current, tableNameBytes)) return false;
297         // Else call super and add this Result to the collection.
298         super.visit(r);
299         // Stop collecting regions from table after we get one.
300         return false;
301       }
302 
303       @Override
304       void add(Result r) {
305         // Add the current HRI.
306         this.results.add(this.current);
307       }
308     };
309     fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableNameBytes));
310     // If visitor has results >= 1 then table exists.
311     return visitor.getResults().size() >= 1;
312   }
313 
314   /**
315    * Gets all of the regions of the specified table.
316    * @param catalogTracker
317    * @param tableName
318    * @return Ordered list of {@link HRegionInfo}.
319    * @throws IOException
320    */
321   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
322       byte [] tableName)
323   throws IOException {
324     return getTableRegions(catalogTracker, tableName, false);
325   }
326 
327   /**
328    * Gets all of the regions of the specified table.
329    * @param catalogTracker
330    * @param tableName
331    * @param excludeOfflinedSplitParents If true, do not include offlined split
332    * parents in the return.
333    * @return Ordered list of {@link HRegionInfo}.
334    * @throws IOException
335    */
336   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
337       byte [] tableName, final boolean excludeOfflinedSplitParents)
338   throws IOException {
339     List<Pair<HRegionInfo, ServerName>> result = null;
340     try {
341       result = getTableRegionsAndLocations(catalogTracker, tableName,
342         excludeOfflinedSplitParents);
343     } catch (InterruptedException e) {
344       throw new RuntimeException(e);
345     }
346     return getListOfHRegionInfos(result);
347   }
348 
349   static List<HRegionInfo> getListOfHRegionInfos(final List<Pair<HRegionInfo, ServerName>> pairs) {
350     if (pairs == null || pairs.isEmpty()) return null;
351     List<HRegionInfo> result = new ArrayList<HRegionInfo>(pairs.size());
352     for (Pair<HRegionInfo, ServerName> pair: pairs) {
353       result.add(pair.getFirst());
354     }
355     return result;
356   }
357 
358   /**
359    * @param current
360    * @param tableName
361    * @return True if <code>current</code> tablename is equal to
362    * <code>tableName</code>
363    */
364   static boolean isInsideTable(final HRegionInfo current, final byte [] tableName) {
365     return Bytes.equals(tableName, current.getTableName());
366   }
367 
368   /**
369    * @param tableName
370    * @return Place to start Scan in <code>.META.</code> when passed a
371    * <code>tableName</code>; returns &lt;tableName&rt; &lt;,&rt; &lt;,&rt;
372    */
373   static byte [] getTableStartRowForMeta(final byte [] tableName) {
374     byte [] startRow = new byte[tableName.length + 2];
375     System.arraycopy(tableName, 0, startRow, 0, tableName.length);
376     startRow[startRow.length - 2] = HConstants.DELIMITER;
377     startRow[startRow.length - 1] = HConstants.DELIMITER;
378     return startRow;
379   }
380 
381   /**
382    * This method creates a Scan object that will only scan catalog rows that
383    * belong to the specified table. It doesn't specify any columns.
384    * This is a better alternative to just using a start row and scan until
385    * it hits a new table since that requires parsing the HRI to get the table
386    * name.
387    * @param tableName bytes of table's name
388    * @return configured Scan object
389    */
390   public static Scan getScanForTableName(byte[] tableName) {
391     String strName = Bytes.toString(tableName);
392     // Start key is just the table name with delimiters
393     byte[] startKey = Bytes.toBytes(strName + ",,");
394     // Stop key appends the smallest possible char to the table name
395     byte[] stopKey = Bytes.toBytes(strName + " ,,");
396 
397     Scan scan = new Scan(startKey);
398     scan.setStopRow(stopKey);
399     return scan;
400   }
401 
402   /**
403    * @param catalogTracker
404    * @param tableName
405    * @return Return list of regioninfos and server.
406    * @throws IOException
407    * @throws InterruptedException
408    */
409   public static List<Pair<HRegionInfo, ServerName>>
410   getTableRegionsAndLocations(CatalogTracker catalogTracker, String tableName)
411   throws IOException, InterruptedException {
412     return getTableRegionsAndLocations(catalogTracker, Bytes.toBytes(tableName),
413       true);
414   }
415 
416   /**
417    * @param catalogTracker
418    * @param tableName
419    * @return Return list of regioninfos and server addresses.
420    * @throws IOException
421    * @throws InterruptedException
422    */
423   public static List<Pair<HRegionInfo, ServerName>>
424   getTableRegionsAndLocations(final CatalogTracker catalogTracker,
425       final byte [] tableName, final boolean excludeOfflinedSplitParents)
426   throws IOException, InterruptedException {
427     if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
428       // If meta, do a bit of special handling.
429       ServerName serverName = catalogTracker.getMetaLocation();
430       List<Pair<HRegionInfo, ServerName>> list =
431           new ArrayList<Pair<HRegionInfo, ServerName>>();
432       list.add(new Pair<HRegionInfo, ServerName>(HRegionInfo.FIRST_META_REGIONINFO,
433           serverName));
434       return list;
435     }
436     // Make a version of CollectingVisitor that collects HRegionInfo and ServerAddress
437     CollectingVisitor<Pair<HRegionInfo, ServerName>> visitor =
438         new CollectingVisitor<Pair<HRegionInfo, ServerName>>() {
439       private Pair<HRegionInfo, ServerName> current = null;
440 
441       @Override
442       public boolean visit(Result r) throws IOException {
443         HRegionInfo hri =
444           HRegionInfo.getHRegionInfo(r, HConstants.REGIONINFO_QUALIFIER);
445         if (hri == null) {
446           LOG.warn("No serialized HRegionInfo in " + r);
447           return true;
448         }
449         if (!isInsideTable(hri, tableName)) return false;
450         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
451         ServerName sn = HRegionInfo.getServerName(r);
452         // Populate this.current so available when we call #add
453         this.current = new Pair<HRegionInfo, ServerName>(hri, sn);
454         // Else call super and add this Result to the collection.
455         return super.visit(r);
456       }
457 
458       @Override
459       void add(Result r) {
460         this.results.add(this.current);
461       }
462     };
463     fullScan(catalogTracker, visitor, getTableStartRowForMeta(tableName));
464     return visitor.getResults();
465   }
466 
467   /**
468    * @param catalogTracker
469    * @param serverName
470    * @return List of user regions installed on this server (does not include
471    * catalog regions).
472    * @throws IOException
473    */
474   public static NavigableMap<HRegionInfo, Result>
475   getServerUserRegions(CatalogTracker catalogTracker, final ServerName serverName)
476   throws IOException {
477     final NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
478     // Fill the above hris map with entries from .META. that have the passed
479     // servername.
480     CollectingVisitor<Result> v = new CollectingVisitor<Result>() {
481       @Override
482       void add(Result r) {
483         if (r == null || r.isEmpty()) return;
484         ServerName sn = HRegionInfo.getServerName(r);
485         if (sn != null && sn.equals(serverName)) this.results.add(r);
486       }
487     };
488     fullScan(catalogTracker, v);
489     List<Result> results = v.getResults();
490     if (results != null && !results.isEmpty()) {
491       // Convert results to Map keyed by HRI
492       for (Result r: results) {
493         Pair<HRegionInfo, ServerName> p = HRegionInfo.getHRegionInfoAndServerName(r);
494         if (p != null && p.getFirst() != null) hris.put(p.getFirst(), r);
495       }
496     }
497     return hris;
498   }
499 
500   public static void fullScanMetaAndPrint(final CatalogTracker catalogTracker)
501   throws IOException {
502     Visitor v = new Visitor() {
503       @Override
504       public boolean visit(Result r) throws IOException {
505         if (r ==  null || r.isEmpty()) return true;
506         LOG.info("fullScanMetaAndPrint.Current Meta Row: " + r);
507         HRegionInfo hrim = HRegionInfo.getHRegionInfo(r);
508         LOG.info("fullScanMetaAndPrint.HRI Print= " + hrim);
509         return true;
510       }
511     };
512     fullScan(catalogTracker, v);
513   }
514 
515   /**
516    * Performs a full scan of a catalog table.
517    * @param catalogTracker
518    * @param visitor Visitor invoked against each row.
519    * @param startrow Where to start the scan. Pass null if want to begin scan
520    * at first row.
521    * <code>.META.</code>, the default (pass false to scan .META.)
522    * @throws IOException
523    */
524   public static void fullScan(CatalogTracker catalogTracker,
525     final Visitor visitor, final byte [] startrow)
526   throws IOException {
527     Scan scan = new Scan();
528     if (startrow != null) scan.setStartRow(startrow);
529     if (startrow == null) {
530       int caching = catalogTracker.getConnection().getConfiguration()
531           .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
532       scan.setCaching(caching);
533     }
534     scan.addFamily(HConstants.CATALOG_FAMILY);
535     HTable metaTable = getMetaHTable(catalogTracker);
536     ResultScanner scanner = metaTable.getScanner(scan);
537     try {
538       Result data;
539       while((data = scanner.next()) != null) {
540         if (data.isEmpty()) continue;
541         // Break if visit returns false.
542         if (!visitor.visit(data)) break;
543       }
544     } finally {
545       scanner.close();
546       metaTable.close();
547     }
548     return;
549   }
550 
551   /**
552    * Implementations 'visit' a catalog table row.
553    */
554   public interface Visitor {
555     /**
556      * Visit the catalog table row.
557      * @param r A row from catalog table
558      * @return True if we are to proceed scanning the table, else false if
559      * we are to stop now.
560      */
561     public boolean visit(final Result r) throws IOException;
562   }
563 
564   /**
565    * A {@link Visitor} that collects content out of passed {@link Result}.
566    */
567   static abstract class CollectingVisitor<T> implements Visitor {
568     final List<T> results = new ArrayList<T>();
569     @Override
570     public boolean visit(Result r) throws IOException {
571       if (r ==  null || r.isEmpty()) return true;
572       add(r);
573       return true;
574     }
575 
576     abstract void add(Result r);
577 
578     /**
579      * @return Collected results; wait till visits complete to collect all
580      * possible results
581      */
582     List<T> getResults() {
583       return this.results;
584     }
585   }
586 
587   /**
588    * Collects all returned.
589    */
590   static class CollectAllVisitor extends CollectingVisitor<Result> {
591     @Override
592     void add(Result r) {
593       this.results.add(r);
594     }
595   }
596 
597   /**
598    * Count regions in <code>.META.</code> for passed table.
599    * @param c
600    * @param tableName
601    * @return Count or regions in table <code>tableName</code>
602    * @throws IOException
603    */
604   public static int getRegionCount(final Configuration c, final String tableName) throws IOException {
605     HTable t = new HTable(c, tableName);
606     try {
607       return t.getRegionLocations().size();
608     } finally {
609       t.close();
610     }
611   }
612 }