View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.catalog;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  import java.util.Map;
26  import java.util.NavigableMap;
27  import java.util.Set;
28  import java.util.TreeMap;
29  import java.util.TreeSet;
30  
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.HServerAddress;
34  import org.apache.hadoop.hbase.HServerInfo;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException;
37  import org.apache.hadoop.hbase.NotServingRegionException;
38  import org.apache.hadoop.hbase.client.Get;
39  import org.apache.hadoop.hbase.client.Result;
40  import org.apache.hadoop.hbase.client.Scan;
41  import org.apache.hadoop.hbase.ipc.HRegionInterface;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.Pair;
44  import org.apache.hadoop.hbase.util.Writables;
45  import org.apache.hadoop.ipc.RemoteException;
46  
47  /**
48   * Reads region and assignment information from <code>.META.</code>.
49   * <p>
50   * Uses the {@link CatalogTracker} to obtain locations and connections to
51   * catalogs.
52   */
53  public class MetaReader {
54    public static final byte [] META_REGION_PREFIX;
55    static {
56      // Copy the prefix from FIRST_META_REGIONINFO into META_REGION_PREFIX.
57      // FIRST_META_REGIONINFO == '.META.,,1'.  META_REGION_PREFIX == '.META.,'
58      int len = HRegionInfo.FIRST_META_REGIONINFO.getRegionName().length - 2;
59      META_REGION_PREFIX = new byte [len];
60      System.arraycopy(HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), 0,
61        META_REGION_PREFIX, 0, len);
62    }
63  
64    /**
65     * @param ct
66     * @param tableName A user tablename or a .META. table name.
67     * @return Interface on to server hosting the <code>-ROOT-</code> or
68     * <code>.META.</code> regions.
69     * @throws NotAllMetaRegionsOnlineException
70     * @throws IOException
71     */
72    private static HRegionInterface getCatalogRegionInterface(final CatalogTracker ct,
73        final byte [] tableName)
74    throws NotAllMetaRegionsOnlineException, IOException {
75      return Bytes.equals(HConstants.META_TABLE_NAME, tableName)?
76        ct.waitForRootServerConnectionDefault():
77        ct.waitForMetaServerConnectionDefault();
78    }
79  
80    /**
81     * @param tableName
82     * @return Returns region name to look in for regions for <code>tableName</code>;
83     * e.g. if we are looking for <code>.META.</code> regions, we need to look
84     * in the <code>-ROOT-</code> region, else if a user table, we need to look
85     * in the <code>.META.</code> region.
86     */
87    private static byte [] getCatalogRegionNameForTable(final byte [] tableName) {
88      return Bytes.equals(HConstants.META_TABLE_NAME, tableName)?
89        HRegionInfo.ROOT_REGIONINFO.getRegionName():
90        HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
91    }
92  
93    /**
94     * @param regionName
95     * @return Returns region name to look in for <code>regionName</code>;
96     * e.g. if we are looking for <code>.META.,,1</code> region, we need to look
97     * in <code>-ROOT-</code> region, else if a user region, we need to look
98     * in the <code>.META.,,1</code> region.
99     */
100   private static byte [] getCatalogRegionNameForRegion(final byte [] regionName) {
101     return isMetaRegion(regionName)?
102       HRegionInfo.ROOT_REGIONINFO.getRegionName():
103       HRegionInfo.FIRST_META_REGIONINFO.getRegionName();
104   }
105 
106   /**
107    * @param regionName
108    * @return True if <code>regionName</code> is from <code>.META.</code> table.
109    */
110   private static boolean isMetaRegion(final byte [] regionName) {
111     if (regionName.length < META_REGION_PREFIX.length + 2 /* ',', + '1' */) {
112       // Can't be meta table region.
113       return false;
114     }
115     // Compare the prefix of regionName.  If it matches META_REGION_PREFIX prefix,
116     // then this is region from .META. table.
117     return Bytes.compareTo(regionName, 0, META_REGION_PREFIX.length,
118       META_REGION_PREFIX, 0, META_REGION_PREFIX.length) == 0;
119   }
120 
121   /**
122    * Performs a full scan of <code>.META.</code>.
123    * <p>
124    * Returns a map of every region to it's currently assigned server, according
125    * to META.  If the region does not have an assignment it will have a null
126    * value in the map.
127    *
128    * @return map of regions to their currently assigned server
129    * @throws IOException
130    */
131   public static Map<HRegionInfo,HServerAddress> fullScan(
132       CatalogTracker catalogTracker)
133   throws IOException {
134     return fullScan(catalogTracker, new TreeSet<String>());
135   }
136 
137   /**
138    * Performs a full scan of <code>.META.</code>, skipping regions from any
139    * tables in the specified set of disabled tables.
140    * <p>
141    * Returns a map of every region to it's currently assigned server, according
142    * to META.  If the region does not have an assignment it will have a null
143    * value in the map.
144    *
145    * @param catalogTracker
146    * @param disabledTables set of disabled tables that will not be returned
147    * @return map of regions to their currently assigned server
148    * @throws IOException
149    */
150   public static Map<HRegionInfo,HServerAddress> fullScan(
151       CatalogTracker catalogTracker, final Set<String> disabledTables)
152   throws IOException {
153     return fullScan(catalogTracker, disabledTables, false);
154   }
155 
156   /**
157    * Performs a full scan of <code>.META.</code>, skipping regions from any
158    * tables in the specified set of disabled tables.
159    * <p>
160    * Returns a map of every region to it's currently assigned server, according
161    * to META.  If the region does not have an assignment it will have a null
162    * value in the map.
163    *
164    * @param catalogTracker
165    * @param disabledTables set of disabled tables that will not be returned
166    * @param excludeOfflinedSplitParents If true, do not include offlined split
167    * parents in the return.
168    * @return map of regions to their currently assigned server
169    * @throws IOException
170    */
171   public static Map<HRegionInfo,HServerAddress> fullScan(
172       CatalogTracker catalogTracker, final Set<String> disabledTables,
173       final boolean excludeOfflinedSplitParents)
174   throws IOException {
175     final Map<HRegionInfo,HServerAddress> regions =
176       new TreeMap<HRegionInfo,HServerAddress>();
177     Visitor v = new Visitor() {
178       @Override
179       public boolean visit(Result r) throws IOException {
180         if (r ==  null || r.isEmpty()) return true;
181         Pair<HRegionInfo,HServerAddress> region = metaRowToRegionPair(r);
182         if (region == null) return true;
183         HRegionInfo hri = region.getFirst();
184         if (disabledTables.contains(
185             hri.getTableDesc().getNameAsString())) return true;
186         // Are we to include split parents in the list?
187         if (excludeOfflinedSplitParents && hri.isSplitParent()) return true;
188         regions.put(hri, region.getSecond());
189         return true;
190       }
191     };
192     fullScan(catalogTracker, v);
193     return regions;
194   }
195 
196   /**
197    * Performs a full scan of <code>.META.</code>.
198    * <p>
199    * Returns a map of every region to it's currently assigned server, according
200    * to META.  If the region does not have an assignment it will have a null
201    * value in the map.
202    * <p>
203    * Returns HServerInfo which includes server startcode.
204    *
205    * @return map of regions to their currently assigned server
206    * @throws IOException
207    */
208   public static List<Result> fullScanOfResults(
209       CatalogTracker catalogTracker)
210   throws IOException {
211     final List<Result> regions = new ArrayList<Result>();
212     Visitor v = new Visitor() {
213       @Override
214       public boolean visit(Result r) throws IOException {
215         if (r ==  null || r.isEmpty()) return true;
216         regions.add(r);
217         return true;
218       }
219     };
220     fullScan(catalogTracker, v);
221     return regions;
222   }
223 
224   /**
225    * Performs a full scan of <code>.META.</code>.
226    * <p>
227    * Returns a map of every region to it's currently assigned server, according
228    * to META.  If the region does not have an assignment it will have a null
229    * value in the map.
230    * @param catalogTracker
231    * @param visitor
232    * @throws IOException
233    */
234   public static void fullScan(CatalogTracker catalogTracker,
235       final Visitor visitor)
236   throws IOException {
237     fullScan(catalogTracker, visitor, null);
238   }
239 
240   /**
241    * Performs a full scan of <code>.META.</code>.
242    * <p>
243    * Returns a map of every region to it's currently assigned server, according
244    * to META.  If the region does not have an assignment it will have a null
245    * value in the map.
246    * @param catalogTracker
247    * @param visitor
248    * @param startrow Where to start the scan. Pass null if want to begin scan
249    * at first row.
250    * @throws IOException
251    */
252   public static void fullScan(CatalogTracker catalogTracker,
253       final Visitor visitor, final byte [] startrow)
254   throws IOException {
255     HRegionInterface metaServer =
256       catalogTracker.waitForMetaServerConnectionDefault();
257     Scan scan = new Scan();
258     if (startrow != null) scan.setStartRow(startrow);
259     scan.addFamily(HConstants.CATALOG_FAMILY);
260     long scannerid = metaServer.openScanner(
261         HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
262     try {
263       Result data;
264       while((data = metaServer.next(scannerid)) != null) {
265         if (!data.isEmpty()) visitor.visit(data);
266       }
267     } finally {
268       metaServer.close(scannerid);
269     }
270     return;
271   }
272 
273   /**
274    * Reads the location of META from ROOT.
275    * @param metaServer connection to server hosting ROOT
276    * @return location of META in ROOT, null if not available
277    * @throws IOException
278    */
279   public static HServerAddress readMetaLocation(HRegionInterface metaServer)
280   throws IOException {
281     return readLocation(metaServer, CatalogTracker.ROOT_REGION,
282         CatalogTracker.META_REGION);
283   }
284 
285   /**
286    * Reads the location of the specified region from META.
287    * @param catalogTracker
288    * @param regionName region to read location of
289    * @return location of region in META, null if not available
290    * @throws IOException
291    */
292   public static HServerAddress readRegionLocation(CatalogTracker catalogTracker,
293       byte [] regionName)
294   throws IOException {
295     if (isMetaRegion(regionName)) throw new IllegalArgumentException("See readMetaLocation");
296     return readLocation(catalogTracker.waitForMetaServerConnectionDefault(),
297         CatalogTracker.META_REGION, regionName);
298   }
299 
300   private static HServerAddress readLocation(HRegionInterface metaServer,
301       byte [] catalogRegionName, byte [] regionName)
302   throws IOException {
303     Result r = null;
304     try {
305       r = metaServer.get(catalogRegionName,
306         new Get(regionName).addColumn(HConstants.CATALOG_FAMILY,
307         HConstants.SERVER_QUALIFIER));
308     } catch (java.net.SocketTimeoutException e) {
309       // Treat this exception + message as unavailable catalog table. Catch it
310       // and fall through to return a null
311     } catch (java.net.SocketException e) {
312       // Treat this exception + message as unavailable catalog table. Catch it
313       // and fall through to return a null
314     } catch (RemoteException re) {
315       IOException ioe = re.unwrapRemoteException();
316       if (ioe instanceof NotServingRegionException) {
317         // Treat this NSRE as unavailable table.  Catch and fall through to
318         // return null below
319       } else if (ioe.getMessage().contains("Server not running")) {
320         // Treat as unavailable table.
321       } else {
322         throw re;
323       }
324     } catch (IOException e) {
325       if (e.getCause() != null && e.getCause() instanceof IOException &&
326           e.getCause().getMessage() != null &&
327           e.getCause().getMessage().contains("Connection reset by peer")) {
328         // Treat this exception + message as unavailable catalog table. Catch it
329         // and fall through to return a null
330       } else {
331         throw e;
332       }
333     }
334     if (r == null || r.isEmpty()) {
335       return null;
336     }
337     byte [] value = r.getValue(HConstants.CATALOG_FAMILY,
338       HConstants.SERVER_QUALIFIER);
339     return new HServerAddress(Bytes.toString(value));
340   }
341 
342   /**
343    * Gets the region info and assignment for the specified region from META.
344    * @param catalogTracker
345    * @param regionName
346    * @return region info and assignment from META, null if not available
347    * @throws IOException
348    */
349   public static Pair<HRegionInfo, HServerAddress> getRegion(
350       CatalogTracker catalogTracker, byte [] regionName)
351   throws IOException {
352     Get get = new Get(regionName);
353     get.addFamily(HConstants.CATALOG_FAMILY);
354     byte [] meta = getCatalogRegionNameForRegion(regionName);
355     Result r = catalogTracker.waitForMetaServerConnectionDefault().get(meta, get);
356     if(r == null || r.isEmpty()) {
357       return null;
358     }
359     return metaRowToRegionPair(r);
360   }
361 
362   /**
363    * @param data A .META. table row.
364    * @return A pair of the regioninfo and the server address from <code>data</code>
365    * or null for server address if no address set in .META. or null for a result
366    * if no HRegionInfo found.
367    * @throws IOException
368    */
369   public static Pair<HRegionInfo, HServerAddress> metaRowToRegionPair(
370       Result data) throws IOException {
371     byte [] bytes =
372       data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
373     if (bytes == null) return null;
374     HRegionInfo info = Writables.getHRegionInfo(bytes);
375     final byte[] value = data.getValue(HConstants.CATALOG_FAMILY,
376       HConstants.SERVER_QUALIFIER);
377     if (value != null && value.length > 0) {
378       HServerAddress server = new HServerAddress(Bytes.toString(value));
379       return new Pair<HRegionInfo,HServerAddress>(info, server);
380     } else {
381       return new Pair<HRegionInfo, HServerAddress>(info, null);
382     }
383   }
384 
385   /**
386    * @param data A .META. table row.
387    * @return A pair of the regioninfo and the server info from <code>data</code>
388    * (or null for server address if no address set in .META.).
389    * @throws IOException
390    */
391   public static Pair<HRegionInfo, HServerInfo> metaRowToRegionPairWithInfo(
392       Result data) throws IOException {
393     byte [] bytes = data.getValue(HConstants.CATALOG_FAMILY,
394       HConstants.REGIONINFO_QUALIFIER);
395     if (bytes == null) return null;
396     HRegionInfo info = Writables.getHRegionInfo(bytes);
397     final byte[] value = data.getValue(HConstants.CATALOG_FAMILY,
398       HConstants.SERVER_QUALIFIER);
399     if (value != null && value.length > 0) {
400       final long startCode = Bytes.toLong(data.getValue(HConstants.CATALOG_FAMILY,
401           HConstants.STARTCODE_QUALIFIER));
402       HServerAddress server = new HServerAddress(Bytes.toString(value));
403       HServerInfo hsi = new HServerInfo(server, startCode, 0,
404           server.getHostname());
405       return new Pair<HRegionInfo,HServerInfo>(info, hsi);
406     } else {
407       return new Pair<HRegionInfo, HServerInfo>(info, null);
408     }
409   }
410 
411   /**
412    * Checks if the specified table exists.  Looks at the META table hosted on
413    * the specified server.
414    * @param catalogTracker
415    * @param tableName table to check
416    * @return true if the table exists in meta, false if not
417    * @throws IOException
418    */
419   public static boolean tableExists(CatalogTracker catalogTracker,
420       String tableName)
421   throws IOException {
422     if (tableName.equals(HTableDescriptor.ROOT_TABLEDESC.getNameAsString()) ||
423         tableName.equals(HTableDescriptor.META_TABLEDESC.getNameAsString())) {
424       // Catalog tables always exist.
425       return true;
426     }
427     HRegionInterface metaServer =
428       catalogTracker.waitForMetaServerConnectionDefault();
429     Scan scan = getScanForTableName(Bytes.toBytes(tableName));
430     scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
431     long scannerid = metaServer.openScanner(
432         HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
433     try {
434       Result data = metaServer.next(scannerid);
435       if (data != null && data.size() > 0) {
436           return true;
437       }
438       return false;
439     } finally {
440       metaServer.close(scannerid);
441     }
442   }
443 
444   /**
445    * Gets all of the regions of the specified table.
446    * @param catalogTracker
447    * @param tableName
448    * @return Ordered list of {@link HRegionInfo}.
449    * @throws IOException
450    */
451   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
452       byte [] tableName)
453   throws IOException {
454     return getTableRegions(catalogTracker, tableName, false);
455   }
456 
457   /**
458    * Gets all of the regions of the specified table.
459    * @param catalogTracker
460    * @param tableName
461    * @param excludeOfflinedSplitParents If true, do not include offlined split
462    * parents in the return.
463    * @return Ordered list of {@link HRegionInfo}.
464    * @throws IOException
465    */
466   public static List<HRegionInfo> getTableRegions(CatalogTracker catalogTracker,
467       byte [] tableName, final boolean excludeOfflinedSplitParents)
468   throws IOException {
469     if (Bytes.equals(tableName, HConstants.ROOT_TABLE_NAME)) {
470       // If root, do a bit of special handling.
471       List<HRegionInfo> list = new ArrayList<HRegionInfo>();
472       list.add(HRegionInfo.ROOT_REGIONINFO);
473       return list;
474     } else if (Bytes.equals(tableName, HConstants.META_TABLE_NAME)) {
475       // Same for .META. table
476       List<HRegionInfo> list = new ArrayList<HRegionInfo>();
477       list.add(HRegionInfo.FIRST_META_REGIONINFO);
478       return list;
479     }
480 
481     // Its a user table.
482     HRegionInterface metaServer =
483       getCatalogRegionInterface(catalogTracker, tableName);
484     List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
485 
486     Scan scan = getScanForTableName(tableName);
487     scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
488     long scannerid =
489       metaServer.openScanner(getCatalogRegionNameForTable(tableName), scan);
490     try {
491       Result data;
492       while((data = metaServer.next(scannerid)) != null) {
493         if (data != null && data.size() > 0) {
494           HRegionInfo info = Writables.getHRegionInfo(
495               data.getValue(HConstants.CATALOG_FAMILY,
496                   HConstants.REGIONINFO_QUALIFIER));
497           if (excludeOfflinedSplitParents && info.isSplitParent()) continue;
498           regions.add(info);
499         }
500       }
501       return regions;
502     } finally {
503       metaServer.close(scannerid);
504     }
505   }
506 
507   /**
508    * This method creates a Scan object that will only scan catalog rows that
509    * belong to the specified table. It doesn't specify any columns.
510    * This is a better alternative to just using a start row and scan until
511    * it hits a new table since that requires parsing the HRI to get the table
512    * name.
513    * @param tableName bytes of table's name
514    * @return configured Scan object
515    */
516   public static Scan getScanForTableName(byte[] tableName) {
517     String strName = Bytes.toString(tableName);
518     // Start key is just the table name with delimiters
519     byte[] startKey = Bytes.toBytes(strName + ",,");
520     // Stop key appends the smallest possible char to the table name
521     byte[] stopKey = Bytes.toBytes(strName + " ,,");
522 
523     Scan scan = new Scan(startKey);
524     scan.setStopRow(stopKey);
525     return scan;
526   }
527 
528   /**
529    * @param catalogTracker
530    * @param tableName
531    * @return Return list of regioninfos and server addresses.
532    * @throws IOException
533    * @throws InterruptedException
534    */
535   public static List<Pair<HRegionInfo, HServerAddress>>
536   getTableRegionsAndLocations(CatalogTracker catalogTracker, String tableName)
537   throws IOException, InterruptedException {
538     byte [] tableNameBytes = Bytes.toBytes(tableName);
539     if (Bytes.equals(tableNameBytes, HConstants.ROOT_TABLE_NAME)) {
540       // If root, do a bit of special handling.
541       HServerAddress hsa = catalogTracker.getRootLocation();
542       List<Pair<HRegionInfo, HServerAddress>> list =
543         new ArrayList<Pair<HRegionInfo, HServerAddress>>();
544       list.add(new Pair<HRegionInfo, HServerAddress>(HRegionInfo.ROOT_REGIONINFO, hsa));
545       return list;
546     }
547     HRegionInterface metaServer =
548       getCatalogRegionInterface(catalogTracker, tableNameBytes);
549     List<Pair<HRegionInfo, HServerAddress>> regions =
550       new ArrayList<Pair<HRegionInfo, HServerAddress>>();
551     Scan scan = getScanForTableName(tableNameBytes);
552     scan.addFamily(HConstants.CATALOG_FAMILY);
553     long scannerid =
554       metaServer.openScanner(getCatalogRegionNameForTable(tableNameBytes), scan);
555     try {
556       Result data;
557       while((data = metaServer.next(scannerid)) != null) {
558         if (data != null && data.size() > 0) {
559           Pair<HRegionInfo, HServerAddress> region = metaRowToRegionPair(data);
560           if (region == null) continue;
561           regions.add(region);
562         }
563       }
564       return regions;
565     } finally {
566       metaServer.close(scannerid);
567     }
568   }
569 
570   /**
571    * @param catalogTracker
572    * @param hsi Server specification
573    * @return List of user regions installed on this server (does not include
574    * catalog regions).
575    * @throws IOException
576    */
577   public static NavigableMap<HRegionInfo, Result>
578   getServerUserRegions(CatalogTracker catalogTracker, final HServerInfo hsi)
579   throws IOException {
580     HRegionInterface metaServer =
581       catalogTracker.waitForMetaServerConnectionDefault();
582     NavigableMap<HRegionInfo, Result> hris = new TreeMap<HRegionInfo, Result>();
583     Scan scan = new Scan();
584     scan.addFamily(HConstants.CATALOG_FAMILY);
585     long scannerid = metaServer.openScanner(
586         HRegionInfo.FIRST_META_REGIONINFO.getRegionName(), scan);
587     try {
588       Result result;
589       while((result = metaServer.next(scannerid)) != null) {
590         if (result != null && result.size() > 0) {
591           Pair<HRegionInfo, HServerInfo> pair =
592             metaRowToRegionPairWithInfo(result);
593           if (pair == null) continue;
594           if (pair.getSecond() == null || !pair.getSecond().equals(hsi)) {
595             continue;
596           }
597           hris.put(pair.getFirst(), result);
598         }
599       }
600       return hris;
601     } finally {
602       metaServer.close(scannerid);
603     }
604   }
605 
606   /**
607    * Implementations 'visit' a catalog table row.
608    */
609   public interface Visitor {
610     /**
611      * Visit the catalog table row.
612      * @param r A row from catalog table
613      * @return True if we are to proceed scanning the table, else false if
614      * we are to stop now.
615      */
616     public boolean visit(final Result r) throws IOException;
617   }
618 }