View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.Closeable;
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.NavigableMap;
27  import java.util.TreeMap;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.classification.InterfaceAudience;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.hbase.TableName;
34  import org.apache.hadoop.hbase.HConstants;
35  import org.apache.hadoop.hbase.HRegionInfo;
36  import org.apache.hadoop.hbase.ServerName;
37  import org.apache.hadoop.hbase.TableNotFoundException;
38  import org.apache.hadoop.hbase.util.Bytes;
39  
40  /**
41   * Scanner class that contains the <code>.META.</code> table scanning logic.
42   * Provided visitors will be called for each row.
43   *
44   * Although public visibility, this is not a public-facing API and may evolve in
45   * minor releases.
46   *
47   * <p> Note that during concurrent region splits, the scanner might not see
48   * META changes across rows (for parent and daughter entries) consistently.
49   * see HBASE-5986, and {@link DefaultMetaScannerVisitor} for details. </p>
50   */
51  @InterfaceAudience.Private
52  public class MetaScanner {
53    private static final Log LOG = LogFactory.getLog(MetaScanner.class);
54    /**
55     * Scans the meta table and calls a visitor on each RowResult and uses a empty
56     * start row value as table name.
57     *
58     * @param configuration conf
59     * @param visitor A custom visitor
60     * @throws IOException e
61     */
62    public static void metaScan(Configuration configuration,
63        MetaScannerVisitor visitor)
64    throws IOException {
65      metaScan(configuration, visitor, null, null, Integer.MAX_VALUE);
66    }
67  
68    /**
69     * Scans the meta table and calls a visitor on each RowResult. Uses a table
70     * name to locate meta regions.
71     *
72     * @param configuration config
73     * @param connection connection to use internally (null to use a new instance)
74     * @param visitor visitor object
75     * @param userTableName User table name in meta table to start scan at.  Pass
76     * null if not interested in a particular table.
77     * @throws IOException e
78     */
79    public static void metaScan(Configuration configuration, HConnection connection,
80        MetaScannerVisitor visitor, TableName userTableName)
81    throws IOException {
82      metaScan(configuration, connection, visitor, userTableName, null, Integer.MAX_VALUE,
83          TableName.META_TABLE_NAME);
84    }
85  
86    /**
87     * Scans the meta table and calls a visitor on each RowResult. Uses a table
88     * name and a row name to locate meta regions. And it only scans at most
89     * <code>rowLimit</code> of rows.
90     *
91     * @param configuration HBase configuration.
92     * @param visitor Visitor object.
93     * @param userTableName User table name in meta table to start scan at.  Pass
94     * null if not interested in a particular table.
95     * @param row Name of the row at the user table. The scan will start from
96     * the region row where the row resides.
97     * @param rowLimit Max of processed rows. If it is less than 0, it
98     * will be set to default value <code>Integer.MAX_VALUE</code>.
99     * @throws IOException e
100    */
101   public static void metaScan(Configuration configuration,
102       MetaScannerVisitor visitor, TableName userTableName, byte[] row,
103       int rowLimit)
104   throws IOException {
105     metaScan(configuration, null, visitor, userTableName, row, rowLimit,
106       TableName.META_TABLE_NAME);
107   }
108 
109   /**
110    * Scans the meta table and calls a visitor on each RowResult. Uses a table
111    * name and a row name to locate meta regions. And it only scans at most
112    * <code>rowLimit</code> of rows.
113    *
114    * @param configuration HBase configuration.
115    * @param connection connection to use internally (null to use a new instance)
116    * @param visitor Visitor object. Closes the visitor before returning.
117    * @param tableName User table name in meta table to start scan at.  Pass
118    * null if not interested in a particular table.
119    * @param row Name of the row at the user table. The scan will start from
120    * the region row where the row resides.
121    * @param rowLimit Max of processed rows. If it is less than 0, it
122    * will be set to default value <code>Integer.MAX_VALUE</code>.
123    * @param metaTableName Meta table to scan, root or meta.
124    * @throws IOException e
125    */
126   public static void metaScan(Configuration configuration, HConnection connection,
127       final MetaScannerVisitor visitor, final TableName tableName,
128       final byte[] row, final int rowLimit, final TableName metaTableName)
129   throws IOException {
130     int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
131     HTable metaTable;
132     if (connection == null) {
133       metaTable = new HTable(configuration, TableName.META_TABLE_NAME, null);
134     } else {
135       metaTable = new HTable(TableName.META_TABLE_NAME, connection, null);
136     }
137     // Calculate startrow for scan.
138     byte[] startRow;
139     ResultScanner scanner = null;
140     try {
141       if (row != null) {
142         // Scan starting at a particular row in a particular table
143         byte[] searchRow = HRegionInfo.createRegionName(tableName, row, HConstants.NINES, false);
144         Result startRowResult = metaTable.getRowOrBefore(searchRow, HConstants.CATALOG_FAMILY);
145         if (startRowResult == null) {
146           throw new TableNotFoundException("Cannot find row in "+ TableName
147               .META_TABLE_NAME.getNameAsString()+" for table: "
148               + tableName + ", row=" + Bytes.toStringBinary(searchRow));
149         }
150         HRegionInfo regionInfo = getHRegionInfo(startRowResult);
151         if (regionInfo == null) {
152           throw new IOException("HRegionInfo was null or empty in Meta for " +
153             tableName + ", row=" + Bytes.toStringBinary(searchRow));
154         }
155         byte[] rowBefore = regionInfo.getStartKey();
156         startRow = HRegionInfo.createRegionName(tableName, rowBefore, HConstants.ZEROES, false);
157       } else if (tableName == null || tableName.getName().length == 0) {
158         // Full META scan
159         startRow = HConstants.EMPTY_START_ROW;
160       } else {
161         // Scan META for an entire table
162         startRow = HRegionInfo.createRegionName(tableName, HConstants.EMPTY_START_ROW,
163           HConstants.ZEROES, false);
164       }
165       final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
166       int rows = Math.min(rowLimit, configuration.getInt(HConstants.HBASE_META_SCANNER_CACHING,
167         HConstants.DEFAULT_HBASE_META_SCANNER_CACHING));
168       scan.setCaching(rows);
169       if (LOG.isTraceEnabled()) {
170         LOG.trace("Scanning " + metaTableName.getNameAsString() + " starting at row=" +
171           Bytes.toStringBinary(startRow) + " for max=" + rowUpperLimit + " with caching=" + rows);
172       }
173       // Run the scan
174       scanner = metaTable.getScanner(scan);
175       Result result = null;
176       int processedRows = 0;
177       while ((result = scanner.next()) != null) {
178         if (visitor != null) {
179           if (!visitor.processRow(result)) break;
180         }
181         processedRows++;
182         if (processedRows >= rowUpperLimit) break;
183       }
184     } finally {
185       if (scanner != null) {
186         try {
187           scanner.close();
188         } catch (Throwable t) {
189           LOG.debug("Got exception in closing the result scanner", t);
190         }
191       }
192       if (visitor != null) {
193         try {
194           visitor.close();
195         } catch (Throwable t) {
196           LOG.debug("Got exception in closing the meta scanner visitor", t);
197         }
198       }
199       if (metaTable != null) {
200         try {
201           metaTable.close();
202         } catch (Throwable t) {
203           LOG.debug("Got exception in closing the meta table", t);
204         }
205       }
206     }
207   }
208 
209   /**
210    * Returns HRegionInfo object from the column
211    * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog
212    * table Result.
213    * @param data a Result object from the catalog table scan
214    * @return HRegionInfo or null
215    */
216   public static HRegionInfo getHRegionInfo(Result data) {
217     byte [] bytes =
218       data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
219     if (bytes == null) return null;
220     HRegionInfo info = HRegionInfo.parseFromOrNull(bytes);
221     if (LOG.isTraceEnabled()) {
222       LOG.trace("Current INFO from scan results = " + info);
223     }
224     return info;
225   }
226 
227   /**
228    * Used in tests.
229    *
230    * Lists all of the regions currently in META.
231    * @param conf
232    * @param offlined True if we are to include offlined regions, false and we'll
233    * leave out offlined regions from returned list.
234    * @return List of all user-space regions.
235    * @throws IOException
236    */
237   public static List<HRegionInfo> listAllRegions(Configuration conf, final boolean offlined)
238   throws IOException {
239     final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
240     MetaScannerVisitor visitor = new DefaultMetaScannerVisitor() {
241         @Override
242         public boolean processRowInternal(Result result) throws IOException {
243           if (result == null || result.isEmpty()) {
244             return true;
245           }
246 
247           HRegionInfo regionInfo = getHRegionInfo(result);
248           if (regionInfo == null) {
249             LOG.warn("Null REGIONINFO_QUALIFIER: " + result);
250             return true;
251           }
252 
253           // If region offline AND we are not to include offlined regions, return.
254           if (regionInfo.isOffline() && !offlined) return true;
255           regions.add(regionInfo);
256           return true;
257         }
258     };
259     metaScan(conf, visitor);
260     return regions;
261   }
262 
263   /**
264    * Lists all of the table regions currently in META.
265    * @param conf
266    * @param offlined True if we are to include offlined regions, false and we'll
267    * leave out offlined regions from returned list.
268    * @return Map of all user-space regions to servers
269    * @throws IOException
270    */
271   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
272       HConnection connection, final TableName tableName,
273       final boolean offlined) throws IOException {
274     final NavigableMap<HRegionInfo, ServerName> regions =
275       new TreeMap<HRegionInfo, ServerName>();
276     MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) {
277       @Override
278       public boolean processRowInternal(Result rowResult) throws IOException {
279         HRegionInfo info = getHRegionInfo(rowResult);
280         ServerName serverName = HRegionInfo.getServerName(rowResult);
281         regions.put(new UnmodifyableHRegionInfo(info), serverName);
282         return true;
283       }
284     };
285     metaScan(conf, connection, visitor, tableName);
286     return regions;
287   }
288 
289   /**
290    * Visitor class called to process each row of the .META. table
291    */
292   public interface MetaScannerVisitor extends Closeable {
293     /**
294      * Visitor method that accepts a RowResult and the meta region location.
295      * Implementations can return false to stop the region's loop if it becomes
296      * unnecessary for some reason.
297      *
298      * @param rowResult result
299      * @return A boolean to know if it should continue to loop in the region
300      * @throws IOException e
301      */
302     boolean processRow(Result rowResult) throws IOException;
303   }
304 
305   public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor {
306     @Override
307     public void close() throws IOException {
308     }
309   }
310 
311   /**
312    * A MetaScannerVisitor that skips offline regions and split parents
313    */
314   public static abstract class DefaultMetaScannerVisitor
315     extends MetaScannerVisitorBase {
316 
317     public DefaultMetaScannerVisitor() {
318       super();
319     }
320 
321     public abstract boolean processRowInternal(Result rowResult) throws IOException;
322 
323     @Override
324     public boolean processRow(Result rowResult) throws IOException {
325       HRegionInfo info = getHRegionInfo(rowResult);
326       if (info == null) {
327         return true;
328       }
329 
330       //skip over offline and split regions
331       if (!(info.isOffline() || info.isSplit())) {
332         return processRowInternal(rowResult);
333       }
334       return true;
335     }
336   }
337 
338   /**
339    * A MetaScannerVisitor for a table. Provides a consistent view of the table's
340    * META entries during concurrent splits (see HBASE-5986 for details). This class
341    * does not guarantee ordered traversal of meta entries, and can block until the
342    * META entries for daughters are available during splits.
343    */
344   public static abstract class TableMetaScannerVisitor extends DefaultMetaScannerVisitor {
345     private TableName tableName;
346 
347     public TableMetaScannerVisitor(TableName tableName) {
348       super();
349       this.tableName = tableName;
350     }
351 
352     @Override
353     public final boolean processRow(Result rowResult) throws IOException {
354       HRegionInfo info = getHRegionInfo(rowResult);
355       if (info == null) {
356         return true;
357       }
358       if (!(info.getTableName().equals(tableName))) {
359         return false;
360       }
361       return super.processRow(rowResult);
362     }
363   }
364 }