View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.Closeable;
23  import java.io.IOException;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.NavigableMap;
27  import java.util.TreeMap;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.hbase.classification.InterfaceAudience;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.hbase.HConstants;
34  import org.apache.hadoop.hbase.HRegionInfo;
35  import org.apache.hadoop.hbase.HRegionLocation;
36  import org.apache.hadoop.hbase.MetaTableAccessor;
37  import org.apache.hadoop.hbase.RegionLocations;
38  import org.apache.hadoop.hbase.ServerName;
39  import org.apache.hadoop.hbase.TableName;
40  import org.apache.hadoop.hbase.TableNotFoundException;
41  import org.apache.hadoop.hbase.util.Bytes;
42  import org.apache.hadoop.hbase.util.ExceptionUtil;
43  
44  import com.google.common.annotations.VisibleForTesting;
45  
46  /**
47   * Scanner class that contains the <code>hbase:meta</code> table scanning logic.
48   * Provided visitors will be called for each row.
49   *
50   * Although public visibility, this is not a public-facing API and may evolve in
51   * minor releases.
52   *
53   * <p> Note that during concurrent region splits, the scanner might not see
54   * hbase:meta changes across rows (for parent and daughter entries) consistently.
55   * see HBASE-5986, and {@link DefaultMetaScannerVisitor} for details. </p>
56   */
57  @InterfaceAudience.Private
58  //TODO: merge this to MetaTableAccessor, get rid of it.
59  public class MetaScanner {
60    private static final Log LOG = LogFactory.getLog(MetaScanner.class);
61    /**
62     * Scans the meta table and calls a visitor on each RowResult and uses a empty
63     * start row value as table name.
64     * 
65     * <p>Visible for testing. Use {@link
66     * #metaScan(Connection, MetaScannerVisitor, TableName)} instead.
67     *
68     * @param visitor A custom visitor
69     * @throws IOException e
70     */
71    @VisibleForTesting // Do not use. Used by tests only and hbck.
72    public static void metaScan(Connection connection,
73        MetaScannerVisitor visitor) throws IOException {
74      metaScan(connection, visitor, null, null, Integer.MAX_VALUE);
75    }
76  
77    /**
78     * Scans the meta table and calls a visitor on each RowResult. Uses a table
79     * name to locate meta regions.
80     *
81     * @param connection connection to use internally (null to use a new instance)
82     * @param visitor visitor object
83     * @param userTableName User table name in meta table to start scan at.  Pass
84     * null if not interested in a particular table.
85     * @throws IOException e
86     */
87    public static void metaScan(Connection connection,
88        MetaScannerVisitor visitor, TableName userTableName) throws IOException {
89      metaScan(connection, visitor, userTableName, null, Integer.MAX_VALUE,
90          TableName.META_TABLE_NAME);
91    }
92  
93    /**
94     * Scans the meta table and calls a visitor on each RowResult. Uses a table
95     * name and a row name to locate meta regions. And it only scans at most
96     * <code>rowLimit</code> of rows.
97     * 
98     * <p>Visible for testing. Use {@link
99     * #metaScan(Connection, MetaScannerVisitor, TableName)} instead.
100    *
101    * @param connection to scan on
102    * @param visitor Visitor object.
103    * @param userTableName User table name in meta table to start scan at.  Pass
104    * null if not interested in a particular table.
105    * @param row Name of the row at the user table. The scan will start from
106    * the region row where the row resides.
107    * @param rowLimit Max of processed rows. If it is less than 0, it
108    * will be set to default value <code>Integer.MAX_VALUE</code>.
109    * @throws IOException e
110    */
111   @VisibleForTesting // Do not use. Used by Master but by a method that is used testing.
112   public static void metaScan(Connection connection,
113       MetaScannerVisitor visitor, TableName userTableName, byte[] row,
114       int rowLimit)
115   throws IOException {
116     metaScan(connection, visitor, userTableName, row, rowLimit, TableName
117         .META_TABLE_NAME);
118   }
119 
120   /**
121    * Scans the meta table and calls a visitor on each RowResult. Uses a table
122    * name and a row name to locate meta regions. And it only scans at most
123    * <code>rowLimit</code> of rows.
124    *
125    * @param connection connection to use internally (null to use a new instance)
126    * @param visitor Visitor object. Closes the visitor before returning.
127    * @param tableName User table name in meta table to start scan at.  Pass
128    * null if not interested in a particular table.
129    * @param row Name of the row at the user table. The scan will start from
130    * the region row where the row resides.
131    * @param rowLimit Max of processed rows. If it is less than 0, it
132    * will be set to default value <code>Integer.MAX_VALUE</code>.
133    * @param metaTableName Meta table to scan, root or meta.
134    * @throws IOException e
135    */
136   static void metaScan(Connection connection,
137       final MetaScannerVisitor visitor, final TableName tableName,
138       final byte[] row, final int rowLimit, final TableName metaTableName)
139     throws IOException {
140 
141     int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
142     // Calculate startrow for scan.
143     byte[] startRow;
144     // If the passed in 'connection' is 'managed' -- i.e. every second test uses
145     // an HTable or an HBaseAdmin with managed connections -- then doing
146     // connection.getTable will throw an exception saying you are NOT to use
147     // managed connections getting tables.  Leaving this as it is for now. Will
148     // revisit when inclined to change all tests.  User code probaby makes use of
149     // managed connections too so don't change it till post hbase 1.0.
150     try (Table metaTable = new HTable(TableName.META_TABLE_NAME, connection, null)) {
151       if (row != null) {
152         // Scan starting at a particular row in a particular table
153         Result startRowResult = getClosestRowOrBefore(metaTable, tableName, row);
154         if (startRowResult == null) {
155           throw new TableNotFoundException("Cannot find row in " + metaTable.getName() +
156             " for table: " + tableName + ", row=" + Bytes.toStringBinary(row));
157         }
158         HRegionInfo regionInfo = getHRegionInfo(startRowResult);
159         if (regionInfo == null) {
160           throw new IOException("HRegionInfo was null or empty in Meta for " +
161             tableName + ", row=" + Bytes.toStringBinary(row));
162         }
163         byte[] rowBefore = regionInfo.getStartKey();
164         startRow = HRegionInfo.createRegionName(tableName, rowBefore, HConstants.ZEROES, false);
165       } else if (tableName == null || tableName.getName().length == 0) {
166         // Full hbase:meta scan
167         startRow = HConstants.EMPTY_START_ROW;
168       } else {
169         // Scan hbase:meta for an entire table
170         startRow = HRegionInfo.createRegionName(tableName, HConstants.EMPTY_START_ROW,
171           HConstants.ZEROES, false);
172       }
173       final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
174       int scannerCaching = connection.getConfiguration()
175           .getInt(HConstants.HBASE_META_SCANNER_CACHING,
176               HConstants.DEFAULT_HBASE_META_SCANNER_CACHING);
177       if (rowUpperLimit <= scannerCaching) {
178           scan.setSmall(true);
179       }
180       int rows = Math.min(rowLimit, scannerCaching);
181       scan.setCaching(rows);
182       if (LOG.isTraceEnabled()) {
183         LOG.trace("Scanning " + metaTableName.getNameAsString() + " starting at row=" +
184           Bytes.toStringBinary(startRow) + " for max=" + rowUpperLimit + " with caching=" + rows);
185       }
186       // Run the scan
187       try (ResultScanner resultScanner = metaTable.getScanner(scan)) {
188         Result result;
189         int processedRows = 0;
190         while ((result = resultScanner.next()) != null) {
191           if (visitor != null) {
192             if (!visitor.processRow(result)) break;
193           }
194           processedRows++;
195           if (processedRows >= rowUpperLimit) break;
196         }
197       }
198     } finally {
199       if (visitor != null) {
200         try {
201           visitor.close();
202         } catch (Throwable t) {
203           ExceptionUtil.rethrowIfInterrupt(t);
204           LOG.debug("Got exception in closing the meta scanner visitor", t);
205         }
206       }
207     }
208   }
209 
210   /**
211    * @return Get closest metatable region row to passed <code>row</code>
212    * @throws IOException
213    */
214   private static Result getClosestRowOrBefore(final Table metaTable, final TableName userTableName,
215       final byte [] row)
216   throws IOException {
217     byte[] searchRow = HRegionInfo.createRegionName(userTableName, row, HConstants.NINES, false);
218     Scan scan = Scan.createGetClosestRowOrBeforeReverseScan(searchRow);
219     try (ResultScanner resultScanner = metaTable.getScanner(scan)) {
220       return resultScanner.next();
221     }
222   }
223 
224   /**
225    * Returns HRegionInfo object from the column
226    * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog
227    * table Result.
228    * @param data a Result object from the catalog table scan
229    * @return HRegionInfo or null
230    * @deprecated Use {@link org.apache.hadoop.hbase.MetaTableAccessor#getRegionLocations(Result)}
231    */
232   @Deprecated
233   public static HRegionInfo getHRegionInfo(Result data) {
234     return HRegionInfo.getHRegionInfo(data);
235   }
236 
237   /**
238    * Lists all of the regions currently in META.
239    * @param conf configuration
240    * @param connection to connect with
241    * @param offlined True if we are to include offlined regions, false and we'll
242    * leave out offlined regions from returned list.
243    * @return List of all user-space regions.
244    * @throws IOException
245    */
246   @VisibleForTesting // And for hbck.
247   public static List<HRegionInfo> listAllRegions(Configuration conf, Connection connection,
248       final boolean offlined)
249   throws IOException {
250     final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
251     MetaScannerVisitor visitor = new MetaScannerVisitorBase() {
252         @Override
253         public boolean processRow(Result result) throws IOException {
254           if (result == null || result.isEmpty()) {
255             return true;
256           }
257 
258           RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
259           if (locations == null) return true;
260           for (HRegionLocation loc : locations.getRegionLocations()) {
261             if (loc != null) {
262               HRegionInfo regionInfo = loc.getRegionInfo();
263               // If region offline AND we are not to include offlined regions, return.
264               if (regionInfo.isOffline() && !offlined) continue;
265               regions.add(regionInfo);
266             }
267           }
268           return true;
269         }
270     };
271     metaScan(connection, visitor);
272     return regions;
273   }
274 
275   /**
276    * Lists all of the table regions currently in META.
277    * @param conf
278    * @param offlined True if we are to include offlined regions, false and we'll
279    * leave out offlined regions from returned list.
280    * @return Map of all user-space regions to servers
281    * @throws IOException
282    * @deprecated Use {@link #allTableRegions(Connection, TableName)} instead
283    */
284   @Deprecated
285   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
286       Connection connection, final TableName tableName, boolean offlined) throws IOException {
287     return allTableRegions(connection, tableName);
288   }
289 
290   /**
291    * Lists all of the table regions currently in META.
292    * @param connection
293    * @param tableName
294    * @return Map of all user-space regions to servers
295    * @throws IOException
296    */
297   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(
298       Connection connection, final TableName tableName) throws IOException {
299     final NavigableMap<HRegionInfo, ServerName> regions =
300       new TreeMap<HRegionInfo, ServerName>();
301     MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) {
302       @Override
303       public boolean processRowInternal(Result result) throws IOException {
304         RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
305         if (locations == null) return true;
306         for (HRegionLocation loc : locations.getRegionLocations()) {
307           if (loc != null) {
308             HRegionInfo regionInfo = loc.getRegionInfo();
309             regions.put(new UnmodifyableHRegionInfo(regionInfo), loc.getServerName());
310           }
311         }
312         return true;
313       }
314     };
315     metaScan(connection, visitor, tableName);
316     return regions;
317   }
318 
319   /**
320    * Lists table regions and locations grouped by region range from META.
321    */
322   public static List<RegionLocations> listTableRegionLocations(Configuration conf,
323       Connection connection, final TableName tableName) throws IOException {
324     final List<RegionLocations> regions = new ArrayList<RegionLocations>();
325     MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) {
326       @Override
327       public boolean processRowInternal(Result result) throws IOException {
328         RegionLocations locations = MetaTableAccessor.getRegionLocations(result);
329         if (locations == null) return true;
330         regions.add(locations);
331         return true;
332       }
333     };
334     metaScan(connection, visitor, tableName);
335     return regions;
336   }
337 
338   /**
339    * Visitor class called to process each row of the hbase:meta table
340    */
341   public interface MetaScannerVisitor extends Closeable {
342     /**
343      * Visitor method that accepts a RowResult and the meta region location.
344      * Implementations can return false to stop the region's loop if it becomes
345      * unnecessary for some reason.
346      *
347      * @param rowResult result
348      * @return A boolean to know if it should continue to loop in the region
349      * @throws IOException e
350      */
351     boolean processRow(Result rowResult) throws IOException;
352   }
353 
354   public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor {
355     @Override
356     public void close() throws IOException {
357     }
358   }
359 
360   /**
361    * A MetaScannerVisitor that skips offline regions and split parents
362    */
363   public static abstract class DefaultMetaScannerVisitor
364     extends MetaScannerVisitorBase {
365 
366     public DefaultMetaScannerVisitor() {
367       super();
368     }
369 
370     public abstract boolean processRowInternal(Result rowResult) throws IOException;
371 
372     @Override
373     public boolean processRow(Result rowResult) throws IOException {
374       HRegionInfo info = getHRegionInfo(rowResult);
375       if (info == null) {
376         return true;
377       }
378 
379       //skip over offline and split regions
380       if (!(info.isOffline() || info.isSplit())) {
381         return processRowInternal(rowResult);
382       }
383       return true;
384     }
385   }
386 
387   /**
388    * A MetaScannerVisitor for a table. Provides a consistent view of the table's
389    * hbase:meta entries during concurrent splits (see HBASE-5986 for details). This class
390    * does not guarantee ordered traversal of meta entries, and can block until the
391    * hbase:meta entries for daughters are available during splits.
392    */
393   public static abstract class TableMetaScannerVisitor extends DefaultMetaScannerVisitor {
394     private TableName tableName;
395 
396     public TableMetaScannerVisitor(TableName tableName) {
397       super();
398       this.tableName = tableName;
399     }
400 
401     @Override
402     public final boolean processRow(Result rowResult) throws IOException {
403       HRegionInfo info = getHRegionInfo(rowResult);
404       if (info == null) {
405         return true;
406       }
407       if (!(info.getTable().equals(tableName))) {
408         return false;
409       }
410       return super.processRow(rowResult);
411     }
412   }
413 }