View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.classification.InterfaceAudience;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.hbase.HConstants;
27  import org.apache.hadoop.hbase.HRegionInfo;
28  import org.apache.hadoop.hbase.ServerName;
29  import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
30  import org.apache.hadoop.hbase.exceptions.TableNotFoundException;
31  import org.apache.hadoop.hbase.util.Bytes;
32  import org.apache.hadoop.hbase.util.PairOfSameType;
33  
34  import java.io.Closeable;
35  import java.io.IOException;
36  import java.util.ArrayList;
37  import java.util.List;
38  import java.util.NavigableMap;
39  import java.util.TreeMap;
40  
41  /**
42   * Scanner class that contains the <code>.META.</code> table scanning logic
43   * and uses a Retryable scanner. Provided visitors will be called
44   * for each row.
45   *
46   * Although public visibility, this is not a public-facing API and may evolve in
47   * minor releases.
48   *
49   * <p> Note that during concurrent region splits, the scanner might not see
50   * META changes across rows (for parent and daughter entries) consistently.
51   * see HBASE-5986, and {@link DefaultMetaScannerVisitor} for details. </p>
52   */
53  @InterfaceAudience.Private
54  public class MetaScanner {
55    private static final Log LOG = LogFactory.getLog(MetaScanner.class);
56    /**
57     * Scans the meta table and calls a visitor on each RowResult and uses a empty
58     * start row value as table name.
59     *
60     * @param configuration conf
61     * @param visitor A custom visitor
62     * @throws IOException e
63     */
64    public static void metaScan(Configuration configuration,
65        MetaScannerVisitor visitor)
66    throws IOException {
67      metaScan(configuration, visitor, null);
68    }
69  
70    /**
71     * Scans the meta table and calls a visitor on each RowResult. Uses a table
72     * name to locate meta regions.
73     *
74     * @param configuration config
75     * @param visitor visitor object
76     * @param userTableName User table name in meta table to start scan at.  Pass
77     * null if not interested in a particular table.
78     * @throws IOException e
79     */
80    public static void metaScan(Configuration configuration,
81        MetaScannerVisitor visitor, byte [] userTableName)
82    throws IOException {
83      metaScan(configuration, visitor, userTableName, null, Integer.MAX_VALUE);
84    }
85  
86    /**
87     * Scans the meta table and calls a visitor on each RowResult. Uses a table
88     * name and a row name to locate meta regions. And it only scans at most
89     * <code>rowLimit</code> of rows.
90     *
91     * @param configuration HBase configuration.
92     * @param visitor Visitor object.
93     * @param userTableName User table name in meta table to start scan at.  Pass
94     * null if not interested in a particular table.
95     * @param row Name of the row at the user table. The scan will start from
96     * the region row where the row resides.
97     * @param rowLimit Max of processed rows. If it is less than 0, it
98     * will be set to default value <code>Integer.MAX_VALUE</code>.
99     * @throws IOException e
100    */
101   public static void metaScan(Configuration configuration,
102       MetaScannerVisitor visitor, byte [] userTableName, byte[] row,
103       int rowLimit)
104   throws IOException {
105     metaScan(configuration, visitor, userTableName, row, rowLimit,
106       HConstants.META_TABLE_NAME);
107   }
108 
109   /**
110    * Scans the meta table and calls a visitor on each RowResult. Uses a table
111    * name and a row name to locate meta regions. And it only scans at most
112    * <code>rowLimit</code> of rows.
113    *
114    * @param configuration HBase configuration.
115    * @param visitor Visitor object. Closes the visitor before returning.
116    * @param tableName User table name in meta table to start scan at.  Pass
117    * null if not interested in a particular table.
118    * @param row Name of the row at the user table. The scan will start from
119    * the region row where the row resides.
120    * @param rowLimit Max of processed rows. If it is less than 0, it
121    * will be set to default value <code>Integer.MAX_VALUE</code>.
122    * @param metaTableName Meta table to scan, root or meta.
123    * @throws IOException e
124    */
125   public static void metaScan(Configuration configuration,
126       final MetaScannerVisitor visitor, final byte[] tableName,
127       final byte[] row, final int rowLimit, final byte[] metaTableName)
128       throws IOException {
129     try {
130       HConnectionManager.execute(new HConnectable<Void>(configuration) {
131         @Override
132         public Void connect(HConnection connection) throws IOException {
133           metaScan(conf, connection, visitor, tableName, row, rowLimit,
134               metaTableName);
135           return null;
136         }
137       });
138     } finally {
139       visitor.close();
140     }
141   }
142 
143   private static void metaScan(Configuration configuration, HConnection connection,
144       MetaScannerVisitor visitor, byte [] tableName, byte[] row,
145       int rowLimit, final byte [] metaTableName)
146   throws IOException {
147     int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE;
148 
149     // if row is not null, we want to use the startKey of the row's region as
150     // the startRow for the meta scan.
151     byte[] startRow;
152     if (row != null) {
153       // Scan starting at a particular row in a particular table
154       assert tableName != null;
155       byte[] searchRow =
156         HRegionInfo.createRegionName(tableName, row, HConstants.NINES,
157           false);
158       HTable metaTable = null;
159       try {
160         metaTable = new HTable(configuration, HConstants.META_TABLE_NAME);
161         Result startRowResult = metaTable.getRowOrBefore(searchRow,
162             HConstants.CATALOG_FAMILY);
163         if (startRowResult == null) {
164           throw new TableNotFoundException("Cannot find row in .META. for table: "
165               + Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow));
166         }
167         HRegionInfo regionInfo = getHRegionInfo(startRowResult);
168         if (regionInfo == null) {
169           throw new IOException("HRegionInfo was null or empty in Meta for " +
170             Bytes.toString(tableName) + ", row=" + Bytes.toStringBinary(searchRow));
171         }
172 
173         byte[] rowBefore = regionInfo.getStartKey();
174         startRow = HRegionInfo.createRegionName(tableName, rowBefore,
175             HConstants.ZEROES, false);
176       } finally {
177         if (metaTable != null) {
178           metaTable.close();
179         }
180       }
181     } else if (tableName == null || tableName.length == 0) {
182       // Full META scan
183       startRow = HConstants.EMPTY_START_ROW;
184     } else {
185       // Scan META for an entire table
186       startRow = HRegionInfo.createRegionName(
187           tableName, HConstants.EMPTY_START_ROW, HConstants.ZEROES, false);
188     }
189 
190     // Scan over each meta region
191     ScannerCallable callable;
192     int rows = Math.min(rowLimit, configuration.getInt(
193         HConstants.HBASE_META_SCANNER_CACHING,
194         HConstants.DEFAULT_HBASE_META_SCANNER_CACHING));
195     do {
196       final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY);
197       if (LOG.isDebugEnabled()) {
198         LOG.debug("Scanning " + Bytes.toString(metaTableName) +
199           " starting at row=" + Bytes.toStringBinary(startRow) + " for max=" +
200           rowUpperLimit + " rows using " + connection.toString());
201       }
202       callable = new ScannerCallable(connection, metaTableName, scan, null);
203       // Open scanner
204       callable.withRetries();
205 
206       int processedRows = 0;
207       try {
208         callable.setCaching(rows);
209         done: do {
210           if (processedRows >= rowUpperLimit) {
211             break;
212           }
213           //we have all the rows here
214           Result [] rrs = callable.withRetries();
215           if (rrs == null || rrs.length == 0 || rrs[0].size() == 0) {
216             break; //exit completely
217           }
218           for (Result rr : rrs) {
219             if (processedRows >= rowUpperLimit) {
220               break done;
221             }
222             if (!visitor.processRow(rr))
223               break done; //exit completely
224             processedRows++;
225           }
226           //here, we didn't break anywhere. Check if we have more rows
227         } while(true);
228         // Advance the startRow to the end key of the current region
229         startRow = callable.getHRegionInfo().getEndKey();
230       } finally {
231         // Close scanner
232         callable.setClose();
233         callable.withRetries();
234       }
235     } while (Bytes.compareTo(startRow, HConstants.LAST_ROW) != 0);
236   }
237 
238   /**
239    * Returns HRegionInfo object from the column
240    * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog
241    * table Result.
242    * @param data a Result object from the catalog table scan
243    * @return HRegionInfo or null
244    */
245   public static HRegionInfo getHRegionInfo(Result data) {
246     byte [] bytes =
247       data.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
248     if (bytes == null) return null;
249     HRegionInfo info = HRegionInfo.parseFromOrNull(bytes);
250     if (LOG.isDebugEnabled()) {
251       LOG.debug("Current INFO from scan results = " + info);
252     }
253     return info;
254   }
255 
256   /**
257    * Lists all of the regions currently in META.
258    * @param conf
259    * @return List of all user-space regions.
260    * @throws IOException
261    */
262   public static List<HRegionInfo> listAllRegions(Configuration conf)
263   throws IOException {
264     return listAllRegions(conf, true);
265   }
266 
267   /**
268    * Lists all of the regions currently in META.
269    * @param conf
270    * @param offlined True if we are to include offlined regions, false and we'll
271    * leave out offlined regions from returned list.
272    * @return List of all user-space regions.
273    * @throws IOException
274    */
275   public static List<HRegionInfo> listAllRegions(Configuration conf, final boolean offlined)
276   throws IOException {
277     final List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
278     MetaScannerVisitor visitor = new DefaultMetaScannerVisitor(conf) {
279         @Override
280         public boolean processRowInternal(Result result) throws IOException {
281           if (result == null || result.isEmpty()) {
282             return true;
283           }
284 
285           HRegionInfo regionInfo = getHRegionInfo(result);
286           if (regionInfo == null) {
287             LOG.warn("Null REGIONINFO_QUALIFIER: " + result);
288             return true;
289           }
290 
291           // If region offline AND we are not to include offlined regions, return.
292           if (regionInfo.isOffline() && !offlined) return true;
293           regions.add(regionInfo);
294           return true;
295         }
296     };
297     metaScan(conf, visitor);
298     return regions;
299   }
300 
301   /**
302    * Lists all of the table regions currently in META.
303    * @param conf
304    * @param offlined True if we are to include offlined regions, false and we'll
305    * leave out offlined regions from returned list.
306    * @return Map of all user-space regions to servers
307    * @throws IOException
308    */
309   public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf,
310       final byte [] tablename, final boolean offlined) throws IOException {
311     final NavigableMap<HRegionInfo, ServerName> regions =
312       new TreeMap<HRegionInfo, ServerName>();
313     MetaScannerVisitor visitor = new TableMetaScannerVisitor(conf, tablename) {
314       @Override
315       public boolean processRowInternal(Result rowResult) throws IOException {
316         HRegionInfo info = getHRegionInfo(rowResult);
317         ServerName serverName = HRegionInfo.getServerName(rowResult);
318         regions.put(new UnmodifyableHRegionInfo(info), serverName);
319         return true;
320       }
321     };
322     metaScan(conf, visitor, tablename);
323     return regions;
324   }
325 
326   /**
327    * Visitor class called to process each row of the .META. table
328    */
329   public interface MetaScannerVisitor extends Closeable {
330     /**
331      * Visitor method that accepts a RowResult and the meta region location.
332      * Implementations can return false to stop the region's loop if it becomes
333      * unnecessary for some reason.
334      *
335      * @param rowResult result
336      * @return A boolean to know if it should continue to loop in the region
337      * @throws IOException e
338      */
339     public boolean processRow(Result rowResult) throws IOException;
340   }
341 
342   public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor {
343     @Override
344     public void close() throws IOException {
345     }
346   }
347 
348   /**
349    * A MetaScannerVisitor that provides a consistent view of the table's
350    * META entries during concurrent splits (see HBASE-5986 for details). This class
351    * does not guarantee ordered traversal of meta entries, and can block until the
352    * META entries for daughters are available during splits.
353    */
354   public static abstract class DefaultMetaScannerVisitor
355     extends MetaScannerVisitorBase {
356 
357     protected Configuration conf;
358 
359     public DefaultMetaScannerVisitor(Configuration conf) {
360       this.conf = conf;
361     }
362 
363     public abstract boolean processRowInternal(Result rowResult) throws IOException;
364 
365     @Override
366     public boolean processRow(Result rowResult) throws IOException {
367       HRegionInfo info = getHRegionInfo(rowResult);
368       if (info == null) {
369         return true;
370       }
371 
372       //skip over offline and split regions
373       if (!(info.isOffline() || info.isSplit())) {
374         return processRowInternal(rowResult);
375       }
376       return true;
377     }
378   }
379 
380   /**
381    * A MetaScannerVisitor for a table. Provides a consistent view of the table's
382    * META entries during concurrent splits (see HBASE-5986 for details). This class
383    * does not guarantee ordered traversal of meta entries, and can block until the
384    * META entries for daughters are available during splits.
385    */
386   public static abstract class TableMetaScannerVisitor extends DefaultMetaScannerVisitor {
387     private byte[] tableName;
388 
389     public TableMetaScannerVisitor(Configuration conf, byte[] tableName) {
390       super(conf);
391       this.tableName = tableName;
392     }
393 
394     @Override
395     public final boolean processRow(Result rowResult) throws IOException {
396       HRegionInfo info = getHRegionInfo(rowResult);
397       if (info == null) {
398         return true;
399       }
400       if (!(Bytes.equals(info.getTableName(), tableName))) {
401         return false;
402       }
403       return super.processRow(rowResult);
404     }
405 
406   }
407 }