View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.mapreduce;
20  
21  import com.google.common.annotations.VisibleForTesting;
22  
23  import org.apache.hadoop.fs.Path;
24  import org.apache.hadoop.hbase.HRegionInfo;
25  import org.apache.hadoop.hbase.HTableDescriptor;
26  import org.apache.hadoop.hbase.classification.InterfaceAudience;
27  import org.apache.hadoop.hbase.classification.InterfaceStability;
28  import org.apache.hadoop.hbase.client.Result;
29  import org.apache.hadoop.hbase.client.Scan;
30  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
31  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
32  import org.apache.hadoop.io.Writable;
33  import org.apache.hadoop.mapreduce.InputFormat;
34  import org.apache.hadoop.mapreduce.InputSplit;
35  import org.apache.hadoop.mapreduce.Job;
36  import org.apache.hadoop.mapreduce.JobContext;
37  import org.apache.hadoop.mapreduce.RecordReader;
38  import org.apache.hadoop.mapreduce.TaskAttemptContext;
39  
40  import java.io.DataInput;
41  import java.io.DataOutput;
42  import java.io.IOException;
43  import java.lang.reflect.Method;
44  import java.util.ArrayList;
45  import java.util.List;
46  
47  /**
48   * TableSnapshotInputFormat allows a MapReduce job to run over a table snapshot. The job
49   * bypasses HBase servers, and directly accesses the underlying files (hfile, recovered edits,
50   * hlogs, etc) directly to provide maximum performance. The snapshot is not required to be
51   * restored to the live cluster or cloned. This also allows to run the mapreduce job from an
52   * online or offline hbase cluster. The snapshot files can be exported by using the
53   * {@link ExportSnapshot} tool, to a pure-hdfs cluster, and this InputFormat can be used to
54   * run the mapreduce job directly over the snapshot files. The snapshot should not be deleted
55   * while there are jobs reading from snapshot files.
56   * <p>
57   * Usage is similar to TableInputFormat, and
58   * {@link TableMapReduceUtil#initTableSnapshotMapperJob(String, Scan, Class, Class, Class, Job,
59   *   boolean, Path)}
60   * can be used to configure the job.
61   * <pre>{@code
62   * Job job = new Job(conf);
63   * Scan scan = new Scan();
64   * TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
65   *      scan, MyTableMapper.class, MyMapKeyOutput.class,
66   *      MyMapOutputValueWritable.class, job, true);
67   * }
68   * </pre>
69   * <p>
70   * Internally, this input format restores the snapshot into the given tmp directory. Similar to
71   * {@link TableInputFormat} an InputSplit is created per region. The region is opened for reading
72   * from each RecordReader. An internal RegionScanner is used to execute the {@link Scan} obtained
73   * from the user.
74   * <p>
75   * HBase owns all the data and snapshot files on the filesystem. Only the 'hbase' user can read from
76   * snapshot files and data files.
77   * To read from snapshot files directly from the file system, the user who is running the MR job
78   * must have sufficient permissions to access snapshot and reference files.
79   * This means that to run mapreduce over snapshot files, the MR job has to be run as the HBase
80   * user or the user must have group or other privileges in the filesystem (See HBASE-8369).
81   * Note that, given other users access to read from snapshot/data files will completely circumvent
82   * the access control enforced by HBase.
83   * @see TableSnapshotScanner
84   */
85  @InterfaceAudience.Public
86  @InterfaceStability.Evolving
87  public class TableSnapshotInputFormat extends InputFormat<ImmutableBytesWritable, Result> {
88  
89    public static class TableSnapshotRegionSplit extends InputSplit implements Writable {
90      private TableSnapshotInputFormatImpl.InputSplit delegate;
91  
92      public TableSnapshotRegionSplit() {
93        this.delegate = new TableSnapshotInputFormatImpl.InputSplit();
94      }
95  
96      public TableSnapshotRegionSplit(TableSnapshotInputFormatImpl.InputSplit delegate) {
97        this.delegate = delegate;
98      }
99  
100     public TableSnapshotRegionSplit(HTableDescriptor htd, HRegionInfo regionInfo,
101         List<String> locations, Scan scan, Path restoreDir) {
102       this.delegate =
103           new TableSnapshotInputFormatImpl.InputSplit(htd, regionInfo, locations, scan, restoreDir);
104     }
105 
106     @Override
107     public long getLength() throws IOException, InterruptedException {
108       return delegate.getLength();
109     }
110 
111     @Override
112     public String[] getLocations() throws IOException, InterruptedException {
113       return delegate.getLocations();
114     }
115 
116     @Override
117     public void write(DataOutput out) throws IOException {
118       delegate.write(out);
119     }
120 
121     @Override
122     public void readFields(DataInput in) throws IOException {
123       delegate.readFields(in);
124     }
125   }
126 
127   @VisibleForTesting
128   static class TableSnapshotRegionRecordReader extends RecordReader<ImmutableBytesWritable, Result> {
129     private TableSnapshotInputFormatImpl.RecordReader delegate =
130       new TableSnapshotInputFormatImpl.RecordReader();
131     private TaskAttemptContext context;
132     private Method getCounter;
133 
134     @Override
135     public void initialize(InputSplit split, TaskAttemptContext context) throws IOException,
136       InterruptedException {
137       this.context = context;
138       getCounter = TableRecordReaderImpl.retrieveGetCounterWithStringsParams(context);
139       delegate.initialize(
140         ((TableSnapshotRegionSplit) split).delegate,
141         context.getConfiguration());
142     }
143 
144     @Override
145     public boolean nextKeyValue() throws IOException, InterruptedException {
146       boolean result = delegate.nextKeyValue();
147       if (result) {
148         ScanMetrics scanMetrics = delegate.getScanner().getScanMetrics();
149         if (scanMetrics != null && context != null) {
150           TableRecordReaderImpl.updateCounters(scanMetrics, 0, getCounter, context);
151         }
152       }
153 
154       return result;
155     }
156 
157     @Override
158     public ImmutableBytesWritable getCurrentKey() throws IOException, InterruptedException {
159       return delegate.getCurrentKey();
160     }
161 
162     @Override
163     public Result getCurrentValue() throws IOException, InterruptedException {
164       return delegate.getCurrentValue();
165     }
166 
167     @Override
168     public float getProgress() throws IOException, InterruptedException {
169       return delegate.getProgress();
170     }
171 
172     @Override
173     public void close() throws IOException {
174       delegate.close();
175     }
176   }
177 
178   @Override
179   public RecordReader<ImmutableBytesWritable, Result> createRecordReader(
180     InputSplit split, TaskAttemptContext context) throws IOException {
181     return new TableSnapshotRegionRecordReader();
182   }
183 
184   @Override
185   public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException {
186     List<InputSplit> results = new ArrayList<InputSplit>();
187     for (TableSnapshotInputFormatImpl.InputSplit split :
188       TableSnapshotInputFormatImpl.getSplits(job.getConfiguration())) {
189       results.add(new TableSnapshotRegionSplit(split));
190     }
191     return results;
192   }
193 
194   /**
195    * Configures the job to use TableSnapshotInputFormat to read from a snapshot.
196    * @param job the job to configure
197    * @param snapshotName the name of the snapshot to read from
198    * @param restoreDir a temporary directory to restore the snapshot into. Current user should
199    * have write permissions to this directory, and this should not be a subdirectory of rootdir.
200    * After the job is finished, restoreDir can be deleted.
201    * @throws IOException if an error occurs
202    */
203   public static void setInput(Job job, String snapshotName, Path restoreDir) throws IOException {
204     TableSnapshotInputFormatImpl.setInput(job.getConfiguration(), snapshotName, restoreDir);
205   }
206 }