View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.mapred;
20  
21  import java.io.IOException;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.hbase.HBaseConfiguration;
26  import org.apache.hadoop.hbase.client.HTable;
27  import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
28  import org.apache.hadoop.hbase.util.Bytes;
29  import org.apache.hadoop.mapred.JobConf;
30  import org.apache.hadoop.mapred.Partitioner;
31  
32  
33  /**
34   * This is used to partition the output keys into groups of keys.
35   * Keys are grouped according to the regions that currently exist
36   * so that each reducer fills a single region so load is distributed.
37   *
38   * @param <K2>
39   * @param <V2>
40   */
41  @Deprecated
42  public class HRegionPartitioner<K2,V2>
43  implements Partitioner<ImmutableBytesWritable, V2> {
44    private final Log LOG = LogFactory.getLog(TableInputFormat.class);
45    private HTable table;
46    private byte[][] startKeys;
47  
48    public void configure(JobConf job) {
49      try {
50        this.table = new HTable(HBaseConfiguration.create(job),
51          job.get(TableOutputFormat.OUTPUT_TABLE));
52      } catch (IOException e) {
53        LOG.error(e);
54      }
55  
56      try {
57        this.startKeys = this.table.getStartKeys();
58      } catch (IOException e) {
59        LOG.error(e);
60      }
61    }
62  
63    public int getPartition(ImmutableBytesWritable key,
64        V2 value, int numPartitions) {
65      byte[] region = null;
66      // Only one region return 0
67      if (this.startKeys.length == 1){
68        return 0;
69      }
70      try {
71        // Not sure if this is cached after a split so we could have problems
72        // here if a region splits while mapping
73        region = table.getRegionLocation(key.get()).getRegionInfo().getStartKey();
74      } catch (IOException e) {
75        LOG.error(e);
76      }
77      for (int i = 0; i < this.startKeys.length; i++){
78        if (Bytes.compareTo(region, this.startKeys[i]) == 0 ){
79          if (i >= numPartitions-1){
80            // cover if we have less reduces then regions.
81            return (Integer.toString(i).hashCode()
82                & Integer.MAX_VALUE) % numPartitions;
83          }
84          return i;
85        }
86      }
87      // if above fails to find start key that match we need to return something
88      return 0;
89    }
90  }