1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapred;
20
21 import java.io.IOException;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.client.HTable;
27 import org.apache.hadoop.hbase.client.Result;
28 import org.apache.hadoop.hbase.filter.Filter;
29 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
30 import org.apache.hadoop.hbase.regionserver.HRegion;
31 import org.apache.hadoop.mapred.InputFormat;
32 import org.apache.hadoop.mapred.InputSplit;
33 import org.apache.hadoop.mapred.JobConf;
34 import org.apache.hadoop.mapred.RecordReader;
35 import org.apache.hadoop.mapred.Reporter;
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66 @Deprecated
67 public abstract class TableInputFormatBase
68 implements InputFormat<ImmutableBytesWritable, Result> {
69 final Log LOG = LogFactory.getLog(TableInputFormatBase.class);
70 private byte [][] inputColumns;
71 private HTable table;
72 private TableRecordReader tableRecordReader;
73 private Filter rowFilter;
74
75
76
77
78
79
80
81
82 public RecordReader<ImmutableBytesWritable, Result> getRecordReader(
83 InputSplit split, JobConf job, Reporter reporter)
84 throws IOException {
85 TableSplit tSplit = (TableSplit) split;
86 TableRecordReader trr = this.tableRecordReader;
87
88 if (trr == null) {
89 trr = new TableRecordReader();
90 }
91 trr.setStartRow(tSplit.getStartRow());
92 trr.setEndRow(tSplit.getEndRow());
93 trr.setHTable(this.table);
94 trr.setInputColumns(this.inputColumns);
95 trr.setRowFilter(this.rowFilter);
96 trr.init();
97 return trr;
98 }
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117 public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
118 if (this.table == null) {
119 throw new IOException("No table was provided");
120 }
121 byte [][] startKeys = this.table.getStartKeys();
122 if (startKeys == null || startKeys.length == 0) {
123 throw new IOException("Expecting at least one region");
124 }
125 if (this.inputColumns == null || this.inputColumns.length == 0) {
126 throw new IOException("Expecting at least one column");
127 }
128 int realNumSplits = numSplits > startKeys.length? startKeys.length:
129 numSplits;
130 InputSplit[] splits = new InputSplit[realNumSplits];
131 int middle = startKeys.length / realNumSplits;
132 int startPos = 0;
133 for (int i = 0; i < realNumSplits; i++) {
134 int lastPos = startPos + middle;
135 lastPos = startKeys.length % realNumSplits > i ? lastPos + 1 : lastPos;
136 String regionLocation = table.getRegionLocation(startKeys[startPos]).
137 getHostname();
138 splits[i] = new TableSplit(this.table.getName(),
139 startKeys[startPos], ((i + 1) < realNumSplits) ? startKeys[lastPos]:
140 HConstants.EMPTY_START_ROW, regionLocation);
141 LOG.info("split: " + i + "->" + splits[i]);
142 startPos = lastPos;
143 }
144 return splits;
145 }
146
147
148
149
150 protected void setInputColumns(byte [][] inputColumns) {
151 this.inputColumns = inputColumns;
152 }
153
154
155
156
157 protected HTable getHTable() {
158 return this.table;
159 }
160
161
162
163
164
165
166 protected void setHTable(HTable table) {
167 this.table = table;
168 }
169
170
171
172
173
174
175
176 protected void setTableRecordReader(TableRecordReader tableRecordReader) {
177 this.tableRecordReader = tableRecordReader;
178 }
179
180
181
182
183
184
185 protected void setRowFilter(Filter rowFilter) {
186 this.rowFilter = rowFilter;
187 }
188 }