1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.mapred;
21
22 import java.io.IOException;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.hbase.HConstants;
27 import org.apache.hadoop.hbase.client.HTable;
28 import org.apache.hadoop.hbase.client.Result;
29 import org.apache.hadoop.hbase.filter.Filter;
30 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
31 import org.apache.hadoop.hbase.regionserver.HRegion;
32 import org.apache.hadoop.mapred.InputFormat;
33 import org.apache.hadoop.mapred.InputSplit;
34 import org.apache.hadoop.mapred.JobConf;
35 import org.apache.hadoop.mapred.RecordReader;
36 import org.apache.hadoop.mapred.Reporter;
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67 @Deprecated
68 public abstract class TableInputFormatBase
69 implements InputFormat<ImmutableBytesWritable, Result> {
70 final Log LOG = LogFactory.getLog(TableInputFormatBase.class);
71 private byte [][] inputColumns;
72 private HTable table;
73 private TableRecordReader tableRecordReader;
74 private Filter rowFilter;
75
76
77
78
79
80
81
82
83 public RecordReader<ImmutableBytesWritable, Result> getRecordReader(
84 InputSplit split, JobConf job, Reporter reporter)
85 throws IOException {
86 TableSplit tSplit = (TableSplit) split;
87 TableRecordReader trr = this.tableRecordReader;
88
89 if (trr == null) {
90 trr = new TableRecordReader();
91 }
92 trr.setStartRow(tSplit.getStartRow());
93 trr.setEndRow(tSplit.getEndRow());
94 trr.setHTable(this.table);
95 trr.setInputColumns(this.inputColumns);
96 trr.setRowFilter(this.rowFilter);
97 trr.init();
98 return trr;
99 }
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118 public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
119 if (this.table == null) {
120 throw new IOException("No table was provided");
121 }
122 byte [][] startKeys = this.table.getStartKeys();
123 if (startKeys == null || startKeys.length == 0) {
124 throw new IOException("Expecting at least one region");
125 }
126 if (this.inputColumns == null || this.inputColumns.length == 0) {
127 throw new IOException("Expecting at least one column");
128 }
129 int realNumSplits = numSplits > startKeys.length? startKeys.length:
130 numSplits;
131 InputSplit[] splits = new InputSplit[realNumSplits];
132 int middle = startKeys.length / realNumSplits;
133 int startPos = 0;
134 for (int i = 0; i < realNumSplits; i++) {
135 int lastPos = startPos + middle;
136 lastPos = startKeys.length % realNumSplits > i ? lastPos + 1 : lastPos;
137 String regionLocation = table.getRegionLocation(startKeys[startPos]).
138 getServerAddress().getHostname();
139 splits[i] = new TableSplit(this.table.getTableName(),
140 startKeys[startPos], ((i + 1) < realNumSplits) ? startKeys[lastPos]:
141 HConstants.EMPTY_START_ROW, regionLocation);
142 LOG.info("split: " + i + "->" + splits[i]);
143 startPos = lastPos;
144 }
145 return splits;
146 }
147
148
149
150
151 protected void setInputColumns(byte [][] inputColumns) {
152 this.inputColumns = inputColumns;
153 }
154
155
156
157
158 protected HTable getHTable() {
159 return this.table;
160 }
161
162
163
164
165
166
167 protected void setHTable(HTable table) {
168 this.table = table;
169 }
170
171
172
173
174
175
176
177 protected void setTableRecordReader(TableRecordReader tableRecordReader) {
178 this.tableRecordReader = tableRecordReader;
179 }
180
181
182
183
184
185
186 protected void setRowFilter(Filter rowFilter) {
187 this.rowFilter = rowFilter;
188 }
189 }