1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import java.io.IOException;
22
23 import org.apache.commons.logging.Log;
24 import org.apache.commons.logging.LogFactory;
25 import org.apache.hadoop.hbase.classification.InterfaceAudience;
26 import org.apache.hadoop.hbase.classification.InterfaceStability;
27 import org.apache.hadoop.conf.Configurable;
28 import org.apache.hadoop.conf.Configuration;
29 import org.apache.hadoop.hbase.HBaseConfiguration;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.client.Delete;
32 import org.apache.hadoop.hbase.client.HTable;
33 import org.apache.hadoop.hbase.client.Mutation;
34 import org.apache.hadoop.hbase.client.Put;
35 import org.apache.hadoop.mapreduce.JobContext;
36 import org.apache.hadoop.mapreduce.OutputCommitter;
37 import org.apache.hadoop.mapreduce.OutputFormat;
38 import org.apache.hadoop.mapreduce.RecordWriter;
39 import org.apache.hadoop.mapreduce.TaskAttemptContext;
40
41
42
43
44
45
46
47
48 @InterfaceAudience.Public
49 @InterfaceStability.Stable
50 public class TableOutputFormat<KEY> extends OutputFormat<KEY, Mutation>
51 implements Configurable {
52
53 private final Log LOG = LogFactory.getLog(TableOutputFormat.class);
54
55
56
57
58
59
60
61
62
63 public static final String OUTPUT_CONF_PREFIX = "hbase.mapred.output.";
64
65
66 public static final String OUTPUT_TABLE = OUTPUT_CONF_PREFIX + "outputtable";
67
68
69
70
71
72
73
74 public static final String QUORUM_ADDRESS = OUTPUT_CONF_PREFIX + "quorum";
75
76
77 public static final String QUORUM_PORT = OUTPUT_CONF_PREFIX + "port";
78
79
80 public static final String REGION_SERVER_CLASS = OUTPUT_CONF_PREFIX + "rs.class";
81
82 public static final String REGION_SERVER_IMPL = OUTPUT_CONF_PREFIX + "rs.impl";
83
84
85 private Configuration conf = null;
86
87 private HTable table;
88
89
90
91
92
93
94 protected static class TableRecordWriter<KEY>
95 extends RecordWriter<KEY, Mutation> {
96
97
98 private HTable table;
99
100
101
102
103
104
105 public TableRecordWriter(HTable table) {
106 this.table = table;
107 }
108
109
110
111
112
113
114
115
116 @Override
117 public void close(TaskAttemptContext context)
118 throws IOException {
119 table.close();
120 }
121
122
123
124
125
126
127
128
129
130 @Override
131 public void write(KEY key, Mutation value)
132 throws IOException {
133 if (value instanceof Put) this.table.put(new Put((Put)value));
134 else if (value instanceof Delete) this.table.delete(new Delete((Delete)value));
135 else throw new IOException("Pass a Delete or a Put");
136 }
137 }
138
139
140
141
142
143
144
145
146
147
148 @Override
149 public RecordWriter<KEY, Mutation> getRecordWriter(
150 TaskAttemptContext context)
151 throws IOException, InterruptedException {
152 return new TableRecordWriter<KEY>(this.table);
153 }
154
155
156
157
158
159
160
161
162
163 @Override
164 public void checkOutputSpecs(JobContext context) throws IOException,
165 InterruptedException {
166
167
168 }
169
170
171
172
173
174
175
176
177
178
179 @Override
180 public OutputCommitter getOutputCommitter(TaskAttemptContext context)
181 throws IOException, InterruptedException {
182 return new TableOutputCommitter();
183 }
184
185 public Configuration getConf() {
186 return conf;
187 }
188
189 @Override
190 public void setConf(Configuration otherConf) {
191 String tableName = otherConf.get(OUTPUT_TABLE);
192 if(tableName == null || tableName.length() <= 0) {
193 throw new IllegalArgumentException("Must specify table name");
194 }
195
196 String address = otherConf.get(QUORUM_ADDRESS);
197 int zkClientPort = otherConf.getInt(QUORUM_PORT, 0);
198 String serverClass = otherConf.get(REGION_SERVER_CLASS);
199 String serverImpl = otherConf.get(REGION_SERVER_IMPL);
200
201 try {
202 this.conf = HBaseConfiguration.createClusterConf(otherConf, address, OUTPUT_CONF_PREFIX);
203
204 if (serverClass != null) {
205 this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
206 }
207 if (zkClientPort != 0) {
208 this.conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
209 }
210 this.table = new BufferedHTable(this.conf, tableName);
211 this.table.setAutoFlush(false, true);
212 LOG.info("Created table instance for " + tableName);
213 } catch(IOException e) {
214 LOG.error(e);
215 throw new RuntimeException(e);
216 }
217 }
218 }