1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.mapreduce;
21
22 import java.io.IOException;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.conf.Configurable;
27 import org.apache.hadoop.conf.Configuration;
28 import org.apache.hadoop.hbase.HBaseConfiguration;
29 import org.apache.hadoop.hbase.HConstants;
30 import org.apache.hadoop.hbase.client.Delete;
31 import org.apache.hadoop.hbase.client.HTable;
32 import org.apache.hadoop.hbase.client.Put;
33 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
34 import org.apache.hadoop.io.Writable;
35 import org.apache.hadoop.mapreduce.JobContext;
36 import org.apache.hadoop.mapreduce.OutputCommitter;
37 import org.apache.hadoop.mapreduce.OutputFormat;
38 import org.apache.hadoop.mapreduce.RecordWriter;
39 import org.apache.hadoop.mapreduce.TaskAttemptContext;
40
41
42
43
44
45
46
47
48 public class TableOutputFormat<KEY> extends OutputFormat<KEY, Writable>
49 implements Configurable {
50
51 private final Log LOG = LogFactory.getLog(TableOutputFormat.class);
52
53
54 public static final String OUTPUT_TABLE = "hbase.mapred.outputtable";
55
56
57
58
59
60
61
62 public static final String QUORUM_ADDRESS = "hbase.mapred.output.quorum";
63
64
65 public static final String QUORUM_PORT = "hbase.mapred.output.quorum.port";
66
67
68 public static final String
69 REGION_SERVER_CLASS = "hbase.mapred.output.rs.class";
70
71 public static final String
72 REGION_SERVER_IMPL = "hbase.mapred.output.rs.impl";
73
74
75 private Configuration conf = null;
76
77 private HTable table;
78
79
80
81
82
83
84 protected static class TableRecordWriter<KEY>
85 extends RecordWriter<KEY, Writable> {
86
87
88 private HTable table;
89
90
91
92
93
94
95 public TableRecordWriter(HTable table) {
96 this.table = table;
97 }
98
99
100
101
102
103
104
105
106 @Override
107 public void close(TaskAttemptContext context)
108 throws IOException {
109 table.close();
110 }
111
112
113
114
115
116
117
118
119
120 @Override
121 public void write(KEY key, Writable value)
122 throws IOException {
123 if (value instanceof Put) this.table.put(new Put((Put)value));
124 else if (value instanceof Delete) this.table.delete(new Delete((Delete)value));
125 else throw new IOException("Pass a Delete or a Put");
126 }
127 }
128
129
130
131
132
133
134
135
136
137
138 @Override
139 public RecordWriter<KEY, Writable> getRecordWriter(
140 TaskAttemptContext context)
141 throws IOException, InterruptedException {
142 return new TableRecordWriter<KEY>(this.table);
143 }
144
145
146
147
148
149
150
151
152
153 @Override
154 public void checkOutputSpecs(JobContext context) throws IOException,
155 InterruptedException {
156
157
158 }
159
160
161
162
163
164
165
166
167
168
169 @Override
170 public OutputCommitter getOutputCommitter(TaskAttemptContext context)
171 throws IOException, InterruptedException {
172 return new TableOutputCommitter();
173 }
174
175 public Configuration getConf() {
176 return conf;
177 }
178
179 @Override
180 public void setConf(Configuration otherConf) {
181 this.conf = HBaseConfiguration.create(otherConf);
182 String tableName = this.conf.get(OUTPUT_TABLE);
183 if(tableName == null || tableName.length() <= 0) {
184 throw new IllegalArgumentException("Must specify table name");
185 }
186 String address = this.conf.get(QUORUM_ADDRESS);
187 int zkClientPort = conf.getInt(QUORUM_PORT, 0);
188 String serverClass = this.conf.get(REGION_SERVER_CLASS);
189 String serverImpl = this.conf.get(REGION_SERVER_IMPL);
190 try {
191 if (address != null) {
192 ZKUtil.applyClusterKeyToConf(this.conf, address);
193 }
194 if (serverClass != null) {
195 this.conf.set(HConstants.REGION_SERVER_CLASS, serverClass);
196 this.conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
197 }
198 if (zkClientPort != 0) {
199 conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
200 }
201 this.table = new HTable(this.conf, tableName);
202 this.table.setAutoFlush(false);
203 LOG.info("Created table instance for " + tableName);
204 } catch(IOException e) {
205 LOG.error(e);
206 throw new RuntimeException(e);
207 }
208 }
209 }