1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.mapreduce;
21
22 import java.io.IOException;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.hbase.HBaseConfiguration;
27 import org.apache.hadoop.hbase.HConstants;
28 import org.apache.hadoop.hbase.client.Delete;
29 import org.apache.hadoop.hbase.client.HTable;
30 import org.apache.hadoop.hbase.client.Put;
31 import org.apache.hadoop.io.Writable;
32 import org.apache.hadoop.mapreduce.JobContext;
33 import org.apache.hadoop.mapreduce.OutputCommitter;
34 import org.apache.hadoop.mapreduce.OutputFormat;
35 import org.apache.hadoop.mapreduce.RecordWriter;
36 import org.apache.hadoop.mapreduce.TaskAttemptContext;
37 import org.apache.hadoop.conf.Configuration;
38
39
40
41
42
43
44
45
46 public class TableOutputFormat<KEY> extends OutputFormat<KEY, Writable> {
47
48 private final Log LOG = LogFactory.getLog(TableOutputFormat.class);
49
50 public static final String OUTPUT_TABLE = "hbase.mapred.outputtable";
51
52 public static final String QUORUM_ADDRESS = "hbase.mapred.output.quorum";
53
54 public static final String
55 REGION_SERVER_CLASS = "hbase.mapred.output.rs.class";
56
57 public static final String
58 REGION_SERVER_IMPL = "hbase.mapred.output.rs.impl";
59
60
61
62
63
64
65 protected static class TableRecordWriter<KEY>
66 extends RecordWriter<KEY, Writable> {
67
68
69 private HTable table;
70
71
72
73
74
75
76 public TableRecordWriter(HTable table) {
77 this.table = table;
78 }
79
80
81
82
83
84
85
86
87 @Override
88 public void close(TaskAttemptContext context)
89 throws IOException {
90 table.flushCommits();
91 }
92
93
94
95
96
97
98
99
100
101 @Override
102 public void write(KEY key, Writable value)
103 throws IOException {
104 if (value instanceof Put) this.table.put(new Put((Put)value));
105 else if (value instanceof Delete) this.table.delete(new Delete((Delete)value));
106 else throw new IOException("Pass a Delete or a Put");
107 }
108 }
109
110
111
112
113
114
115
116
117
118
119 @Override
120 public RecordWriter<KEY, Writable> getRecordWriter(
121 TaskAttemptContext context)
122 throws IOException, InterruptedException {
123
124 Configuration conf = new Configuration(context.getConfiguration());
125 String tableName = conf.get(OUTPUT_TABLE);
126 String address = conf.get(QUORUM_ADDRESS);
127 String serverClass = conf.get(REGION_SERVER_CLASS);
128 String serverImpl = conf.get(REGION_SERVER_IMPL);
129 HTable table = null;
130 try {
131 HBaseConfiguration.addHbaseResources(conf);
132 if (address != null) {
133
134 String[] parts = address.split(":");
135 conf.set(HConstants.ZOOKEEPER_QUORUM, parts[0]);
136 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, parts[1]);
137 }
138 if (serverClass != null) {
139 conf.set(HConstants.REGION_SERVER_CLASS, serverClass);
140 conf.set(HConstants.REGION_SERVER_IMPL, serverImpl);
141 }
142 table = new HTable(conf, tableName);
143 } catch(IOException e) {
144 LOG.error(e);
145 throw e;
146 }
147 table.setAutoFlush(false);
148 return new TableRecordWriter<KEY>(table);
149 }
150
151
152
153
154
155
156
157
158
159 @Override
160 public void checkOutputSpecs(JobContext context) throws IOException,
161 InterruptedException {
162
163
164 }
165
166
167
168
169
170
171
172
173
174
175 @Override
176 public OutputCommitter getOutputCommitter(TaskAttemptContext context)
177 throws IOException, InterruptedException {
178 return new TableOutputCommitter();
179 }
180
181 }