1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.mapreduce;
20
21 import java.io.IOException;
22 import java.util.HashMap;
23 import java.util.Map;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.classification.InterfaceAudience;
28 import org.apache.hadoop.classification.InterfaceStability;
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.HBaseConfiguration;
31 import org.apache.hadoop.hbase.client.Delete;
32 import org.apache.hadoop.hbase.client.HTable;
33 import org.apache.hadoop.hbase.client.Mutation;
34 import org.apache.hadoop.hbase.client.Put;
35 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.apache.hadoop.mapreduce.JobContext;
38 import org.apache.hadoop.mapreduce.OutputCommitter;
39 import org.apache.hadoop.mapreduce.OutputFormat;
40 import org.apache.hadoop.mapreduce.RecordWriter;
41 import org.apache.hadoop.mapreduce.TaskAttemptContext;
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59 @InterfaceAudience.Public
60 @InterfaceStability.Stable
61 public class MultiTableOutputFormat extends OutputFormat<ImmutableBytesWritable, Mutation> {
62
63 public static final String WAL_PROPERTY = "hbase.mapreduce.multitableoutputformat.wal";
64
65 public static final boolean WAL_ON = true;
66
67 public static final boolean WAL_OFF = false;
68
69
70
71 protected static class MultiTableRecordWriter extends
72 RecordWriter<ImmutableBytesWritable, Mutation> {
73 private static final Log LOG = LogFactory.getLog(MultiTableRecordWriter.class);
74 Map<ImmutableBytesWritable, HTable> tables;
75 Configuration conf;
76 boolean useWriteAheadLogging;
77
78
79
80
81
82
83
84
85 public MultiTableRecordWriter(Configuration conf,
86 boolean useWriteAheadLogging) {
87 LOG.debug("Created new MultiTableRecordReader with WAL "
88 + (useWriteAheadLogging ? "on" : "off"));
89 this.tables = new HashMap<ImmutableBytesWritable, HTable>();
90 this.conf = conf;
91 this.useWriteAheadLogging = useWriteAheadLogging;
92 }
93
94
95
96
97
98
99
100
101 HTable getTable(ImmutableBytesWritable tableName) throws IOException {
102 if (!tables.containsKey(tableName)) {
103 LOG.debug("Opening HTable \"" + Bytes.toString(tableName.get())+ "\" for writing");
104 HTable table = new HTable(conf, tableName.get());
105 table.setAutoFlush(false);
106 tables.put(tableName, table);
107 }
108 return tables.get(tableName);
109 }
110
111 @Override
112 public void close(TaskAttemptContext context) throws IOException {
113 for (HTable table : tables.values()) {
114 table.flushCommits();
115 }
116 }
117
118
119
120
121
122
123
124
125
126
127
128 @Override
129 public void write(ImmutableBytesWritable tableName, Mutation action) throws IOException {
130 HTable table = getTable(tableName);
131
132 if (action instanceof Put) {
133 Put put = new Put((Put) action);
134 put.setWriteToWAL(useWriteAheadLogging);
135 table.put(put);
136 } else if (action instanceof Delete) {
137 Delete delete = new Delete((Delete) action);
138 table.delete(delete);
139 } else
140 throw new IllegalArgumentException(
141 "action must be either Delete or Put");
142 }
143 }
144
145 @Override
146 public void checkOutputSpecs(JobContext context) throws IOException,
147 InterruptedException {
148
149
150 }
151
152 @Override
153 public OutputCommitter getOutputCommitter(TaskAttemptContext context)
154 throws IOException, InterruptedException {
155 return new TableOutputCommitter();
156 }
157
158 @Override
159 public RecordWriter<ImmutableBytesWritable, Mutation> getRecordWriter(TaskAttemptContext context)
160 throws IOException, InterruptedException {
161 Configuration conf = context.getConfiguration();
162 return new MultiTableRecordWriter(HBaseConfiguration.create(conf),
163 conf.getBoolean(WAL_PROPERTY, WAL_ON));
164 }
165
166 }