1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.replication.regionserver;
21
22 import org.apache.commons.logging.Log;
23 import org.apache.commons.logging.LogFactory;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.KeyValue;
27 import org.apache.hadoop.hbase.TableNotFoundException;
28 import org.apache.hadoop.hbase.client.Delete;
29 import org.apache.hadoop.hbase.client.HTableInterface;
30 import org.apache.hadoop.hbase.client.HTablePool;
31 import org.apache.hadoop.hbase.client.Put;
32 import org.apache.hadoop.hbase.regionserver.wal.HLog;
33 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
34 import org.apache.hadoop.hbase.util.Bytes;
35
36 import java.io.IOException;
37 import java.util.ArrayList;
38 import java.util.List;
39 import java.util.concurrent.atomic.AtomicBoolean;
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55 public class ReplicationSink {
56
57 private static final Log LOG = LogFactory.getLog(ReplicationSink.class);
58
59 public static final String REPLICATION_LOG_DIR = ".replogs";
60 private final Configuration conf;
61
62 private final HTablePool pool;
63
64 private final AtomicBoolean stop;
65 private final ReplicationSinkMetrics metrics;
66
67
68
69
70
71
72
73
74 public ReplicationSink(Configuration conf, AtomicBoolean stopper)
75 throws IOException {
76 this.conf = conf;
77 this.pool = new HTablePool(this.conf,
78 conf.getInt("replication.sink.htablepool.capacity", 10));
79 this.stop = stopper;
80 this.metrics = new ReplicationSinkMetrics();
81 }
82
83
84
85
86
87
88
89
90 public synchronized void replicateEntries(HLog.Entry[] entries)
91 throws IOException {
92 if (entries.length == 0) {
93 return;
94 }
95
96
97 try {
98 long totalReplicated = 0;
99 byte[] lastTable = HConstants.EMPTY_BYTE_ARRAY;
100 List<Put> puts = new ArrayList<Put>();
101 for (HLog.Entry entry : entries) {
102 WALEdit edit = entry.getEdit();
103 List<KeyValue> kvs = edit.getKeyValues();
104 if (kvs.get(0).isDelete()) {
105 Delete delete = new Delete(kvs.get(0).getRow(),
106 kvs.get(0).getTimestamp(), null);
107 for (KeyValue kv : kvs) {
108 if (kv.isDeleteFamily()) {
109 delete.deleteFamily(kv.getFamily());
110 } else if (!kv.isEmptyColumn()) {
111 delete.deleteColumn(kv.getFamily(),
112 kv.getQualifier());
113 }
114 }
115 delete(entry.getKey().getTablename(), delete);
116 } else {
117
118 if (!Bytes.equals(lastTable, entry.getKey().getTablename())) {
119 put(lastTable, puts);
120 }
121
122 byte[] lastKey = kvs.get(0).getRow();
123 Put put = new Put(kvs.get(0).getRow(),
124 kvs.get(0).getTimestamp());
125 for (KeyValue kv : kvs) {
126 if (!Bytes.equals(lastKey, kv.getRow())) {
127 puts.add(put);
128 put = new Put(kv.getRow(), kv.getTimestamp());
129 }
130 put.add(kv.getFamily(), kv.getQualifier(), kv.getValue());
131 lastKey = kv.getRow();
132 }
133 puts.add(put);
134 lastTable = entry.getKey().getTablename();
135 }
136 totalReplicated++;
137 }
138 put(lastTable, puts);
139 this.metrics.setAgeOfLastAppliedOp(
140 entries[entries.length-1].getKey().getWriteTime());
141 this.metrics.appliedBatchesRate.inc(1);
142 LOG.info("Total replicated: " + totalReplicated);
143 } catch (IOException ex) {
144 if (ex.getCause() instanceof TableNotFoundException) {
145 LOG.warn("Losing edits because: ", ex);
146 } else {
147
148 LOG.error("Unable to accept edit because", ex);
149 this.stop.set(true);
150 throw ex;
151 }
152 } catch (RuntimeException re) {
153 if (re.getCause() instanceof TableNotFoundException) {
154 LOG.warn("Losing edits because: ", re);
155 } else {
156 this.stop.set(true);
157 throw re;
158 }
159 }
160 }
161
162
163
164
165
166
167
168 private void put(byte[] tableName, List<Put> puts) throws IOException {
169 if (puts.isEmpty()) {
170 return;
171 }
172 HTableInterface table = null;
173 try {
174 table = this.pool.getTable(tableName);
175 table.put(puts);
176 this.metrics.appliedOpsRate.inc(puts.size());
177 this.pool.putTable(table);
178 puts.clear();
179 } finally {
180 if (table != null) {
181 this.pool.putTable(table);
182 }
183 }
184 }
185
186
187
188
189
190
191
192 private void delete(byte[] tableName, Delete delete) throws IOException {
193 HTableInterface table = null;
194 try {
195 table = this.pool.getTable(tableName);
196 table.delete(delete);
197 this.metrics.appliedOpsRate.inc(1);
198 this.pool.putTable(table);
199 } finally {
200 if (table != null) {
201 this.pool.putTable(table);
202 }
203 }
204 }
205 }