1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.replication;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.List;
24 import java.util.UUID;
25 import java.util.concurrent.atomic.AtomicBoolean;
26 import java.util.concurrent.atomic.AtomicInteger;
27 import java.util.concurrent.atomic.AtomicReference;
28
29 import org.apache.hadoop.hbase.KeyValue;
30 import org.apache.hadoop.hbase.testclassification.MediumTests;
31 import org.apache.hadoop.hbase.Waiter;
32 import org.apache.hadoop.hbase.client.HTable;
33 import org.apache.hadoop.hbase.client.Put;
34 import org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException;
35 import org.apache.hadoop.hbase.regionserver.wal.HLog;
36 import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
37 import org.apache.hadoop.hbase.util.Bytes;
38 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
39 import org.apache.hadoop.hbase.zookeeper.ZKConfig;
40 import org.junit.AfterClass;
41 import org.junit.Assert;
42 import org.junit.Before;
43 import org.junit.BeforeClass;
44 import org.junit.Test;
45 import org.junit.experimental.categories.Category;
46
47
48
49
50 @Category(MediumTests.class)
51 public class TestReplicationEndpoint extends TestReplicationBase {
52
53 static int numRegionServers;
54
55 @BeforeClass
56 public static void setUpBeforeClass() throws Exception {
57 TestReplicationBase.setUpBeforeClass();
58 utility2.shutdownMiniCluster();
59 admin.removePeer("2");
60 numRegionServers = utility1.getHBaseCluster().getRegionServerThreads().size();
61 }
62
63 @AfterClass
64 public static void tearDownAfterClass() throws Exception {
65 TestReplicationBase.tearDownAfterClass();
66
67 Assert.assertTrue(ReplicationEndpointForTest.stoppedCount.get() > 0);
68 }
69
70 @Before
71 public void setup() throws FailedLogCloseException, IOException {
72 ReplicationEndpointForTest.contructedCount.set(0);
73 ReplicationEndpointForTest.startedCount.set(0);
74 ReplicationEndpointForTest.replicateCount.set(0);
75 ReplicationEndpointForTest.lastEntries = null;
76 for (RegionServerThread rs : utility1.getMiniHBaseCluster().getRegionServerThreads()) {
77 utility1.getHBaseAdmin().rollHLogWriter(rs.getRegionServer().getServerName().toString());
78 }
79 }
80
81 @Test
82 public void testCustomReplicationEndpoint() throws Exception {
83
84 admin.addPeer("testCustomReplicationEndpoint",
85 new ReplicationPeerConfig().setClusterKey(ZKConfig.getZooKeeperClusterKey(conf1))
86 .setReplicationEndpointImpl(ReplicationEndpointForTest.class.getName()), null);
87
88
89 Waiter.waitFor(conf1, 60000, new Waiter.Predicate<Exception>() {
90 @Override
91 public boolean evaluate() throws Exception {
92 return ReplicationEndpointForTest.contructedCount.get() >= numRegionServers;
93 }
94 });
95
96 Waiter.waitFor(conf1, 60000, new Waiter.Predicate<Exception>() {
97 @Override
98 public boolean evaluate() throws Exception {
99 return ReplicationEndpointForTest.startedCount.get() >= numRegionServers;
100 }
101 });
102
103 Assert.assertEquals(0, ReplicationEndpointForTest.replicateCount.get());
104
105
106 doPut(Bytes.toBytes("row42"));
107
108 Waiter.waitFor(conf1, 60000, new Waiter.Predicate<Exception>() {
109 @Override
110 public boolean evaluate() throws Exception {
111 return ReplicationEndpointForTest.replicateCount.get() >= 1;
112 }
113 });
114
115 doAssert(Bytes.toBytes("row42"));
116
117 admin.removePeer("testCustomReplicationEndpoint");
118 }
119
120 @Test
121 public void testReplicationEndpointReturnsFalseOnReplicate() throws Exception {
122 Assert.assertEquals(0, ReplicationEndpointForTest.replicateCount.get());
123 Assert.assertTrue(!ReplicationEndpointReturningFalse.replicated.get());
124 final String id = "testReplicationEndpointReturnsFalseOnReplicate";
125 admin.addPeer(id,
126 new ReplicationPeerConfig().setClusterKey(ZKConfig.getZooKeeperClusterKey(conf1))
127 .setReplicationEndpointImpl(ReplicationEndpointReturningFalse.class.getName()), null);
128
129 doPut(row);
130
131 Waiter.waitFor(conf1, 60000, new Waiter.Predicate<Exception>() {
132 @Override
133 public boolean evaluate() throws Exception {
134 return ReplicationEndpointReturningFalse.replicated.get();
135 }
136 });
137 if (ReplicationEndpointReturningFalse.ex.get() != null) {
138 throw ReplicationEndpointReturningFalse.ex.get();
139 }
140
141 admin.removePeer("testReplicationEndpointReturnsFalseOnReplicate");
142 }
143
144 @Test (timeout=120000)
145 public void testWALEntryFilterFromReplicationEndpoint() throws Exception {
146 admin.addPeer("testWALEntryFilterFromReplicationEndpoint",
147 new ReplicationPeerConfig().setClusterKey(ZKConfig.getZooKeeperClusterKey(conf1))
148 .setReplicationEndpointImpl(ReplicationEndpointWithWALEntryFilter.class.getName()), null);
149
150 doPut(Bytes.toBytes("row1"));
151 doPut(row);
152 doPut(Bytes.toBytes("row2"));
153
154 Waiter.waitFor(conf1, 60000, new Waiter.Predicate<Exception>() {
155 @Override
156 public boolean evaluate() throws Exception {
157 return ReplicationEndpointForTest.replicateCount.get() >= 1;
158 }
159 });
160
161 Assert.assertNull(ReplicationEndpointWithWALEntryFilter.ex.get());
162 admin.removePeer("testWALEntryFilterFromReplicationEndpoint");
163 }
164
165
166 private void doPut(byte[] row) throws IOException {
167 Put put = new Put(row);
168 put.add(famName, row, row);
169 htable1 = new HTable(conf1, tableName);
170 htable1.put(put);
171 htable1.close();
172 }
173
174 private static void doAssert(byte[] row) throws Exception {
175 if (ReplicationEndpointForTest.lastEntries == null) {
176 return;
177 }
178 Assert.assertEquals(1, ReplicationEndpointForTest.lastEntries.size());
179 List<KeyValue> kvs = ReplicationEndpointForTest.lastEntries.get(0).getEdit().getKeyValues();
180 Assert.assertEquals(1, kvs.size());
181 Assert.assertTrue(Bytes.equals(kvs.get(0).getRowArray(), kvs.get(0).getRowOffset(),
182 kvs.get(0).getRowLength(), row, 0, row.length));
183 }
184
185 public static class ReplicationEndpointForTest extends BaseReplicationEndpoint {
186 static UUID uuid = UUID.randomUUID();
187 static AtomicInteger contructedCount = new AtomicInteger();
188 static AtomicInteger startedCount = new AtomicInteger();
189 static AtomicInteger stoppedCount = new AtomicInteger();
190 static AtomicInteger replicateCount = new AtomicInteger();
191 static volatile List<HLog.Entry> lastEntries = null;
192
193 public ReplicationEndpointForTest() {
194 contructedCount.incrementAndGet();
195 }
196
197 @Override
198 public UUID getPeerUUID() {
199 return uuid;
200 }
201
202 @Override
203 public boolean replicate(ReplicateContext replicateContext) {
204 replicateCount.incrementAndGet();
205 lastEntries = replicateContext.entries;
206 return true;
207 }
208
209 @Override
210 protected void doStart() {
211 startedCount.incrementAndGet();
212 notifyStarted();
213 }
214
215 @Override
216 protected void doStop() {
217 stoppedCount.incrementAndGet();
218 notifyStopped();
219 }
220 }
221
222 public static class ReplicationEndpointReturningFalse extends ReplicationEndpointForTest {
223 static AtomicReference<Exception> ex = new AtomicReference<Exception>(null);
224 static AtomicBoolean replicated = new AtomicBoolean(false);
225 @Override
226 public boolean replicate(ReplicateContext replicateContext) {
227 try {
228
229 doAssert(row);
230 } catch (Exception e) {
231 ex.set(e);
232 }
233
234 super.replicate(replicateContext);
235
236 replicated.set(replicateCount.get() > 10);
237 return replicated.get();
238 }
239 }
240
241
242 public static class ReplicationEndpointWithWALEntryFilter extends ReplicationEndpointForTest {
243 static AtomicReference<Exception> ex = new AtomicReference<Exception>(null);
244
245 @Override
246 public boolean replicate(ReplicateContext replicateContext) {
247 try {
248 super.replicate(replicateContext);
249 doAssert(row);
250 } catch (Exception e) {
251 ex.set(e);
252 }
253 return true;
254 }
255
256 @Override
257 public WALEntryFilter getWALEntryfilter() {
258 return new ChainWALEntryFilter(super.getWALEntryfilter(), new WALEntryFilter() {
259 @Override
260 public Entry filter(Entry entry) {
261 ArrayList<KeyValue> kvs = entry.getEdit().getKeyValues();
262 int size = kvs.size();
263 for (int i = size-1; i >= 0; i--) {
264 KeyValue kv = kvs.get(i);
265 if (!Bytes.equals(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
266 row, 0, row.length)) {
267 kvs.remove(i);
268 }
269 }
270 return entry;
271 }
272 });
273 }
274 }
275 }