1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.replication.regionserver;
21
22 import static org.junit.Assert.assertEquals;
23
24 import java.net.URLEncoder;
25 import java.util.ArrayList;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.conf.Configuration;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HBaseConfiguration;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.HColumnDescriptor;
36 import org.apache.hadoop.hbase.HConstants;
37 import org.apache.hadoop.hbase.HRegionInfo;
38 import org.apache.hadoop.hbase.HTableDescriptor;
39 import org.apache.hadoop.hbase.KeyValue;
40 import org.apache.hadoop.hbase.MediumTests;
41 import org.apache.hadoop.hbase.Server;
42 import org.apache.hadoop.hbase.ServerName;
43 import org.apache.hadoop.hbase.catalog.CatalogTracker;
44 import org.apache.hadoop.hbase.regionserver.wal.HLog;
45 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
46 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
47 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
48 import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
49 import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
52 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
53 import org.junit.After;
54 import org.junit.AfterClass;
55 import org.junit.Before;
56 import org.junit.BeforeClass;
57 import org.junit.Test;
58 import org.junit.experimental.categories.Category;
59
60 @Category(MediumTests.class)
61 public class TestReplicationSourceManager {
62
63 private static final Log LOG =
64 LogFactory.getLog(TestReplicationSourceManager.class);
65
66 private static Configuration conf;
67
68 private static HBaseTestingUtility utility;
69
70 private static Replication replication;
71
72 private static ReplicationSourceManager manager;
73
74 private static ZooKeeperWatcher zkw;
75
76 private static HTableDescriptor htd;
77
78 private static HRegionInfo hri;
79
80 private static final byte[] r1 = Bytes.toBytes("r1");
81
82 private static final byte[] r2 = Bytes.toBytes("r2");
83
84 private static final byte[] f1 = Bytes.toBytes("f1");
85
86 private static final byte[] test = Bytes.toBytes("test");
87
88 private static final String slaveId = "1";
89
90 private static FileSystem fs;
91
92 private static Path oldLogDir;
93
94 private static Path logDir;
95
96
97 @BeforeClass
98 public static void setUpBeforeClass() throws Exception {
99
100 conf = HBaseConfiguration.create();
101 conf.set("replication.replicationsource.implementation",
102 ReplicationSourceDummy.class.getCanonicalName());
103 conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
104 utility = new HBaseTestingUtility(conf);
105 utility.startMiniZKCluster();
106
107 zkw = new ZooKeeperWatcher(conf, "test", null);
108 ZKUtil.createWithParents(zkw, "/hbase/replication");
109 ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1");
110 ZKUtil.setData(zkw, "/hbase/replication/peers/1",
111 Bytes.toBytes(conf.get(HConstants.ZOOKEEPER_QUORUM) + ":"
112 + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":/1"));
113 ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/peer-state");
114 ZKUtil.setData(zkw, "/hbase/replication/peers/1/peer-state",
115 Bytes.toBytes(ReplicationZookeeper.PeerState.ENABLED.name()));
116 ZKUtil.createWithParents(zkw, "/hbase/replication/state");
117 ZKUtil.setData(zkw, "/hbase/replication/state", Bytes.toBytes("true"));
118
119 replication = new Replication(new DummyServer(), fs, logDir, oldLogDir);
120 manager = replication.getReplicationManager();
121 fs = FileSystem.get(conf);
122 oldLogDir = new Path(utility.getDataTestDir(),
123 HConstants.HREGION_OLDLOGDIR_NAME);
124 logDir = new Path(utility.getDataTestDir(),
125 HConstants.HREGION_LOGDIR_NAME);
126
127 manager.addSource(slaveId);
128
129 htd = new HTableDescriptor(test);
130 HColumnDescriptor col = new HColumnDescriptor("f1");
131 col.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
132 htd.addFamily(col);
133 col = new HColumnDescriptor("f2");
134 col.setScope(HConstants.REPLICATION_SCOPE_LOCAL);
135 htd.addFamily(col);
136
137 hri = new HRegionInfo(htd.getName(), r1, r2);
138
139
140 }
141
142 @AfterClass
143 public static void tearDownAfterClass() throws Exception {
144 manager.join();
145 utility.shutdownMiniCluster();
146 }
147
148 @Before
149 public void setUp() throws Exception {
150 fs.delete(logDir, true);
151 fs.delete(oldLogDir, true);
152 }
153
154 @After
155 public void tearDown() throws Exception {
156 setUp();
157 }
158
159 @Test
160 public void testLogRoll() throws Exception {
161 long seq = 0;
162 long baseline = 1000;
163 long time = baseline;
164 KeyValue kv = new KeyValue(r1, f1, r1);
165 WALEdit edit = new WALEdit();
166 edit.add(kv);
167
168 List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
169 listeners.add(replication);
170 HLog hlog = new HLog(fs, logDir, oldLogDir, conf, listeners,
171 URLEncoder.encode("regionserver:60020", "UTF8"));
172
173 manager.init();
174 HTableDescriptor htd = new HTableDescriptor();
175 htd.addFamily(new HColumnDescriptor(f1));
176
177 for(long i = 1; i < 101; i++) {
178 if(i > 1 && i % 20 == 0) {
179 hlog.rollWriter();
180 }
181 LOG.info(i);
182 HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
183 System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
184 hlog.append(hri, key, edit, htd, true);
185 }
186
187
188
189 LOG.info(baseline + " and " + time);
190 baseline += 101;
191 time = baseline;
192 LOG.info(baseline + " and " + time);
193
194 for (int i = 0; i < 3; i++) {
195 HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
196 System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
197 hlog.append(hri, key, edit, htd, true);
198 }
199
200 assertEquals(6, manager.getHLogs().get(slaveId).size());
201
202 hlog.rollWriter();
203
204 manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
205 "1", 0, false, false);
206
207 HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
208 System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
209 hlog.append(hri, key, edit, htd, true);
210
211 assertEquals(1, manager.getHLogs().size());
212
213
214
215 }
216
217 static class DummyServer implements Server {
218
219 @Override
220 public Configuration getConfiguration() {
221 return conf;
222 }
223
224 @Override
225 public ZooKeeperWatcher getZooKeeper() {
226 return zkw;
227 }
228
229 @Override
230 public CatalogTracker getCatalogTracker() {
231 return null;
232 }
233
234 @Override
235 public ServerName getServerName() {
236 return new ServerName("hostname.example.org", 1234, -1L);
237 }
238
239 @Override
240 public void abort(String why, Throwable e) {
241
242 }
243
244 @Override
245 public boolean isAborted() {
246 return false;
247 }
248
249 @Override
250 public void stop(String why) {
251
252 }
253
254 @Override
255 public boolean isStopped() {
256 return false;
257 }
258 }
259
260
261 @org.junit.Rule
262 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
263 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
264 }
265