1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.replication.regionserver;
21
22 import org.apache.commons.logging.Log;
23 import org.apache.commons.logging.LogFactory;
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.FileSystem;
26 import org.apache.hadoop.fs.Path;
27 import org.apache.hadoop.hbase.HBaseConfiguration;
28 import org.apache.hadoop.hbase.HBaseTestingUtility;
29 import org.apache.hadoop.hbase.HColumnDescriptor;
30 import org.apache.hadoop.hbase.HConstants;
31 import org.apache.hadoop.hbase.HRegionInfo;
32 import org.apache.hadoop.hbase.HTableDescriptor;
33 import org.apache.hadoop.hbase.KeyValue;
34 import org.apache.hadoop.hbase.regionserver.HRegionServer;
35 import org.apache.hadoop.hbase.regionserver.wal.HLog;
36 import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
37 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
38 import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
39 import org.apache.hadoop.hbase.replication.ReplicationZookeeperWrapper;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
42 import org.junit.After;
43 import org.junit.AfterClass;
44 import org.junit.Before;
45 import org.junit.BeforeClass;
46 import org.junit.Test;
47
48 import java.net.URLEncoder;
49 import java.util.concurrent.atomic.AtomicBoolean;
50
51 import static org.junit.Assert.assertEquals;
52
53 public class TestReplicationSourceManager {
54
55 private static final Log LOG =
56 LogFactory.getLog(TestReplicationSourceManager.class);
57
58 private static Configuration conf;
59
60 private static HBaseTestingUtility utility;
61
62 private static final AtomicBoolean STOPPER = new AtomicBoolean(false);
63
64 private static final AtomicBoolean REPLICATING = new AtomicBoolean(false);
65
66 private static ReplicationSourceManager manager;
67
68 private static ZooKeeperWrapper zkw;
69
70 private static HTableDescriptor htd;
71
72 private static HRegionInfo hri;
73
74 private static final byte[] r1 = Bytes.toBytes("r1");
75
76 private static final byte[] r2 = Bytes.toBytes("r2");
77
78 private static final byte[] f1 = Bytes.toBytes("f1");
79
80 private static final byte[] f2 = Bytes.toBytes("f2");
81
82 private static final byte[] test = Bytes.toBytes("test");
83
84 private static FileSystem fs;
85
86 private static Path oldLogDir;
87
88 private static Path logDir;
89
90
91 @BeforeClass
92 public static void setUpBeforeClass() throws Exception {
93
94 conf = HBaseConfiguration.create();
95 conf.set("replication.replicationsource.implementation",
96 ReplicationSourceDummy.class.getCanonicalName());
97 conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
98 utility = new HBaseTestingUtility(conf);
99 utility.startMiniZKCluster();
100
101 zkw = ZooKeeperWrapper.createInstance(conf, "test");
102 zkw.writeZNode("/hbase", "replication", "");
103 zkw.writeZNode("/hbase/replication", "master",
104 conf.get(HConstants.ZOOKEEPER_QUORUM)+":" +
105 conf.get("hbase.zookeeper.property.clientPort")+":/1");
106 zkw.writeZNode("/hbase/replication/peers", "1",
107 conf.get(HConstants.ZOOKEEPER_QUORUM)+":" +
108 conf.get("hbase.zookeeper.property.clientPort")+":/1");
109
110 HRegionServer server = new HRegionServer(conf);
111 ReplicationZookeeperWrapper helper = new ReplicationZookeeperWrapper(
112 server.getZooKeeperWrapper(), conf,
113 REPLICATING, "123456789");
114 fs = FileSystem.get(conf);
115 oldLogDir = new Path(utility.getTestDir(),
116 HConstants.HREGION_OLDLOGDIR_NAME);
117 logDir = new Path(utility.getTestDir(),
118 HConstants.HREGION_LOGDIR_NAME);
119
120 manager = new ReplicationSourceManager(helper,
121 conf, STOPPER, fs, REPLICATING, logDir, oldLogDir);
122 manager.addSource("1");
123
124 htd = new HTableDescriptor(test);
125 HColumnDescriptor col = new HColumnDescriptor("f1");
126 col.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
127 htd.addFamily(col);
128 col = new HColumnDescriptor("f2");
129 col.setScope(HConstants.REPLICATION_SCOPE_LOCAL);
130 htd.addFamily(col);
131
132 hri = new HRegionInfo(htd, r1, r2);
133
134
135 }
136
137 @AfterClass
138 public static void tearDownAfterClass() throws Exception {
139 manager.join();
140 utility.shutdownMiniCluster();
141 }
142
143 @Before
144 public void setUp() throws Exception {
145 fs.delete(logDir, true);
146 fs.delete(oldLogDir, true);
147 }
148
149 @After
150 public void tearDown() throws Exception {
151 setUp();
152 }
153
154 @Test
155 public void testLogRoll() throws Exception {
156 long seq = 0;
157 long baseline = 1000;
158 long time = baseline;
159 KeyValue kv = new KeyValue(r1, f1, r1);
160 WALEdit edit = new WALEdit();
161 edit.add(kv);
162
163 HLog hlog = new HLog(fs, logDir, oldLogDir, conf, null, manager,
164 URLEncoder.encode("regionserver:60020", "UTF8"));
165
166 manager.init();
167
168
169 for(long i = 1; i < 101; i++) {
170 if(i > 1 && i % 20 == 0) {
171 hlog.rollWriter();
172 }
173 LOG.info(i);
174 HLogKey key = new HLogKey(hri.getRegionName(),
175 test, seq++, System.currentTimeMillis());
176 hlog.append(hri, key, edit);
177 }
178
179
180
181 LOG.info(baseline + " and " + time);
182 baseline += 101;
183 time = baseline;
184 LOG.info(baseline + " and " + time);
185
186 for (int i = 0; i < 3; i++) {
187 HLogKey key = new HLogKey(hri.getRegionName(),
188 test, seq++, System.currentTimeMillis());
189 hlog.append(hri, key, edit);
190 }
191
192 assertEquals(6, manager.getHLogs().size());
193
194 hlog.rollWriter();
195
196 manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
197 "1", 0, false);
198
199 HLogKey key = new HLogKey(hri.getRegionName(),
200 test, seq++, System.currentTimeMillis());
201 hlog.append(hri, key, edit);
202
203 assertEquals(1, manager.getHLogs().size());
204
205
206
207 }
208
209 }