1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.replication.regionserver;
21  
22  import static org.junit.Assert.assertEquals;
23  
24  import java.net.URLEncoder;
25  import java.util.ArrayList;
26  import java.util.List;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.Path;
33  import org.apache.hadoop.hbase.HBaseConfiguration;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.HColumnDescriptor;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.HTableDescriptor;
39  import org.apache.hadoop.hbase.KeyValue;
40  import org.apache.hadoop.hbase.MediumTests;
41  import org.apache.hadoop.hbase.Server;
42  import org.apache.hadoop.hbase.ServerName;
43  import org.apache.hadoop.hbase.catalog.CatalogTracker;
44  import org.apache.hadoop.hbase.regionserver.wal.HLog;
45  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
46  import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
47  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
48  import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
49  import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
52  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
53  import org.junit.After;
54  import org.junit.AfterClass;
55  import org.junit.Before;
56  import org.junit.BeforeClass;
57  import org.junit.Test;
58  import org.junit.experimental.categories.Category;
59  
60  @Category(MediumTests.class)
61  public class TestReplicationSourceManager {
62  
63    private static final Log LOG =
64        LogFactory.getLog(TestReplicationSourceManager.class);
65  
66    private static Configuration conf;
67  
68    private static HBaseTestingUtility utility;
69  
70    private static Replication replication;
71  
72    private static ReplicationSourceManager manager;
73  
74    private static ZooKeeperWatcher zkw;
75  
76    private static HTableDescriptor htd;
77  
78    private static HRegionInfo hri;
79  
80    private static final byte[] r1 = Bytes.toBytes("r1");
81  
82    private static final byte[] r2 = Bytes.toBytes("r2");
83  
84    private static final byte[] f1 = Bytes.toBytes("f1");
85  
86    private static final byte[] test = Bytes.toBytes("test");
87  
88    private static final String slaveId = "1";
89  
90    private static FileSystem fs;
91  
92    private static Path oldLogDir;
93  
94    private static Path logDir;
95  
96  
97    @BeforeClass
98    public static void setUpBeforeClass() throws Exception {
99  
100     conf = HBaseConfiguration.create();
101     conf.set("replication.replicationsource.implementation",
102         ReplicationSourceDummy.class.getCanonicalName());
103     conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
104     utility = new HBaseTestingUtility(conf);
105     utility.startMiniZKCluster();
106 
107     zkw = new ZooKeeperWatcher(conf, "test", null);
108     ZKUtil.createWithParents(zkw, "/hbase/replication");
109     ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1");
110     ZKUtil.setData(zkw, "/hbase/replication/peers/1",
111         Bytes.toBytes(conf.get(HConstants.ZOOKEEPER_QUORUM) + ":"
112             + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":/1"));
113     ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/peer-state");
114     ZKUtil.setData(zkw, "/hbase/replication/peers/1/peer-state",
115         Bytes.toBytes(ReplicationZookeeper.PeerState.ENABLED.name()));
116     ZKUtil.createWithParents(zkw, "/hbase/replication/state");
117     ZKUtil.setData(zkw, "/hbase/replication/state", Bytes.toBytes("true"));
118 
119     replication = new Replication(new DummyServer(), fs, logDir, oldLogDir);
120     manager = replication.getReplicationManager();
121     fs = FileSystem.get(conf);
122     oldLogDir = new Path(utility.getDataTestDir(),
123         HConstants.HREGION_OLDLOGDIR_NAME);
124     logDir = new Path(utility.getDataTestDir(),
125         HConstants.HREGION_LOGDIR_NAME);
126 
127     manager.addSource(slaveId);
128 
129     htd = new HTableDescriptor(test);
130     HColumnDescriptor col = new HColumnDescriptor("f1");
131     col.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
132     htd.addFamily(col);
133     col = new HColumnDescriptor("f2");
134     col.setScope(HConstants.REPLICATION_SCOPE_LOCAL);
135     htd.addFamily(col);
136 
137     hri = new HRegionInfo(htd.getName(), r1, r2);
138 
139 
140   }
141 
142   @AfterClass
143   public static void tearDownAfterClass() throws Exception {
144     manager.join();
145     utility.shutdownMiniCluster();
146   }
147 
148   @Before
149   public void setUp() throws Exception {
150     fs.delete(logDir, true);
151     fs.delete(oldLogDir, true);
152   }
153 
154   @After
155   public void tearDown() throws Exception {
156     setUp();
157   }
158 
159   @Test
160   public void testLogRoll() throws Exception {
161     long seq = 0;
162     long baseline = 1000;
163     long time = baseline;
164     KeyValue kv = new KeyValue(r1, f1, r1);
165     WALEdit edit = new WALEdit();
166     edit.add(kv);
167 
168     List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
169     listeners.add(replication);
170     HLog hlog = new HLog(fs, logDir, oldLogDir, conf, listeners,
171       URLEncoder.encode("regionserver:60020", "UTF8"));
172 
173     manager.init();
174     HTableDescriptor htd = new HTableDescriptor();
175     htd.addFamily(new HColumnDescriptor(f1));
176     // Testing normal log rolling every 20
177     for(long i = 1; i < 101; i++) {
178       if(i > 1 && i % 20 == 0) {
179         hlog.rollWriter();
180       }
181       LOG.info(i);
182       HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
183           System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
184       hlog.append(hri, key, edit, htd, true);
185     }
186 
187     // Simulate a rapid insert that's followed
188     // by a report that's still not totally complete (missing last one)
189     LOG.info(baseline + " and " + time);
190     baseline += 101;
191     time = baseline;
192     LOG.info(baseline + " and " + time);
193 
194     for (int i = 0; i < 3; i++) {
195       HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
196           System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
197       hlog.append(hri, key, edit, htd, true);
198     }
199 
200     assertEquals(6, manager.getHLogs().get(slaveId).size());
201 
202     hlog.rollWriter();
203 
204     manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
205         "1", 0, false, false);
206 
207     HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
208         System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
209     hlog.append(hri, key, edit, htd, true);
210 
211     assertEquals(1, manager.getHLogs().size());
212 
213 
214     // TODO Need a case with only 2 HLogs and we only want to delete the first one
215   }
216 
217   static class DummyServer implements Server {
218 
219     @Override
220     public Configuration getConfiguration() {
221       return conf;
222     }
223 
224     @Override
225     public ZooKeeperWatcher getZooKeeper() {
226       return zkw;
227     }
228 
229     @Override
230     public CatalogTracker getCatalogTracker() {
231       return null;  //To change body of implemented methods use File | Settings | File Templates.
232     }
233 
234     @Override
235     public ServerName getServerName() {
236       return new ServerName("hostname.example.org", 1234, -1L);
237     }
238 
239     @Override
240     public void abort(String why, Throwable e) {
241       //To change body of implemented methods use File | Settings | File Templates.
242     }
243     
244     @Override
245     public boolean isAborted() {
246       return false;
247     }
248 
249     @Override
250     public void stop(String why) {
251       //To change body of implemented methods use File | Settings | File Templates.
252     }
253 
254     @Override
255     public boolean isStopped() {
256       return false;  //To change body of implemented methods use File | Settings | File Templates.
257     }
258   }
259 
260 
261   @org.junit.Rule
262   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
263     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
264 }
265