1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.replication.regionserver;
21  
22  import org.apache.commons.logging.Log;
23  import org.apache.commons.logging.LogFactory;
24  import org.apache.hadoop.conf.Configuration;
25  import org.apache.hadoop.fs.FileSystem;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.HBaseConfiguration;
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HConstants;
31  import org.apache.hadoop.hbase.HRegionInfo;
32  import org.apache.hadoop.hbase.HTableDescriptor;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.regionserver.HRegionServer;
35  import org.apache.hadoop.hbase.regionserver.wal.HLog;
36  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
37  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
38  import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
39  import org.apache.hadoop.hbase.replication.ReplicationZookeeperWrapper;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWrapper;
42  import org.junit.After;
43  import org.junit.AfterClass;
44  import org.junit.Before;
45  import org.junit.BeforeClass;
46  import org.junit.Test;
47  
48  import java.net.URLEncoder;
49  import java.util.concurrent.atomic.AtomicBoolean;
50  
51  import static org.junit.Assert.assertEquals;
52  
53  public class TestReplicationSourceManager {
54  
55    private static final Log LOG =
56        LogFactory.getLog(TestReplicationSourceManager.class);
57  
58    private static Configuration conf;
59  
60    private static HBaseTestingUtility utility;
61  
62    private static final AtomicBoolean STOPPER = new AtomicBoolean(false);
63  
64    private static final AtomicBoolean REPLICATING = new AtomicBoolean(false);
65  
66    private static ReplicationSourceManager manager;
67  
68    private static ZooKeeperWrapper zkw;
69  
70    private static HTableDescriptor htd;
71  
72    private static HRegionInfo hri;
73  
74    private static final byte[] r1 = Bytes.toBytes("r1");
75  
76    private static final byte[] r2 = Bytes.toBytes("r2");
77  
78    private static final byte[] f1 = Bytes.toBytes("f1");
79  
80    private static final byte[] f2 = Bytes.toBytes("f2");
81  
82    private static final byte[] test = Bytes.toBytes("test");
83  
84    private static FileSystem fs;
85  
86    private static Path oldLogDir;
87  
88    private static Path logDir;
89  
90  
91    @BeforeClass
92    public static void setUpBeforeClass() throws Exception {
93  
94      conf = HBaseConfiguration.create();
95      conf.set("replication.replicationsource.implementation",
96          ReplicationSourceDummy.class.getCanonicalName());
97      conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
98      utility = new HBaseTestingUtility(conf);
99      utility.startMiniZKCluster();
100 
101     zkw = ZooKeeperWrapper.createInstance(conf, "test");
102     zkw.writeZNode("/hbase", "replication", "");
103     zkw.writeZNode("/hbase/replication", "master",
104         conf.get(HConstants.ZOOKEEPER_QUORUM)+":" +
105     conf.get("hbase.zookeeper.property.clientPort")+":/1");
106     zkw.writeZNode("/hbase/replication/peers", "1",
107           conf.get(HConstants.ZOOKEEPER_QUORUM)+":" +
108           conf.get("hbase.zookeeper.property.clientPort")+":/1");
109 
110     HRegionServer server = new HRegionServer(conf);
111     ReplicationZookeeperWrapper helper = new ReplicationZookeeperWrapper(
112         server.getZooKeeperWrapper(), conf,
113         REPLICATING, "123456789");
114     fs = FileSystem.get(conf);
115     oldLogDir = new Path(utility.getTestDir(),
116         HConstants.HREGION_OLDLOGDIR_NAME);
117     logDir = new Path(utility.getTestDir(),
118         HConstants.HREGION_LOGDIR_NAME);
119 
120     manager = new ReplicationSourceManager(helper,
121         conf, STOPPER, fs, REPLICATING, logDir, oldLogDir);
122     manager.addSource("1");
123 
124     htd = new HTableDescriptor(test);
125     HColumnDescriptor col = new HColumnDescriptor("f1");
126     col.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
127     htd.addFamily(col);
128     col = new HColumnDescriptor("f2");
129     col.setScope(HConstants.REPLICATION_SCOPE_LOCAL);
130     htd.addFamily(col);
131 
132     hri = new HRegionInfo(htd, r1, r2);
133 
134 
135   }
136 
137   @AfterClass
138   public static void tearDownAfterClass() throws Exception {
139     manager.join();
140     utility.shutdownMiniCluster();
141   }
142 
143   @Before
144   public void setUp() throws Exception {
145     fs.delete(logDir, true);
146     fs.delete(oldLogDir, true);
147   }
148 
149   @After
150   public void tearDown() throws Exception {
151     setUp();
152   }
153 
154   @Test
155   public void testLogRoll() throws Exception {
156     long seq = 0;
157     long baseline = 1000;
158     long time = baseline;
159     KeyValue kv = new KeyValue(r1, f1, r1);
160     WALEdit edit = new WALEdit();
161     edit.add(kv);
162 
163     HLog hlog = new HLog(fs, logDir, oldLogDir, conf, null, manager,
164       URLEncoder.encode("regionserver:60020", "UTF8"));
165 
166     manager.init();
167 
168     // Testing normal log rolling every 20
169     for(long i = 1; i < 101; i++) {
170       if(i > 1 && i % 20 == 0) {
171         hlog.rollWriter();
172       }
173       LOG.info(i);
174       HLogKey key = new HLogKey(hri.getRegionName(),
175         test, seq++, System.currentTimeMillis());
176       hlog.append(hri, key, edit);
177     }
178 
179     // Simulate a rapid insert that's followed
180     // by a report that's still not totally complete (missing last one)
181     LOG.info(baseline + " and " + time);
182     baseline += 101;
183     time = baseline;
184     LOG.info(baseline + " and " + time);
185 
186     for (int i = 0; i < 3; i++) {
187       HLogKey key = new HLogKey(hri.getRegionName(),
188         test, seq++, System.currentTimeMillis());
189       hlog.append(hri, key, edit);
190     }
191 
192     assertEquals(6, manager.getHLogs().size());
193 
194     hlog.rollWriter();
195 
196     manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
197         "1", 0, false);
198 
199     HLogKey key = new HLogKey(hri.getRegionName(),
200           test, seq++, System.currentTimeMillis());
201     hlog.append(hri, key, edit);
202 
203     assertEquals(1, manager.getHLogs().size());
204 
205 
206     // TODO Need a case with only 2 HLogs and we only want to delete the first one
207   }
208 
209 }