1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.replication.regionserver;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.net.URLEncoder;
26  import java.util.ArrayList;
27  import java.util.Collection;
28  import java.util.List;
29  import java.util.SortedMap;
30  import java.util.SortedSet;
31  import java.util.concurrent.CountDownLatch;
32  import java.util.concurrent.atomic.AtomicBoolean;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.fs.Path;
39  import org.apache.hadoop.hbase.HBaseConfiguration;
40  import org.apache.hadoop.hbase.HBaseTestingUtility;
41  import org.apache.hadoop.hbase.HColumnDescriptor;
42  import org.apache.hadoop.hbase.HConstants;
43  import org.apache.hadoop.hbase.HRegionInfo;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.KeyValue;
46  import org.apache.hadoop.hbase.MediumTests;
47  import org.apache.hadoop.hbase.Server;
48  import org.apache.hadoop.hbase.ServerName;
49  import org.apache.hadoop.hbase.catalog.CatalogTracker;
50  import org.apache.hadoop.hbase.regionserver.wal.HLog;
51  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
52  import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
53  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
54  import org.apache.hadoop.hbase.replication.ReplicationSourceDummy;
55  import org.apache.hadoop.hbase.replication.ReplicationZookeeper;
56  import org.apache.hadoop.hbase.util.Bytes;
57  import org.apache.hadoop.hbase.zookeeper.ZKUtil;
58  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
59  import org.junit.After;
60  import org.junit.AfterClass;
61  import org.junit.Before;
62  import org.junit.BeforeClass;
63  import org.junit.Test;
64  import org.junit.experimental.categories.Category;
65  
66  @Category(MediumTests.class)
67  public class TestReplicationSourceManager {
68  
69    private static final Log LOG =
70        LogFactory.getLog(TestReplicationSourceManager.class);
71  
72    private static Configuration conf;
73  
74    private static HBaseTestingUtility utility;
75  
76    private static Replication replication;
77  
78    private static ReplicationSourceManager manager;
79  
80    private static ZooKeeperWatcher zkw;
81  
82    private static HTableDescriptor htd;
83  
84    private static HRegionInfo hri;
85  
86    private static final byte[] r1 = Bytes.toBytes("r1");
87  
88    private static final byte[] r2 = Bytes.toBytes("r2");
89  
90    private static final byte[] f1 = Bytes.toBytes("f1");
91  
92    private static final byte[] test = Bytes.toBytes("test");
93  
94    private static final String slaveId = "1";
95  
96    private static FileSystem fs;
97  
98    private static Path oldLogDir;
99  
100   private static Path logDir;
101   
102   private static CountDownLatch latch;
103 
104   private static List<String> files = new ArrayList<String>();
105 
106   @BeforeClass
107   public static void setUpBeforeClass() throws Exception {
108 
109     conf = HBaseConfiguration.create();
110     conf.set("replication.replicationsource.implementation",
111         ReplicationSourceDummy.class.getCanonicalName());
112     conf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
113     utility = new HBaseTestingUtility(conf);
114     utility.startMiniZKCluster();
115 
116     zkw = new ZooKeeperWatcher(conf, "test", null);
117     ZKUtil.createWithParents(zkw, "/hbase/replication");
118     ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1");
119     ZKUtil.setData(zkw, "/hbase/replication/peers/1",
120         Bytes.toBytes(conf.get(HConstants.ZOOKEEPER_QUORUM) + ":"
121             + conf.get(HConstants.ZOOKEEPER_CLIENT_PORT) + ":/1"));
122     ZKUtil.createWithParents(zkw, "/hbase/replication/peers/1/peer-state");
123     ZKUtil.setData(zkw, "/hbase/replication/peers/1/peer-state",
124         Bytes.toBytes(ReplicationZookeeper.PeerState.ENABLED.name()));
125     ZKUtil.createWithParents(zkw, "/hbase/replication/state");
126     ZKUtil.setData(zkw, "/hbase/replication/state", Bytes.toBytes("true"));
127 
128     replication = new Replication(new DummyServer(), fs, logDir, oldLogDir);
129     manager = replication.getReplicationManager();
130     fs = FileSystem.get(conf);
131     oldLogDir = new Path(utility.getDataTestDir(),
132         HConstants.HREGION_OLDLOGDIR_NAME);
133     logDir = new Path(utility.getDataTestDir(),
134         HConstants.HREGION_LOGDIR_NAME);
135 
136     manager.addSource(slaveId);
137 
138     htd = new HTableDescriptor(test);
139     HColumnDescriptor col = new HColumnDescriptor("f1");
140     col.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
141     htd.addFamily(col);
142     col = new HColumnDescriptor("f2");
143     col.setScope(HConstants.REPLICATION_SCOPE_LOCAL);
144     htd.addFamily(col);
145 
146     hri = new HRegionInfo(htd.getName(), r1, r2);
147 
148 
149   }
150 
151   @AfterClass
152   public static void tearDownAfterClass() throws Exception {
153     manager.join();
154     utility.shutdownMiniCluster();
155   }
156 
157   @Before
158   public void setUp() throws Exception {
159     fs.delete(logDir, true);
160     fs.delete(oldLogDir, true);
161   }
162 
163   @After
164   public void tearDown() throws Exception {
165     setUp();
166   }
167 
168   @Test
169   public void testLogRoll() throws Exception {
170     long seq = 0;
171     long baseline = 1000;
172     long time = baseline;
173     KeyValue kv = new KeyValue(r1, f1, r1);
174     WALEdit edit = new WALEdit();
175     edit.add(kv);
176 
177     List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
178     listeners.add(replication);
179     HLog hlog = new HLog(fs, logDir, oldLogDir, conf, listeners,
180       URLEncoder.encode("regionserver:60020", "UTF8"));
181 
182     manager.init();
183     HTableDescriptor htd = new HTableDescriptor();
184     htd.addFamily(new HColumnDescriptor(f1));
185     // Testing normal log rolling every 20
186     for(long i = 1; i < 101; i++) {
187       if(i > 1 && i % 20 == 0) {
188         hlog.rollWriter();
189       }
190       LOG.info(i);
191       HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
192           System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
193       hlog.append(hri, key, edit, htd, true);
194     }
195 
196     // Simulate a rapid insert that's followed
197     // by a report that's still not totally complete (missing last one)
198     LOG.info(baseline + " and " + time);
199     baseline += 101;
200     time = baseline;
201     LOG.info(baseline + " and " + time);
202 
203     for (int i = 0; i < 3; i++) {
204       HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
205           System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
206       hlog.append(hri, key, edit, htd, true);
207     }
208 
209     assertEquals(6, manager.getHLogs().get(slaveId).size());
210 
211     hlog.rollWriter();
212 
213     manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
214         "1", 0, false, false);
215 
216     HLogKey key = new HLogKey(hri.getRegionName(), test, seq++,
217         System.currentTimeMillis(), HConstants.DEFAULT_CLUSTER_ID);
218     hlog.append(hri, key, edit, htd, true);
219 
220     assertEquals(1, manager.getHLogs().size());
221 
222 
223     // TODO Need a case with only 2 HLogs and we only want to delete the first one
224   }
225   
226   @Test
227   public void testNodeFailoverWorkerCopyQueuesFromRSUsingMulti() throws Exception {
228     LOG.debug("testNodeFailoverWorkerCopyQueuesFromRSUsingMulti");
229     conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true);
230     final Server server = new DummyServer("hostname0.example.org");
231     AtomicBoolean replicating = new AtomicBoolean(true);
232     ReplicationZookeeper rz = new ReplicationZookeeper(server, replicating);
233     // populate some znodes in the peer znode
234     files.add("log1");
235     files.add("log2");
236     for (String file : files) {
237       rz.addLogToList(file, "1");
238     }
239     // create 3 DummyServers
240     Server s1 = new DummyServer("dummyserver1.example.org");
241     Server s2 = new DummyServer("dummyserver2.example.org");
242     Server s3 = new DummyServer("dummyserver3.example.org");
243 
244     // create 3 DummyNodeFailoverWorkers
245     DummyNodeFailoverWorker w1 = new DummyNodeFailoverWorker(
246         server.getServerName().getServerName(), s1);
247     DummyNodeFailoverWorker w2 = new DummyNodeFailoverWorker(
248         server.getServerName().getServerName(), s2);
249     DummyNodeFailoverWorker w3 = new DummyNodeFailoverWorker(
250         server.getServerName().getServerName(), s3);
251 
252     latch = new CountDownLatch(3);
253     // start the threads
254     w1.start();
255     w2.start();
256     w3.start();
257     // make sure only one is successful
258     int populatedMap = 0;
259     // wait for result now... till all the workers are done.
260     latch.await();
261     populatedMap += w1.isLogZnodesMapPopulated() + w2.isLogZnodesMapPopulated()
262         + w3.isLogZnodesMapPopulated();
263     assertEquals(1, populatedMap);
264     // close out the resources.
265     server.abort("", null);
266   }
267 
268   @Test
269   public void testNodeFailoverDeadServerParsing() throws Exception {
270     LOG.debug("testNodeFailoverDeadServerParsing");
271     conf.setBoolean(HConstants.ZOOKEEPER_USEMULTI, true);
272     final Server server = new DummyServer("ec2-54-234-230-108.compute-1.amazonaws.com");
273     AtomicBoolean replicating = new AtomicBoolean(true);
274     ReplicationZookeeper rz = new ReplicationZookeeper(server, replicating);
275     // populate some znodes in the peer znode
276     files.add("log1");
277     files.add("log2");
278     for (String file : files) {
279       rz.addLogToList(file, "1");
280     }
281     // create 3 DummyServers
282     Server s1 = new DummyServer("ip-10-8-101-114.ec2.internal");
283     Server s2 = new DummyServer("ec2-107-20-52-47.compute-1.amazonaws.com");
284     Server s3 = new DummyServer("ec2-23-20-187-167.compute-1.amazonaws.com");
285 
286     // simulate three server fail sequentially
287     ReplicationZookeeper rz1 = new ReplicationZookeeper(s1, new AtomicBoolean(true));
288     SortedMap<String, SortedSet<String>> testMap =
289         rz1.copyQueuesFromRSUsingMulti(server.getServerName().getServerName());
290     ReplicationZookeeper rz2 = new ReplicationZookeeper(s2, new AtomicBoolean(true));
291     testMap = rz2.copyQueuesFromRSUsingMulti(s1.getServerName().getServerName());
292     ReplicationZookeeper rz3 = new ReplicationZookeeper(s3, new AtomicBoolean(true));
293     testMap = rz3.copyQueuesFromRSUsingMulti(s2.getServerName().getServerName());
294 
295     ReplicationSource s = new ReplicationSource();
296     s.checkIfQueueRecovered(testMap.firstKey());
297     List<String> result = s.getDeadRegionServers();
298 
299     // verify
300     assertTrue(result.contains(server.getServerName().getServerName()));
301     assertTrue(result.contains(s1.getServerName().getServerName()));
302     assertTrue(result.contains(s2.getServerName().getServerName()));
303 
304     server.abort("", null);
305   }
306 
307   static class DummyNodeFailoverWorker extends Thread {
308     private SortedMap<String, SortedSet<String>> logZnodesMap;
309     Server server;
310     private String deadRsZnode;
311     ReplicationZookeeper rz;
312 
313     public DummyNodeFailoverWorker(String znode, Server s) throws Exception {
314       this.deadRsZnode = znode;
315       this.server = s;
316       rz = new ReplicationZookeeper(server, new AtomicBoolean(true));
317     }
318 
319     @Override
320     public void run() {
321       try {
322         logZnodesMap = rz.copyQueuesFromRSUsingMulti(deadRsZnode);
323         server.abort("Done with testing", null);
324       } catch (Exception e) {
325         LOG.error("Got exception while running NodeFailoverWorker", e);
326       } finally {
327         latch.countDown();
328       }
329     }
330 
331     /**
332      * @return 1 when the map is not empty.
333      */
334     private int isLogZnodesMapPopulated() {
335       Collection<SortedSet<String>> sets = logZnodesMap.values();
336       if (sets.size() > 1) {
337         throw new RuntimeException("unexpected size of logZnodesMap: " + sets.size());
338       }
339       if (sets.size() == 1) {
340         SortedSet<String> s = sets.iterator().next();
341         for (String file : files) {
342           // at least one file was missing
343           if (!s.contains(file)) {
344             return 0;
345           }
346         }
347         return 1; // we found all the files
348       }
349       return 0;
350     }
351   }
352   
353   static class DummyServer implements Server {
354     String hostname;
355 
356     DummyServer() {
357       hostname = "hostname.example.org";
358     }
359 
360     DummyServer(String hostname) {
361       this.hostname = hostname;
362     }
363 
364     @Override
365     public Configuration getConfiguration() {
366       return conf;
367     }
368 
369     @Override
370     public ZooKeeperWatcher getZooKeeper() {
371       return zkw;
372     }
373 
374     @Override
375     public CatalogTracker getCatalogTracker() {
376       return null; // To change body of implemented methods use File | Settings | File Templates.
377     }
378 
379     @Override
380     public ServerName getServerName() {
381       return new ServerName(hostname, 1234, 1L);
382     }
383 
384     @Override
385     public void abort(String why, Throwable e) {
386       // To change body of implemented methods use File | Settings | File Templates.
387     }
388 
389     @Override
390     public boolean isAborted() {
391       return false;
392     }
393 
394     @Override
395     public void stop(String why) {
396       //To change body of implemented methods use File | Settings | File Templates.
397     }
398 
399     @Override
400     public boolean isStopped() {
401       return false;  //To change body of implemented methods use File | Settings | File Templates.
402     }
403   }
404 
405 
406   @org.junit.Rule
407   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
408     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
409 }
410