1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.replication;
21  
22  import static org.junit.Assert.assertArrayEquals;
23  import static org.junit.Assert.assertEquals;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.*;
33  import org.apache.hadoop.hbase.client.Delete;
34  import org.apache.hadoop.hbase.client.Get;
35  import org.apache.hadoop.hbase.client.HBaseAdmin;
36  import org.apache.hadoop.hbase.client.HTable;
37  import org.apache.hadoop.hbase.client.Put;
38  import org.apache.hadoop.hbase.client.Result;
39  import org.apache.hadoop.hbase.client.replication.ReplicationAdmin;
40  import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
41  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
42  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
43  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
46  import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
47  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
48  import org.junit.After;
49  import org.junit.Before;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  
53  @Category(LargeTests.class)
54  public class TestMasterReplication {
55  
56    private static final Log LOG = LogFactory.getLog(TestReplicationBase.class);
57  
58    private Configuration conf1;
59    private Configuration conf2;
60    private Configuration conf3;
61  
62    private HBaseTestingUtility utility1;
63    private HBaseTestingUtility utility2;
64    private HBaseTestingUtility utility3;
65    
66    private MiniZooKeeperCluster miniZK; 
67  
68    private static final long SLEEP_TIME = 500;
69    private static final int NB_RETRIES = 10;
70  
71    private static final byte[] tableName = Bytes.toBytes("test");
72    private static final byte[] famName = Bytes.toBytes("f");
73    private static final byte[] row = Bytes.toBytes("row");
74    private static final byte[] row1 = Bytes.toBytes("row1");
75    private static final byte[] row2 = Bytes.toBytes("row2");
76    private static final byte[] noRepfamName = Bytes.toBytes("norep");
77  
78    private static final byte[] count = Bytes.toBytes("count");
79    private static final byte[] put = Bytes.toBytes("put");
80    private static final byte[] delete = Bytes.toBytes("delete");
81  
82    private HTableDescriptor table;
83  
84    @Before
85    public void setUp() throws Exception {
86      conf1 = HBaseConfiguration.create();
87      conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
88      // smaller block size and capacity to trigger more operations
89      // and test them
90      conf1.setInt("hbase.regionserver.hlog.blocksize", 1024*20);
91      conf1.setInt("replication.source.size.capacity", 1024);
92      conf1.setLong("replication.source.sleepforretries", 100);
93      conf1.setInt("hbase.regionserver.maxlogs", 10);
94      conf1.setLong("hbase.master.logcleaner.ttl", 10);
95      conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);
96      conf1.setBoolean("dfs.support.append", true);
97      conf1.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
98      conf1.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
99          CoprocessorCounter.class.getName());
100 
101     utility1 = new HBaseTestingUtility(conf1);
102     utility1.startMiniZKCluster();
103     miniZK = utility1.getZkCluster();
104     // By setting the mini ZK cluster through this method, even though this is
105     // already utility1's mini ZK cluster, we are telling utility1 not to shut
106     // the mini ZK cluster when we shut down the HBase cluster.
107     utility1.setZkCluster(miniZK);
108     new ZooKeeperWatcher(conf1, "cluster1", null, true);
109 
110     conf2 = new Configuration(conf1);
111     conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
112 
113     utility2 = new HBaseTestingUtility(conf2);
114     utility2.setZkCluster(miniZK);
115     new ZooKeeperWatcher(conf2, "cluster2", null, true);
116 
117     conf3 = new Configuration(conf1);
118     conf3.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/3");
119 
120     utility3 = new HBaseTestingUtility(conf3);
121     utility3.setZkCluster(miniZK);
122     new ZooKeeperWatcher(conf3, "cluster3", null, true);
123 
124     table = new HTableDescriptor(tableName);
125     HColumnDescriptor fam = new HColumnDescriptor(famName);
126     fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
127     table.addFamily(fam);
128     fam = new HColumnDescriptor(noRepfamName);
129     table.addFamily(fam);
130   }
131 
132   @After
133   public void tearDown() throws IOException {
134     miniZK.shutdown();
135   }
136 
137   @Test(timeout=300000)
138   public void testCyclicReplication() throws Exception {
139     LOG.info("testCyclicReplication");
140     utility1.startMiniCluster();
141     utility2.startMiniCluster();
142     utility3.startMiniCluster();
143     ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
144     ReplicationAdmin admin2 = new ReplicationAdmin(conf2);
145     ReplicationAdmin admin3 = new ReplicationAdmin(conf3);
146 
147     new HBaseAdmin(conf1).createTable(table);
148     new HBaseAdmin(conf2).createTable(table);
149     new HBaseAdmin(conf3).createTable(table);
150     HTable htable1 = new HTable(conf1, tableName);
151     htable1.setWriteBufferSize(1024);
152     HTable htable2 = new HTable(conf2, tableName);
153     htable2.setWriteBufferSize(1024);
154     HTable htable3 = new HTable(conf3, tableName);
155     htable3.setWriteBufferSize(1024);
156     
157     admin1.addPeer("1", utility2.getClusterKey());
158     admin2.addPeer("1", utility3.getClusterKey());
159     admin3.addPeer("1", utility1.getClusterKey());
160 
161     // put "row" and wait 'til it got around
162     putAndWait(row, famName, htable1, htable3);
163     // it should have passed through table2
164     check(row,famName,htable2);
165 
166     putAndWait(row1, famName, htable2, htable1);
167     check(row,famName,htable3);
168     putAndWait(row2, famName, htable3, htable2);
169     check(row,famName,htable1);
170     
171     deleteAndWait(row,htable1,htable3);
172     deleteAndWait(row1,htable2,htable1);
173     deleteAndWait(row2,htable3,htable2);
174 
175     assertEquals("Puts were replicated back ", 3, getCount(htable1, put));
176     assertEquals("Puts were replicated back ", 3, getCount(htable2, put));
177     assertEquals("Puts were replicated back ", 3, getCount(htable3, put));
178     assertEquals("Deletes were replicated back ", 3, getCount(htable1, delete));
179     assertEquals("Deletes were replicated back ", 3, getCount(htable2, delete));
180     assertEquals("Deletes were replicated back ", 3, getCount(htable3, delete));
181     utility3.shutdownMiniCluster();
182     utility2.shutdownMiniCluster();
183     utility1.shutdownMiniCluster();
184   }
185 
186   /**
187    * Add a row to a table in each cluster, check it's replicated,
188    * delete it, check's gone
189    * Also check the puts and deletes are not replicated back to
190    * the originating cluster.
191    */
192   @Test(timeout=300000)
193   public void testSimplePutDelete() throws Exception {
194     LOG.info("testSimplePutDelete");
195     utility1.startMiniCluster();
196     utility2.startMiniCluster();
197 
198     ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
199     ReplicationAdmin admin2 = new ReplicationAdmin(conf2);
200 
201     new HBaseAdmin(conf1).createTable(table);
202     new HBaseAdmin(conf2).createTable(table);
203     HTable htable1 = new HTable(conf1, tableName);
204     htable1.setWriteBufferSize(1024);
205     HTable htable2 = new HTable(conf2, tableName);
206     htable2.setWriteBufferSize(1024);
207 
208     // set M-M
209     admin1.addPeer("1", utility2.getClusterKey());
210     admin2.addPeer("1", utility1.getClusterKey());
211 
212     // add rows to both clusters,
213     // make sure they are both replication
214     putAndWait(row, famName, htable1, htable2);
215     putAndWait(row1, famName, htable2, htable1);
216 
217     // make sure "row" did not get replicated back.
218     assertEquals("Puts were replicated back ", 2, getCount(htable1, put));
219 
220     // delete "row" and wait
221     deleteAndWait(row, htable1, htable2);
222 
223     // make the 2nd cluster replicated back
224     assertEquals("Puts were replicated back ", 2, getCount(htable2, put));
225 
226     deleteAndWait(row1, htable2, htable1);
227 
228     assertEquals("Deletes were replicated back ", 2, getCount(htable1, delete));
229     utility2.shutdownMiniCluster();
230     utility1.shutdownMiniCluster();
231   }
232 
233   private int getCount(HTable t, byte[] type)  throws IOException {
234     Get test = new Get(row);
235     test.setAttribute("count", new byte[]{});
236     Result res = t.get(test);
237     return Bytes.toInt(res.getValue(count, type));
238   }
239 
240   private void deleteAndWait(byte[] row, HTable source, HTable target)
241   throws Exception {
242     Delete del = new Delete(row);
243     source.delete(del);
244 
245     Get get = new Get(row);
246     for (int i = 0; i < NB_RETRIES; i++) {
247       if (i==NB_RETRIES-1) {
248         fail("Waited too much time for del replication");
249       }
250       Result res = target.get(get);
251       if (res.size() >= 1) {
252         LOG.info("Row not deleted");
253         Thread.sleep(SLEEP_TIME);
254       } else {
255         break;
256       }
257     }
258   }
259 
260   private void check(byte[] row, byte[] fam, HTable t) throws IOException {
261     Get get = new Get(row);
262     Result res = t.get(get);
263     if (res.size() == 0) {
264       fail("Row is missing");
265     }
266   }
267 
268   private void putAndWait(byte[] row, byte[] fam, HTable source, HTable target)
269   throws Exception {
270     Put put = new Put(row);
271     put.add(fam, row, row);
272     source.put(put);
273 
274     Get get = new Get(row);
275     for (int i = 0; i < NB_RETRIES; i++) {
276       if (i==NB_RETRIES-1) {
277         fail("Waited too much time for put replication");
278       }
279       Result res = target.get(get);
280       if (res.size() == 0) {
281         LOG.info("Row not available");
282         Thread.sleep(SLEEP_TIME);
283       } else {
284         assertArrayEquals(res.value(), row);
285         break;
286       }
287     }
288   }
289 
290   /**
291    * Use a coprocessor to count puts and deletes.
292    * as KVs would be replicated back with the same timestamp
293    * there is otherwise no way to count them.
294    */
295   public static class CoprocessorCounter extends BaseRegionObserver {
296     private int nCount = 0;
297     private int nDelete = 0;
298 
299     @Override
300     public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e,
301         final Put put, final WALEdit edit, 
302         final boolean writeToWAL)
303         throws IOException {
304       nCount++;
305     }
306     @Override
307     public void postDelete(final ObserverContext<RegionCoprocessorEnvironment> c,
308         final Delete delete, final WALEdit edit, 
309         final boolean writeToWAL)
310         throws IOException {
311       nDelete++;
312     }
313     @Override
314     public void preGet(final ObserverContext<RegionCoprocessorEnvironment> c,
315         final Get get, final List<KeyValue> result) throws IOException {
316       if (get.getAttribute("count") != null) {
317         result.clear();
318         // order is important!
319         result.add(new KeyValue(count, count, delete, Bytes.toBytes(nDelete)));
320         result.add(new KeyValue(count, count, put, Bytes.toBytes(nCount)));
321         c.bypass();
322       }
323     }
324   }
325 
326   @org.junit.Rule
327   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
328     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
329 }
330