View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.wal;
20  
21  import java.io.IOException;
22  import java.util.HashSet;
23  import java.util.Random;
24  import java.util.Set;
25  
26  import static org.junit.Assert.assertEquals;
27  import static org.apache.hadoop.hbase.wal.BoundedRegionGroupingProvider.NUM_REGION_GROUPS;
28  import static org.apache.hadoop.hbase.wal.BoundedRegionGroupingProvider.DEFAULT_NUM_REGION_GROUPS;
29  import static org.apache.hadoop.hbase.wal.WALFactory.WAL_PROVIDER;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.fs.FileStatus;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.testclassification.LargeTests;
39  import org.apache.hadoop.hbase.util.Bytes;
40  import org.junit.After;
41  import org.junit.AfterClass;
42  import org.junit.Before;
43  import org.junit.BeforeClass;
44  import org.junit.Rule;
45  import org.junit.Test;
46  import org.junit.experimental.categories.Category;
47  import org.junit.rules.TestName;
48  
49  @Category(LargeTests.class)
50  public class TestBoundedRegionGroupingProvider {
51    protected static final Log LOG = LogFactory.getLog(TestBoundedRegionGroupingProvider.class);
52  
53    @Rule
54    public TestName currentTest = new TestName();
55    protected static Configuration conf;
56    protected static FileSystem fs;
57    protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
58  
59    @Before
60    public void setUp() throws Exception {
61      FileStatus[] entries = fs.listStatus(new Path("/"));
62      for (FileStatus dir : entries) {
63        fs.delete(dir.getPath(), true);
64      }
65    }
66  
67    @After
68    public void tearDown() throws Exception {
69    }
70  
71    @BeforeClass
72    public static void setUpBeforeClass() throws Exception {
73      conf = TEST_UTIL.getConfiguration();
74      // Make block sizes small.
75      conf.setInt("dfs.blocksize", 1024 * 1024);
76      // quicker heartbeat interval for faster DN death notification
77      conf.setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
78      conf.setInt("dfs.heartbeat.interval", 1);
79      conf.setInt("dfs.client.socket-timeout", 5000);
80  
81      // faster failover with cluster.shutdown();fs.close() idiom
82      conf.setInt("hbase.ipc.client.connect.max.retries", 1);
83      conf.setInt("dfs.client.block.recovery.retries", 1);
84      conf.setInt("hbase.ipc.client.connection.maxidletime", 500);
85  
86      conf.setClass(WAL_PROVIDER, BoundedRegionGroupingProvider.class, WALProvider.class);
87  
88      TEST_UTIL.startMiniDFSCluster(3);
89  
90      fs = TEST_UTIL.getDFSCluster().getFileSystem();
91    }
92  
93    @AfterClass
94    public static void tearDownAfterClass() throws Exception {
95      TEST_UTIL.shutdownMiniCluster();
96    }
97  
98    /**
99     * Write to a log file with three concurrent threads and verifying all data is written.
100    */
101   @Test
102   public void testConcurrentWrites() throws Exception {
103     // Run the WPE tool with three threads writing 3000 edits each concurrently.
104     // When done, verify that all edits were written.
105     int errCode = WALPerformanceEvaluation.innerMain(new Configuration(conf),
106         new String [] {"-threads", "3", "-verify", "-noclosefs", "-iterations", "3000"});
107     assertEquals(0, errCode);
108   }
109 
110   /**
111    * Make sure we can successfully run with more regions then our bound.
112    */
113   @Test
114   public void testMoreRegionsThanBound() throws Exception {
115     final String parallelism = Integer.toString(DEFAULT_NUM_REGION_GROUPS * 2);
116     int errCode = WALPerformanceEvaluation.innerMain(new Configuration(conf),
117         new String [] {"-threads", parallelism, "-verify", "-noclosefs", "-iterations", "3000",
118             "-regions", parallelism});
119     assertEquals(0, errCode);
120   }
121 
122   @Test
123   public void testBoundsGreaterThanDefault() throws Exception {
124     final int temp = conf.getInt(NUM_REGION_GROUPS, DEFAULT_NUM_REGION_GROUPS);
125     try {
126       conf.setInt(NUM_REGION_GROUPS, temp*4);
127       final String parallelism = Integer.toString(temp*4);
128       int errCode = WALPerformanceEvaluation.innerMain(new Configuration(conf),
129           new String [] {"-threads", parallelism, "-verify", "-noclosefs", "-iterations", "3000",
130               "-regions", parallelism});
131       assertEquals(0, errCode);
132     } finally {
133       conf.setInt(NUM_REGION_GROUPS, temp);
134     }
135   }
136 
137   @Test
138   public void testMoreRegionsThanBoundWithBoundsGreaterThanDefault() throws Exception {
139     final int temp = conf.getInt(NUM_REGION_GROUPS, DEFAULT_NUM_REGION_GROUPS);
140     try {
141       conf.setInt(NUM_REGION_GROUPS, temp*4);
142       final String parallelism = Integer.toString(temp*4*2);
143       int errCode = WALPerformanceEvaluation.innerMain(new Configuration(conf),
144           new String [] {"-threads", parallelism, "-verify", "-noclosefs", "-iterations", "3000",
145               "-regions", parallelism});
146       assertEquals(0, errCode);
147     } finally {
148       conf.setInt(NUM_REGION_GROUPS, temp);
149     }
150   }
151 
152   /**
153    * Ensure that we can use Set.add to deduplicate WALs
154    */
155   @Test
156   public void setMembershipDedups() throws IOException {
157     final int temp = conf.getInt(NUM_REGION_GROUPS, DEFAULT_NUM_REGION_GROUPS);
158     WALFactory wals = null;
159     try {
160       conf.setInt(NUM_REGION_GROUPS, temp*4);
161       wals = new WALFactory(conf, null, currentTest.getMethodName());
162       final Set<WAL> seen = new HashSet<WAL>(temp*4);
163       final Random random = new Random();
164       int count = 0;
165       // we know that this should see one of the wals more than once
166       for (int i = 0; i < temp*8; i++) {
167         final WAL maybeNewWAL = wals.getWAL(Bytes.toBytes(random.nextInt()));
168         LOG.info("Iteration " + i + ", checking wal " + maybeNewWAL);
169         if (seen.add(maybeNewWAL)) {
170           count++;
171         }
172       }
173       assertEquals("received back a different number of WALs that are not equal() to each other " +
174           "than the bound we placed.", temp*4, count);
175     } finally {
176       if (wals != null) {
177         wals.close();
178       }
179       conf.setInt(NUM_REGION_GROUPS, temp);
180     }
181   }
182 }