1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.wal;
20
21 import java.io.IOException;
22 import java.util.HashSet;
23 import java.util.Random;
24 import java.util.Set;
25
26 import static org.junit.Assert.assertEquals;
27 import static org.apache.hadoop.hbase.wal.BoundedRegionGroupingProvider.NUM_REGION_GROUPS;
28 import static org.apache.hadoop.hbase.wal.BoundedRegionGroupingProvider.DEFAULT_NUM_REGION_GROUPS;
29 import static org.apache.hadoop.hbase.wal.WALFactory.WAL_PROVIDER;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileStatus;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.testclassification.LargeTests;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.junit.After;
41 import org.junit.AfterClass;
42 import org.junit.Before;
43 import org.junit.BeforeClass;
44 import org.junit.Rule;
45 import org.junit.Test;
46 import org.junit.experimental.categories.Category;
47 import org.junit.rules.TestName;
48
49 @Category(LargeTests.class)
50 public class TestBoundedRegionGroupingProvider {
51 protected static final Log LOG = LogFactory.getLog(TestBoundedRegionGroupingProvider.class);
52
53 @Rule
54 public TestName currentTest = new TestName();
55 protected static Configuration conf;
56 protected static FileSystem fs;
57 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
58
59 @Before
60 public void setUp() throws Exception {
61 FileStatus[] entries = fs.listStatus(new Path("/"));
62 for (FileStatus dir : entries) {
63 fs.delete(dir.getPath(), true);
64 }
65 }
66
67 @After
68 public void tearDown() throws Exception {
69 }
70
71 @BeforeClass
72 public static void setUpBeforeClass() throws Exception {
73 conf = TEST_UTIL.getConfiguration();
74
75 conf.setInt("dfs.blocksize", 1024 * 1024);
76
77 conf.setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
78 conf.setInt("dfs.heartbeat.interval", 1);
79 conf.setInt("dfs.client.socket-timeout", 5000);
80
81
82 conf.setInt("hbase.ipc.client.connect.max.retries", 1);
83 conf.setInt("dfs.client.block.recovery.retries", 1);
84 conf.setInt("hbase.ipc.client.connection.maxidletime", 500);
85
86 conf.setClass(WAL_PROVIDER, BoundedRegionGroupingProvider.class, WALProvider.class);
87
88 TEST_UTIL.startMiniDFSCluster(3);
89
90 fs = TEST_UTIL.getDFSCluster().getFileSystem();
91 }
92
93 @AfterClass
94 public static void tearDownAfterClass() throws Exception {
95 TEST_UTIL.shutdownMiniCluster();
96 }
97
98
99
100
101 @Test
102 public void testConcurrentWrites() throws Exception {
103
104
105 int errCode = WALPerformanceEvaluation.innerMain(new Configuration(conf),
106 new String [] {"-threads", "3", "-verify", "-noclosefs", "-iterations", "3000"});
107 assertEquals(0, errCode);
108 }
109
110
111
112
113 @Test
114 public void testMoreRegionsThanBound() throws Exception {
115 final String parallelism = Integer.toString(DEFAULT_NUM_REGION_GROUPS * 2);
116 int errCode = WALPerformanceEvaluation.innerMain(new Configuration(conf),
117 new String [] {"-threads", parallelism, "-verify", "-noclosefs", "-iterations", "3000",
118 "-regions", parallelism});
119 assertEquals(0, errCode);
120 }
121
122 @Test
123 public void testBoundsGreaterThanDefault() throws Exception {
124 final int temp = conf.getInt(NUM_REGION_GROUPS, DEFAULT_NUM_REGION_GROUPS);
125 try {
126 conf.setInt(NUM_REGION_GROUPS, temp*4);
127 final String parallelism = Integer.toString(temp*4);
128 int errCode = WALPerformanceEvaluation.innerMain(new Configuration(conf),
129 new String [] {"-threads", parallelism, "-verify", "-noclosefs", "-iterations", "3000",
130 "-regions", parallelism});
131 assertEquals(0, errCode);
132 } finally {
133 conf.setInt(NUM_REGION_GROUPS, temp);
134 }
135 }
136
137 @Test
138 public void testMoreRegionsThanBoundWithBoundsGreaterThanDefault() throws Exception {
139 final int temp = conf.getInt(NUM_REGION_GROUPS, DEFAULT_NUM_REGION_GROUPS);
140 try {
141 conf.setInt(NUM_REGION_GROUPS, temp*4);
142 final String parallelism = Integer.toString(temp*4*2);
143 int errCode = WALPerformanceEvaluation.innerMain(new Configuration(conf),
144 new String [] {"-threads", parallelism, "-verify", "-noclosefs", "-iterations", "3000",
145 "-regions", parallelism});
146 assertEquals(0, errCode);
147 } finally {
148 conf.setInt(NUM_REGION_GROUPS, temp);
149 }
150 }
151
152
153
154
155 @Test
156 public void setMembershipDedups() throws IOException {
157 final int temp = conf.getInt(NUM_REGION_GROUPS, DEFAULT_NUM_REGION_GROUPS);
158 WALFactory wals = null;
159 try {
160 conf.setInt(NUM_REGION_GROUPS, temp*4);
161 wals = new WALFactory(conf, null, currentTest.getMethodName());
162 final Set<WAL> seen = new HashSet<WAL>(temp*4);
163 final Random random = new Random();
164 int count = 0;
165
166 for (int i = 0; i < temp*8; i++) {
167 final WAL maybeNewWAL = wals.getWAL(Bytes.toBytes(random.nextInt()));
168 LOG.info("Iteration " + i + ", checking wal " + maybeNewWAL);
169 if (seen.add(maybeNewWAL)) {
170 count++;
171 }
172 }
173 assertEquals("received back a different number of WALs that are not equal() to each other " +
174 "than the bound we placed.", temp*4, count);
175 } finally {
176 if (wals != null) {
177 wals.close();
178 }
179 conf.setInt(NUM_REGION_GROUPS, temp);
180 }
181 }
182 }