1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.wal;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.IOException;
27 import java.util.HashSet;
28 import java.util.Random;
29 import java.util.Set;
30 import java.util.concurrent.atomic.AtomicLong;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.KeyValue;
44 import org.apache.hadoop.hbase.testclassification.MediumTests;
45 import org.apache.hadoop.hbase.ServerName;
46 import org.apache.hadoop.hbase.TableName;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.hadoop.hbase.util.FSUtils;
49 import org.junit.After;
50 import org.junit.AfterClass;
51 import org.junit.Before;
52 import org.junit.BeforeClass;
53 import org.junit.Rule;
54 import org.junit.Test;
55 import org.junit.experimental.categories.Category;
56 import org.junit.rules.TestName;
57
58
59 import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
60
61 @Category(MediumTests.class)
62 public class TestDefaultWALProvider {
63 protected static final Log LOG = LogFactory.getLog(TestDefaultWALProvider.class);
64
65 protected static Configuration conf;
66 protected static FileSystem fs;
67 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
68
69 @Rule
70 public final TestName currentTest = new TestName();
71
72 @Before
73 public void setUp() throws Exception {
74 FileStatus[] entries = fs.listStatus(new Path("/"));
75 for (FileStatus dir : entries) {
76 fs.delete(dir.getPath(), true);
77 }
78 }
79
80 @After
81 public void tearDown() throws Exception {
82 }
83
84 @BeforeClass
85 public static void setUpBeforeClass() throws Exception {
86
87 TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);
88
89 TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
90 TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);
91 TEST_UTIL.getConfiguration().setInt("dfs.client.socket-timeout", 5000);
92
93
94 TEST_UTIL.getConfiguration()
95 .setInt("hbase.ipc.client.connect.max.retries", 1);
96 TEST_UTIL.getConfiguration().setInt(
97 "dfs.client.block.recovery.retries", 1);
98 TEST_UTIL.getConfiguration().setInt(
99 "hbase.ipc.client.connection.maxidletime", 500);
100 TEST_UTIL.startMiniDFSCluster(3);
101
102
103 TEST_UTIL.createRootDir();
104 conf = TEST_UTIL.getConfiguration();
105 fs = TEST_UTIL.getDFSCluster().getFileSystem();
106 }
107
108 @AfterClass
109 public static void tearDownAfterClass() throws Exception {
110 TEST_UTIL.shutdownMiniCluster();
111 }
112
113 static String getName() {
114 return "TestDefaultWALProvider";
115 }
116
117 @Test
118 public void testGetServerNameFromWALDirectoryName() throws IOException {
119 ServerName sn = ServerName.valueOf("hn", 450, 1398);
120 String hl = FSUtils.getRootDir(conf) + "/" +
121 DefaultWALProvider.getWALDirectoryName(sn.toString());
122
123
124 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, null));
125 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf,
126 FSUtils.getRootDir(conf).toUri().toString()));
127 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, ""));
128 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, " "));
129 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, hl));
130 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, hl + "qdf"));
131 assertNull(DefaultWALProvider.getServerNameFromWALDirectoryName(conf, "sfqf" + hl + "qdf"));
132
133 final String wals = "/WALs/";
134 ServerName parsed = DefaultWALProvider.getServerNameFromWALDirectoryName(conf,
135 FSUtils.getRootDir(conf).toUri().toString() + wals + sn +
136 "/localhost%2C32984%2C1343316388997.1343316390417");
137 assertEquals("standard", sn, parsed);
138
139 parsed = DefaultWALProvider.getServerNameFromWALDirectoryName(conf, hl + "/qdf");
140 assertEquals("subdir", sn, parsed);
141
142 parsed = DefaultWALProvider.getServerNameFromWALDirectoryName(conf,
143 FSUtils.getRootDir(conf).toUri().toString() + wals + sn +
144 "-splitting/localhost%3A57020.1340474893931");
145 assertEquals("split", sn, parsed);
146 }
147
148
149 protected void addEdits(WAL log, HRegionInfo hri, TableName tableName,
150 int times, AtomicLong sequenceId) throws IOException {
151 HTableDescriptor htd = new HTableDescriptor();
152 htd.addFamily(new HColumnDescriptor("row"));
153
154 final byte [] row = Bytes.toBytes("row");
155 for (int i = 0; i < times; i++) {
156 long timestamp = System.currentTimeMillis();
157 WALEdit cols = new WALEdit();
158 cols.add(new KeyValue(row, row, row, timestamp, row));
159 log.append(htd, hri, getWalKey(hri.getEncodedNameAsBytes(), tableName, timestamp), cols,
160 sequenceId, true, null);
161 }
162 log.sync();
163 }
164
165
166
167
168 WALKey getWalKey(final byte[] info, final TableName tableName, final long timestamp) {
169 return new WALKey(info, tableName, timestamp);
170 }
171
172
173
174
175
176
177 protected void flushRegion(WAL wal, byte[] regionEncodedName) {
178 wal.startCacheFlush(regionEncodedName);
179 wal.completeCacheFlush(regionEncodedName);
180 }
181
182 private static final byte[] UNSPECIFIED_REGION = new byte[]{};
183
184 @Test
185 public void testLogCleaning() throws Exception {
186 LOG.info("testLogCleaning");
187 final TableName tableName =
188 TableName.valueOf("testLogCleaning");
189 final TableName tableName2 =
190 TableName.valueOf("testLogCleaning2");
191 final Configuration localConf = new Configuration(conf);
192 localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName());
193 final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName());
194 final AtomicLong sequenceId = new AtomicLong(1);
195 try {
196 HRegionInfo hri = new HRegionInfo(tableName,
197 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
198 HRegionInfo hri2 = new HRegionInfo(tableName2,
199 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
200
201 final WAL log = wals.getWAL(UNSPECIFIED_REGION);
202
203
204
205 addEdits(log, hri, tableName, 1, sequenceId);
206 log.rollWriter();
207 assertEquals(1, DefaultWALProvider.getNumRolledLogFiles(log));
208
209
210 addEdits(log, hri, tableName, 2, sequenceId);
211 log.rollWriter();
212 assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(log));
213
214
215 addEdits(log, hri, tableName, 1, sequenceId);
216 addEdits(log, hri2, tableName2, 1, sequenceId);
217 addEdits(log, hri, tableName, 1, sequenceId);
218 addEdits(log, hri2, tableName2, 1, sequenceId);
219 log.rollWriter();
220 assertEquals(3, DefaultWALProvider.getNumRolledLogFiles(log));
221
222
223
224 addEdits(log, hri2, tableName2, 1, sequenceId);
225 log.startCacheFlush(hri.getEncodedNameAsBytes());
226 log.completeCacheFlush(hri.getEncodedNameAsBytes());
227 log.rollWriter();
228 assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(log));
229
230
231
232
233 addEdits(log, hri2, tableName2, 1, sequenceId);
234 log.startCacheFlush(hri2.getEncodedNameAsBytes());
235 log.completeCacheFlush(hri2.getEncodedNameAsBytes());
236 log.rollWriter();
237 assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(log));
238 } finally {
239 if (wals != null) {
240 wals.close();
241 }
242 }
243 }
244
245
246
247
248
249
250
251
252
253
254
255
256
257 @Test
258 public void testWALArchiving() throws IOException {
259 LOG.debug("testWALArchiving");
260 TableName table1 = TableName.valueOf("t1");
261 TableName table2 = TableName.valueOf("t2");
262 final Configuration localConf = new Configuration(conf);
263 localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName());
264 final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName());
265 try {
266 final WAL wal = wals.getWAL(UNSPECIFIED_REGION);
267 assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(wal));
268 HRegionInfo hri1 = new HRegionInfo(table1, HConstants.EMPTY_START_ROW,
269 HConstants.EMPTY_END_ROW);
270 HRegionInfo hri2 = new HRegionInfo(table2, HConstants.EMPTY_START_ROW,
271 HConstants.EMPTY_END_ROW);
272
273 hri1.setSplit(false);
274 hri2.setSplit(false);
275
276 final AtomicLong sequenceId1 = new AtomicLong(1);
277 final AtomicLong sequenceId2 = new AtomicLong(1);
278
279 addEdits(wal, hri1, table1, 1, sequenceId1);
280 wal.rollWriter();
281
282 assertEquals(1, DefaultWALProvider.getNumRolledLogFiles(wal));
283
284 addEdits(wal, hri1, table1, 1, sequenceId1);
285 wal.rollWriter();
286
287 assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(wal));
288
289 addEdits(wal, hri1, table1, 3, sequenceId1);
290 flushRegion(wal, hri1.getEncodedNameAsBytes());
291
292 wal.rollWriter();
293 assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(wal));
294
295 addEdits(wal, hri2, table2, 1, sequenceId2);
296 wal.rollWriter();
297 assertEquals(1, DefaultWALProvider.getNumRolledLogFiles(wal));
298
299 addEdits(wal, hri1, table1, 2, sequenceId1);
300 wal.rollWriter();
301 assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(wal));
302
303 addEdits(wal, hri2, table2, 2, sequenceId2);
304 flushRegion(wal, hri1.getEncodedNameAsBytes());
305
306
307
308
309
310 wal.rollWriter();
311 assertEquals(2, DefaultWALProvider.getNumRolledLogFiles(wal));
312
313 addEdits(wal, hri2, table2, 2, sequenceId2);
314 flushRegion(wal, hri2.getEncodedNameAsBytes());
315 wal.rollWriter();
316 assertEquals(0, DefaultWALProvider.getNumRolledLogFiles(wal));
317 } finally {
318 if (wals != null) {
319 wals.close();
320 }
321 }
322 }
323
324
325
326
327
328 @Test
329 public void testConcurrentWrites() throws Exception {
330
331
332 int errCode = WALPerformanceEvaluation.
333 innerMain(new Configuration(TEST_UTIL.getConfiguration()),
334 new String [] {"-threads", "3", "-verify", "-noclosefs", "-iterations", "3000"});
335 assertEquals(0, errCode);
336 }
337
338
339
340
341 @Test
342 public void setMembershipDedups() throws IOException {
343 final Configuration localConf = new Configuration(conf);
344 localConf.set(WALFactory.WAL_PROVIDER, DefaultWALProvider.class.getName());
345 final WALFactory wals = new WALFactory(localConf, null, currentTest.getMethodName());
346 try {
347 final Set<WAL> seen = new HashSet<WAL>(1);
348 final Random random = new Random();
349 assertTrue("first attempt to add WAL from default provider should work.",
350 seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()))));
351 for (int i = 0; i < 1000; i++) {
352 assertFalse("default wal provider is only supposed to return a single wal, which should " +
353 "compare as .equals itself.", seen.add(wals.getWAL(Bytes.toBytes(random.nextInt()))));
354 }
355 } finally {
356 wals.close();
357 }
358 }
359 }