1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertTrue;
25 import static org.junit.Assert.fail;
26
27 import java.io.FileNotFoundException;
28 import java.io.IOException;
29 import java.lang.reflect.Method;
30 import java.security.PrivilegedExceptionAction;
31 import java.util.ArrayList;
32 import java.util.Collections;
33 import java.util.HashMap;
34 import java.util.List;
35 import java.util.Map;
36 import java.util.NavigableSet;
37 import java.util.concurrent.CountDownLatch;
38 import java.util.concurrent.atomic.AtomicBoolean;
39 import java.util.concurrent.atomic.AtomicInteger;
40 import java.util.concurrent.atomic.AtomicLong;
41
42 import org.apache.commons.logging.Log;
43 import org.apache.commons.logging.LogFactory;
44 import org.apache.commons.logging.impl.Log4JLogger;
45 import org.apache.hadoop.hbase.TableName;
46 import org.apache.log4j.Level;
47 import org.apache.hadoop.hdfs.server.datanode.DataNode;
48 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
49 import org.apache.hadoop.hdfs.server.namenode.LeaseManager;
50 import org.apache.hadoop.conf.Configuration;
51 import org.apache.hadoop.fs.FSDataInputStream;
52 import org.apache.hadoop.fs.FSDataOutputStream;
53 import org.apache.hadoop.fs.FileStatus;
54 import org.apache.hadoop.fs.FileSystem;
55 import org.apache.hadoop.fs.FileUtil;
56 import org.apache.hadoop.fs.Path;
57 import org.apache.hadoop.hbase.HBaseConfiguration;
58 import org.apache.hadoop.hbase.HBaseTestingUtility;
59 import org.apache.hadoop.hbase.HColumnDescriptor;
60 import org.apache.hadoop.hbase.HConstants;
61 import org.apache.hadoop.hbase.HRegionInfo;
62 import org.apache.hadoop.hbase.HTableDescriptor;
63 import org.apache.hadoop.hbase.KeyValue;
64 import org.apache.hadoop.hbase.LargeTests;
65 import org.apache.hadoop.hbase.regionserver.HRegion;
66 import org.apache.hadoop.hbase.regionserver.wal.HLog.Entry;
67 import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader;
68 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter.CorruptedLogFileException;
69 import org.apache.hadoop.hbase.security.User;
70 import org.apache.hadoop.hbase.util.Bytes;
71 import org.apache.hadoop.hbase.util.CancelableProgressable;
72 import org.apache.hadoop.hbase.util.FSUtils;
73 import org.apache.hadoop.hbase.util.Threads;
74 import org.apache.hadoop.hdfs.DFSTestUtil;
75 import org.apache.hadoop.hdfs.DistributedFileSystem;
76 import org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException;
77 import org.apache.hadoop.ipc.RemoteException;
78 import org.junit.After;
79 import org.junit.AfterClass;
80 import org.junit.Assert;
81 import org.junit.Before;
82 import org.junit.BeforeClass;
83 import org.junit.Ignore;
84 import org.junit.Test;
85 import org.junit.experimental.categories.Category;
86 import org.mockito.Mockito;
87 import org.mockito.invocation.InvocationOnMock;
88 import org.mockito.stubbing.Answer;
89
90 import com.google.common.base.Joiner;
91 import com.google.common.collect.ImmutableList;
92
93
94
95
96 @Category(LargeTests.class)
97 public class TestHLogSplit {
98 {
99 ((Log4JLogger)DataNode.LOG).getLogger().setLevel(Level.ALL);
100 ((Log4JLogger)LeaseManager.LOG).getLogger().setLevel(Level.ALL);
101 ((Log4JLogger)FSNamesystem.LOG).getLogger().setLevel(Level.ALL);
102 }
103 private final static Log LOG = LogFactory.getLog(TestHLogSplit.class);
104
105 private Configuration conf;
106 private FileSystem fs;
107
108 protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
109
110 private static final Path HBASEDIR = new Path("/hbase");
111 private static final Path HLOGDIR = new Path(HBASEDIR, "hlog");
112 private static final Path OLDLOGDIR = new Path(HBASEDIR, "hlog.old");
113 private static final Path CORRUPTDIR = new Path(HBASEDIR, HConstants.CORRUPT_DIR_NAME);
114
115 private static final int NUM_WRITERS = 10;
116 private static final int ENTRIES = 10;
117
118 private static final TableName TABLE_NAME =
119 TableName.valueOf("t1");
120 private static final byte[] FAMILY = "f1".getBytes();
121 private static final byte[] QUALIFIER = "q1".getBytes();
122 private static final byte[] VALUE = "v1".getBytes();
123 private static final String HLOG_FILE_PREFIX = "hlog.dat.";
124 private static List<String> REGIONS = new ArrayList<String>();
125 private static final String HBASE_SKIP_ERRORS = "hbase.hlog.split.skip.errors";
126 private static final Path TABLEDIR = FSUtils.getTableDir(HBASEDIR, TABLE_NAME);
127 private static String ROBBER;
128 private static String ZOMBIE;
129 private static String [] GROUP = new String [] {"supergroup"};
130
131 static enum Corruptions {
132 INSERT_GARBAGE_ON_FIRST_LINE,
133 INSERT_GARBAGE_IN_THE_MIDDLE,
134 APPEND_GARBAGE,
135 TRUNCATE,
136 TRUNCATE_TRAILER
137 }
138
139 @BeforeClass
140 public static void setUpBeforeClass() throws Exception {
141 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), HBASEDIR);
142 TEST_UTIL.getConfiguration().setClass("hbase.regionserver.hlog.writer.impl",
143 InstrumentedSequenceFileLogWriter.class, HLog.Writer.class);
144 TEST_UTIL.getConfiguration().setBoolean("dfs.support.broken.append", true);
145 TEST_UTIL.getConfiguration().setBoolean("dfs.support.append", true);
146
147 System.setProperty("hbase.tests.use.shortcircuit.reads", "false");
148
149 Map<String, String []> u2g_map = new HashMap<String, String []>(2);
150 ROBBER = User.getCurrent().getName() + "-robber";
151 ZOMBIE = User.getCurrent().getName() + "-zombie";
152 u2g_map.put(ROBBER, GROUP);
153 u2g_map.put(ZOMBIE, GROUP);
154 DFSTestUtil.updateConfWithFakeGroupMapping(TEST_UTIL.getConfiguration(), u2g_map);
155 TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);
156 TEST_UTIL.startMiniDFSCluster(2);
157 }
158
159 @AfterClass
160 public static void tearDownAfterClass() throws Exception {
161 TEST_UTIL.shutdownMiniDFSCluster();
162 }
163
164 @Before
165 public void setUp() throws Exception {
166 flushToConsole("Cleaning up cluster for new test\n"
167 + "--------------------------");
168 conf = TEST_UTIL.getConfiguration();
169 fs = TEST_UTIL.getDFSCluster().getFileSystem();
170 FileStatus[] entries = fs.listStatus(new Path("/"));
171 flushToConsole("Num entries in /:" + entries.length);
172 for (FileStatus dir : entries){
173 assertTrue("Deleting " + dir.getPath(), fs.delete(dir.getPath(), true));
174 }
175
176 fs.mkdirs(HLOGDIR);
177 REGIONS.clear();
178 Collections.addAll(REGIONS, "bbb", "ccc");
179 InstrumentedSequenceFileLogWriter.activateFailure = false;
180 }
181
182 @After
183 public void tearDown() throws Exception {
184 }
185
186
187
188
189
190
191
192 @Test (timeout=300000)
193 public void testLogCannotBeWrittenOnceParsed() throws IOException, InterruptedException {
194 final AtomicLong counter = new AtomicLong(0);
195 AtomicBoolean stop = new AtomicBoolean(false);
196
197 final String region = REGIONS.get(0);
198 Thread zombie = new ZombieLastLogWriterRegionServer(this.conf, counter, stop, region);
199 try {
200 long startCount = counter.get();
201 zombie.start();
202
203 while (startCount == counter.get()) Threads.sleep(1);
204
205 Threads.sleep(1000);
206 final Configuration conf2 = HBaseConfiguration.create(this.conf);
207 final User robber = User.createUserForTesting(conf2, ROBBER, GROUP);
208 int count = robber.runAs(new PrivilegedExceptionAction<Integer>() {
209 @Override
210 public Integer run() throws Exception {
211 FileSystem fs = FileSystem.get(conf2);
212 int expectedFiles = fs.listStatus(HLOGDIR).length;
213 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf2);
214 Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);
215 assertEquals(expectedFiles, logfiles.length);
216 int count = 0;
217 for (Path logfile: logfiles) {
218 count += countHLog(logfile, fs, conf2);
219 }
220 return count;
221 }
222 });
223 LOG.info("zombie=" + counter.get() + ", robber=" + count);
224 assertTrue("The log file could have at most 1 extra log entry, but can't have less. Zombie could write " +
225 counter.get() + " and logfile had only " + count,
226 counter.get() == count || counter.get() + 1 == count);
227 } finally {
228 stop.set(true);
229 zombie.interrupt();
230 Threads.threadDumpingIsAlive(zombie);
231 }
232 }
233
234
235
236
237
238
239 static class ZombieLastLogWriterRegionServer extends Thread {
240 final AtomicLong editsCount;
241 final AtomicBoolean stop;
242
243
244
245
246 final String region;
247 final Configuration conf;
248 final User user;
249
250 public ZombieLastLogWriterRegionServer(final Configuration conf, AtomicLong counter, AtomicBoolean stop,
251 final String region)
252 throws IOException, InterruptedException {
253 super("ZombieLastLogWriterRegionServer");
254 setDaemon(true);
255 this.stop = stop;
256 this.editsCount = counter;
257 this.region = region;
258 this.conf = HBaseConfiguration.create(conf);
259 this.user = User.createUserForTesting(this.conf, ZOMBIE, GROUP);
260 }
261
262 @Override
263 public void run() {
264 try {
265 doWriting();
266 } catch (IOException e) {
267 LOG.warn(getName() + " Writer exiting " + e);
268 } catch (InterruptedException e) {
269 LOG.warn(getName() + " Writer exiting " + e);
270 }
271 }
272
273 private void doWriting() throws IOException, InterruptedException {
274 this.user.runAs(new PrivilegedExceptionAction<Object>() {
275 @Override
276 public Object run() throws Exception {
277
278 int walToKeepOpen = 2;
279
280 final int numOfWriters = walToKeepOpen + 1;
281
282
283 HLog.Writer[] writers = null;
284 try {
285 DistributedFileSystem dfs = (DistributedFileSystem)FileSystem.get(conf);
286 writers = generateHLogs(dfs, numOfWriters, ENTRIES, walToKeepOpen);
287 } catch (IOException e1) {
288 throw new RuntimeException("Failed", e1);
289 }
290
291 editsCount.addAndGet(numOfWriters * NUM_WRITERS);
292
293 HLog.Writer writer = writers[walToKeepOpen];
294 loop(writer);
295 return null;
296 }
297 });
298 }
299
300 private void loop(final HLog.Writer writer) {
301 byte [] regionBytes = Bytes.toBytes(this.region);
302 while (true) {
303 try {
304 long seq = appendEntry(writer, TABLE_NAME, regionBytes, ("r" + editsCount.get()).getBytes(),
305 regionBytes, QUALIFIER, VALUE, 0);
306 long count = editsCount.incrementAndGet();
307 flushToConsole(getName() + " sync count=" + count + ", seq=" + seq);
308 try {
309 Thread.sleep(1);
310 } catch (InterruptedException e) {
311
312 }
313 } catch (IOException ex) {
314 flushToConsole(getName() + " ex " + ex.toString());
315 if (ex instanceof RemoteException) {
316 flushToConsole("Juliet: got RemoteException " + ex.getMessage() +
317 " while writing " + (editsCount.get() + 1));
318 } else {
319 flushToConsole(getName() + " failed to write....at " + editsCount.get());
320 assertTrue("Failed to write " + editsCount.get(), false);
321 }
322 break;
323 } catch (Throwable t) {
324 flushToConsole(getName() + " HOW? " + t);
325 t.printStackTrace();
326 break;
327 }
328 }
329 flushToConsole(getName() + " Writer exiting");
330 }
331 }
332
333
334
335
336
337 @Test (timeout=300000)
338 public void testRecoveredEditsPathForMeta() throws IOException {
339 FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
340 byte [] encoded = HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes();
341 Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME);
342 Path regiondir = new Path(tdir,
343 HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
344 fs.mkdirs(regiondir);
345 long now = System.currentTimeMillis();
346 HLog.Entry entry =
347 new HLog.Entry(new HLogKey(encoded,
348 TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),
349 new WALEdit());
350 Path p = HLogSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR, true);
351 String parentOfParent = p.getParent().getParent().getName();
352 assertEquals(parentOfParent, HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
353 }
354
355
356
357
358
359 @Test (timeout=300000)
360 public void testOldRecoveredEditsFileSidelined() throws IOException {
361 FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
362 byte [] encoded = HRegionInfo.FIRST_META_REGIONINFO.getEncodedNameAsBytes();
363 Path tdir = FSUtils.getTableDir(HBASEDIR, TableName.META_TABLE_NAME);
364 Path regiondir = new Path(tdir,
365 HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
366 fs.mkdirs(regiondir);
367 long now = System.currentTimeMillis();
368 HLog.Entry entry =
369 new HLog.Entry(new HLogKey(encoded,
370 TableName.META_TABLE_NAME, 1, now, HConstants.DEFAULT_CLUSTER_ID),
371 new WALEdit());
372 Path parent = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
373 assertEquals(parent.getName(), HConstants.RECOVERED_EDITS_DIR);
374 fs.createNewFile(parent);
375
376 Path p = HLogSplitter.getRegionSplitEditsPath(fs, entry, HBASEDIR, true);
377 String parentOfParent = p.getParent().getParent().getName();
378 assertEquals(parentOfParent, HRegionInfo.FIRST_META_REGIONINFO.getEncodedName());
379 HLogFactory.createRecoveredEditsWriter(fs, p, conf).close();
380 }
381
382 @Test (timeout=300000)
383 public void testSplitPreservesEdits() throws IOException{
384 final String REGION = "region__1";
385 REGIONS.removeAll(REGIONS);
386 REGIONS.add(REGION);
387
388 generateHLogs(1, 10, -1);
389 fs.initialize(fs.getUri(), conf);
390 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
391 Path originalLog = (fs.listStatus(OLDLOGDIR))[0].getPath();
392 Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, REGION);
393 assertEquals(1, splitLog.length);
394
395 assertEquals("edits differ after split", true, logsAreEqual(originalLog, splitLog[0]));
396 }
397
398
399 @Test (timeout=300000)
400 public void testEmptyLogFiles() throws IOException {
401
402 injectEmptyFile(".empty", true);
403 generateHLogs(Integer.MAX_VALUE);
404 injectEmptyFile("empty", true);
405
406
407
408 fs.initialize(fs.getUri(), conf);
409
410 int expectedFiles = fs.listStatus(HLOGDIR).length - 2;
411 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
412 for (String region : REGIONS) {
413 Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);
414 assertEquals(expectedFiles, logfiles.length);
415 int count = 0;
416 for (Path logfile: logfiles) {
417 count += countHLog(logfile, fs, conf);
418 }
419 assertEquals(NUM_WRITERS * ENTRIES, count);
420 }
421 }
422
423
424 @Test (timeout=300000)
425 public void testEmptyOpenLogFiles() throws IOException {
426 injectEmptyFile(".empty", false);
427 generateHLogs(Integer.MAX_VALUE);
428 injectEmptyFile("empty", false);
429
430
431
432 fs.initialize(fs.getUri(), conf);
433
434 int expectedFiles = fs.listStatus(HLOGDIR).length - 2 ;
435 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
436 for (String region : REGIONS) {
437 Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);
438 assertEquals(expectedFiles, logfiles.length);
439 int count = 0;
440 for (Path logfile: logfiles) {
441 count += countHLog(logfile, fs, conf);
442 }
443 assertEquals(NUM_WRITERS * ENTRIES, count);
444 }
445 }
446
447 @Test (timeout=300000)
448 public void testOpenZeroLengthReportedFileButWithDataGetsSplit() throws IOException {
449
450 generateHLogs(5);
451
452 fs.initialize(fs.getUri(), conf);
453
454 int expectedFiles = fs.listStatus(HLOGDIR).length;
455 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
456 for (String region : REGIONS) {
457 Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);
458 assertEquals(expectedFiles, logfiles.length);
459 int count = 0;
460 for (Path logfile: logfiles) {
461 count += countHLog(logfile, fs, conf);
462 }
463 assertEquals(NUM_WRITERS * ENTRIES, count);
464 }
465 }
466
467
468 @Test (timeout=300000)
469 public void testTralingGarbageCorruptionFileSkipErrorsPasses() throws IOException {
470 conf.setBoolean(HBASE_SKIP_ERRORS, true);
471 generateHLogs(Integer.MAX_VALUE);
472 corruptHLog(new Path(HLOGDIR, HLOG_FILE_PREFIX + "5"),
473 Corruptions.APPEND_GARBAGE, true, fs);
474 fs.initialize(fs.getUri(), conf);
475
476 int expectedFiles = fs.listStatus(HLOGDIR).length;
477 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
478 for (String region : REGIONS) {
479 Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);
480 assertEquals(expectedFiles, logfiles.length);
481 int count = 0;
482 for (Path logfile: logfiles) {
483 count += countHLog(logfile, fs, conf);
484 }
485 assertEquals(NUM_WRITERS * ENTRIES, count);
486 }
487 }
488
489 @Test (timeout=300000)
490 public void testFirstLineCorruptionLogFileSkipErrorsPasses() throws IOException {
491 conf.setBoolean(HBASE_SKIP_ERRORS, true);
492 generateHLogs(Integer.MAX_VALUE);
493 corruptHLog(new Path(HLOGDIR, HLOG_FILE_PREFIX + "5"),
494 Corruptions.INSERT_GARBAGE_ON_FIRST_LINE, true, fs);
495 fs.initialize(fs.getUri(), conf);
496
497 int expectedFiles = fs.listStatus(HLOGDIR).length - 1;
498 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
499 for (String region : REGIONS) {
500 Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);
501 assertEquals(expectedFiles, logfiles.length);
502 int count = 0;
503 for (Path logfile: logfiles) {
504 count += countHLog(logfile, fs, conf);
505 }
506 assertEquals((NUM_WRITERS - 1) * ENTRIES, count);
507 }
508 }
509
510 @Test (timeout=300000)
511 public void testMiddleGarbageCorruptionSkipErrorsReadsHalfOfFile() throws IOException {
512 conf.setBoolean(HBASE_SKIP_ERRORS, true);
513 generateHLogs(Integer.MAX_VALUE);
514 corruptHLog(new Path(HLOGDIR, HLOG_FILE_PREFIX + "5"),
515 Corruptions.INSERT_GARBAGE_IN_THE_MIDDLE, false, fs);
516 fs.initialize(fs.getUri(), conf);
517
518 int expectedFiles = fs.listStatus(HLOGDIR).length;
519 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
520 for (String region : REGIONS) {
521 Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);
522 assertEquals(expectedFiles, logfiles.length);
523 int count = 0;
524 for (Path logfile: logfiles) {
525 count += countHLog(logfile, fs, conf);
526 }
527
528
529
530 int goodEntries = (NUM_WRITERS - 1) * ENTRIES;
531 int firstHalfEntries = (int) Math.ceil(ENTRIES / 2) - 1;
532 assertTrue("The file up to the corrupted area hasn't been parsed",
533 goodEntries + firstHalfEntries <= count);
534 }
535 }
536
537 @Test (timeout=300000)
538 public void testCorruptedFileGetsArchivedIfSkipErrors() throws IOException {
539 conf.setBoolean(HBASE_SKIP_ERRORS, true);
540 Class<?> backupClass = conf.getClass("hbase.regionserver.hlog.reader.impl",
541 Reader.class);
542 InstrumentedSequenceFileLogWriter.activateFailure = false;
543 HLogFactory.resetLogReaderClass();
544
545 try {
546 Path c1 = new Path(HLOGDIR, HLOG_FILE_PREFIX + "0");
547 conf.setClass("hbase.regionserver.hlog.reader.impl",
548 FaultySequenceFileLogReader.class, HLog.Reader.class);
549 for (FaultySequenceFileLogReader.FailureType failureType : FaultySequenceFileLogReader.FailureType.values()) {
550 conf.set("faultysequencefilelogreader.failuretype", failureType.name());
551 generateHLogs(1, ENTRIES, -1);
552 fs.initialize(fs.getUri(), conf);
553 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
554 FileStatus[] archivedLogs = fs.listStatus(CORRUPTDIR);
555 assertEquals("expected a different file", c1.getName(), archivedLogs[0]
556 .getPath().getName());
557 assertEquals(archivedLogs.length, 1);
558 fs.delete(new Path(OLDLOGDIR, HLOG_FILE_PREFIX + "0"), false);
559 }
560 } finally {
561 conf.setClass("hbase.regionserver.hlog.reader.impl", backupClass,
562 Reader.class);
563 HLogFactory.resetLogReaderClass();
564 }
565 }
566
567 @Test (timeout=300000, expected = IOException.class)
568 public void testTrailingGarbageCorruptionLogFileSkipErrorsFalseThrows()
569 throws IOException {
570 conf.setBoolean(HBASE_SKIP_ERRORS, false);
571 Class<?> backupClass = conf.getClass("hbase.regionserver.hlog.reader.impl",
572 Reader.class);
573 InstrumentedSequenceFileLogWriter.activateFailure = false;
574 HLogFactory.resetLogReaderClass();
575
576 try {
577 conf.setClass("hbase.regionserver.hlog.reader.impl",
578 FaultySequenceFileLogReader.class, HLog.Reader.class);
579 conf.set("faultysequencefilelogreader.failuretype", FaultySequenceFileLogReader.FailureType.BEGINNING.name());
580 generateHLogs(Integer.MAX_VALUE);
581 fs.initialize(fs.getUri(), conf);
582 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
583 } finally {
584 conf.setClass("hbase.regionserver.hlog.reader.impl", backupClass,
585 Reader.class);
586 HLogFactory.resetLogReaderClass();
587 }
588 }
589
590 @Test (timeout=300000)
591 public void testCorruptedLogFilesSkipErrorsFalseDoesNotTouchLogs()
592 throws IOException {
593 conf.setBoolean(HBASE_SKIP_ERRORS, false);
594 Class<?> backupClass = conf.getClass("hbase.regionserver.hlog.reader.impl",
595 Reader.class);
596 InstrumentedSequenceFileLogWriter.activateFailure = false;
597 HLogFactory.resetLogReaderClass();
598
599 try {
600 conf.setClass("hbase.regionserver.hlog.reader.impl",
601 FaultySequenceFileLogReader.class, HLog.Reader.class);
602 conf.set("faultysequencefilelogreader.failuretype", FaultySequenceFileLogReader.FailureType.BEGINNING.name());
603 generateHLogs(-1);
604 fs.initialize(fs.getUri(), conf);
605 try {
606 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
607 } catch (IOException e) {
608 assertEquals(
609 "if skip.errors is false all files should remain in place",
610 NUM_WRITERS, fs.listStatus(HLOGDIR).length);
611 }
612 } finally {
613 conf.setClass("hbase.regionserver.hlog.reader.impl", backupClass,
614 Reader.class);
615 HLogFactory.resetLogReaderClass();
616 }
617 }
618
619 @Test (timeout=300000)
620 public void testEOFisIgnored() throws IOException {
621 conf.setBoolean(HBASE_SKIP_ERRORS, false);
622
623 final String REGION = "region__1";
624 REGIONS.removeAll(REGIONS);
625 REGIONS.add(REGION);
626
627 int entryCount = 10;
628 Path c1 = new Path(HLOGDIR, HLOG_FILE_PREFIX + "0");
629 generateHLogs(1, entryCount, -1);
630 corruptHLog(c1, Corruptions.TRUNCATE, true, fs);
631
632 fs.initialize(fs.getUri(), conf);
633 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
634
635 Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, REGION);
636 assertEquals(1, splitLog.length);
637
638 int actualCount = 0;
639 HLog.Reader in = HLogFactory.createReader(fs, splitLog[0], conf);
640 @SuppressWarnings("unused")
641 HLog.Entry entry;
642 while ((entry = in.next()) != null) ++actualCount;
643 assertEquals(entryCount-1, actualCount);
644
645
646 FileStatus[] archivedLogs = fs.listStatus(CORRUPTDIR);
647 assertEquals(archivedLogs.length, 0);
648 }
649
650 @Test (timeout=300000)
651 public void testCorruptWALTrailer() throws IOException {
652 conf.setBoolean(HBASE_SKIP_ERRORS, false);
653
654 final String REGION = "region__1";
655 REGIONS.removeAll(REGIONS);
656 REGIONS.add(REGION);
657
658 int entryCount = 10;
659 Path c1 = new Path(HLOGDIR, HLOG_FILE_PREFIX + "0");
660 generateHLogs(1, entryCount, -1);
661 corruptHLog(c1, Corruptions.TRUNCATE_TRAILER, true, fs);
662
663 fs.initialize(fs.getUri(), conf);
664 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
665
666 Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, REGION);
667 assertEquals(1, splitLog.length);
668
669 int actualCount = 0;
670 HLog.Reader in = HLogFactory.createReader(fs, splitLog[0], conf);
671 @SuppressWarnings("unused")
672 HLog.Entry entry;
673 while ((entry = in.next()) != null) ++actualCount;
674 assertEquals(entryCount, actualCount);
675
676
677 FileStatus[] archivedLogs = fs.listStatus(CORRUPTDIR);
678 assertEquals(archivedLogs.length, 0);
679 }
680
681 @Test (timeout=300000)
682 public void testLogsGetArchivedAfterSplit() throws IOException {
683 conf.setBoolean(HBASE_SKIP_ERRORS, false);
684 generateHLogs(-1);
685 fs.initialize(fs.getUri(), conf);
686 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
687 FileStatus[] archivedLogs = fs.listStatus(OLDLOGDIR);
688 assertEquals("wrong number of files in the archive log", NUM_WRITERS, archivedLogs.length);
689 }
690
691 @Test (timeout=300000)
692 public void testSplit() throws IOException {
693 generateHLogs(-1);
694 fs.initialize(fs.getUri(), conf);
695
696 int expectedFiles = fs.listStatus(HLOGDIR).length;
697 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
698 for (String region : REGIONS) {
699 Path[] logfiles = getLogForRegion(HBASEDIR, TABLE_NAME, region);
700 assertEquals(expectedFiles, logfiles.length);
701 int count = 0;
702 for (Path logfile: logfiles) {
703 count += countHLog(logfile, fs, conf);
704 }
705 assertEquals(NUM_WRITERS * ENTRIES, count);
706 }
707 }
708
709 @Test (timeout=300000)
710 public void testLogDirectoryShouldBeDeletedAfterSuccessfulSplit()
711 throws IOException {
712 generateHLogs(-1);
713 fs.initialize(fs.getUri(), conf);
714 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
715 FileStatus [] statuses = null;
716 try {
717 statuses = fs.listStatus(HLOGDIR);
718 if (statuses != null) {
719 Assert.fail("Files left in log dir: " +
720 Joiner.on(",").join(FileUtil.stat2Paths(statuses)));
721 }
722 } catch (FileNotFoundException e) {
723
724 }
725 }
726
727 @Test(timeout=300000, expected = IOException.class)
728 public void testSplitWillFailIfWritingToRegionFails() throws Exception {
729
730 HLog.Writer [] writer = generateHLogs(4);
731
732 fs.initialize(fs.getUri(), conf);
733
734 String region = "break";
735 Path regiondir = new Path(TABLEDIR, region);
736 fs.mkdirs(regiondir);
737
738 InstrumentedSequenceFileLogWriter.activateFailure = false;
739 appendEntry(writer[4], TABLE_NAME, Bytes.toBytes(region),
740 ("r" + 999).getBytes(), FAMILY, QUALIFIER, VALUE, 0);
741 writer[4].close();
742
743 try {
744 InstrumentedSequenceFileLogWriter.activateFailure = true;
745 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
746 } catch (IOException e) {
747 assertTrue(e.getMessage().
748 contains("This exception is instrumented and should only be thrown for testing"));
749 throw e;
750 } finally {
751 InstrumentedSequenceFileLogWriter.activateFailure = false;
752 }
753 }
754
755
756
757
758
759
760 public void testSplittingLargeNumberOfRegionsConsistency() throws IOException {
761
762 REGIONS.removeAll(REGIONS);
763 for (int i=0; i<100; i++) {
764 REGIONS.add("region__"+i);
765 }
766
767 generateHLogs(1, 100, -1);
768 fs.initialize(fs.getUri(), conf);
769
770 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
771 fs.rename(OLDLOGDIR, HLOGDIR);
772 Path firstSplitPath = new Path(HBASEDIR, TABLE_NAME+ ".first");
773 Path splitPath = new Path(HBASEDIR, TABLE_NAME.getNameAsString());
774 fs.rename(splitPath,
775 firstSplitPath);
776
777 fs.initialize(fs.getUri(), conf);
778 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
779 assertEquals(0, compareHLogSplitDirs(firstSplitPath, splitPath));
780 }
781
782 @Test (timeout=300000)
783 public void testSplitDeletedRegion() throws IOException {
784 REGIONS.removeAll(REGIONS);
785 String region = "region_that_splits";
786 REGIONS.add(region);
787
788 generateHLogs(1);
789 fs.initialize(fs.getUri(), conf);
790
791 Path regiondir = new Path(TABLEDIR, region);
792 fs.delete(regiondir, true);
793 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
794 assertFalse(fs.exists(regiondir));
795 }
796
797 @Test (timeout=300000)
798 public void testIOEOnOutputThread() throws Exception {
799 conf.setBoolean(HBASE_SKIP_ERRORS, false);
800
801 generateHLogs(-1);
802 fs.initialize(fs.getUri(), conf);
803 FileStatus[] logfiles = fs.listStatus(HLOGDIR);
804 assertTrue("There should be some log file",
805 logfiles != null && logfiles.length > 0);
806
807 HLogSplitter logSplitter = new HLogSplitter(
808 conf, HBASEDIR, fs, null, null) {
809 protected HLog.Writer createWriter(FileSystem fs,
810 Path logfile, Configuration conf) throws IOException {
811 HLog.Writer mockWriter = Mockito.mock(HLog.Writer.class);
812 Mockito.doThrow(new IOException("Injected")).when(
813 mockWriter).append(Mockito.<HLog.Entry>any());
814 return mockWriter;
815 }
816 };
817
818
819 final AtomicBoolean stop = new AtomicBoolean(false);
820 final Thread someOldThread = new Thread("Some-old-thread") {
821 @Override
822 public void run() {
823 while(!stop.get()) Threads.sleep(10);
824 }
825 };
826 someOldThread.setDaemon(true);
827 someOldThread.start();
828 final Thread t = new Thread("Background-thread-dumper") {
829 public void run() {
830 try {
831 Threads.threadDumpingIsAlive(someOldThread);
832 } catch (InterruptedException e) {
833 e.printStackTrace();
834 }
835 }
836 };
837 t.setDaemon(true);
838 t.start();
839 try {
840 logSplitter.splitLogFile(logfiles[0], null);
841 fail("Didn't throw!");
842 } catch (IOException ioe) {
843 assertTrue(ioe.toString().contains("Injected"));
844 } finally {
845
846 stop.set(true);
847 }
848 }
849
850
851 @Test (timeout=300000)
852 public void testMovedHLogDuringRecovery() throws Exception {
853 generateHLogs(-1);
854
855 fs.initialize(fs.getUri(), conf);
856
857
858
859 FileSystem spiedFs = Mockito.spy(fs);
860
861
862 Mockito.doThrow(new LeaseExpiredException("Injected: File does not exist")).
863 when(spiedFs).append(Mockito.<Path>any());
864
865 try {
866 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, spiedFs, conf);
867 assertEquals(NUM_WRITERS, fs.listStatus(OLDLOGDIR).length);
868 assertFalse(fs.exists(HLOGDIR));
869 } catch (IOException e) {
870 fail("There shouldn't be any exception but: " + e.toString());
871 }
872 }
873
874 @Test (timeout=300000)
875 public void testRetryOpenDuringRecovery() throws Exception {
876 generateHLogs(-1);
877
878 fs.initialize(fs.getUri(), conf);
879
880 FileSystem spiedFs = Mockito.spy(fs);
881
882
883
884
885
886
887
888
889
890 Mockito.doAnswer(new Answer<FSDataInputStream>() {
891 private final String[] errors = new String[] {
892 "Cannot obtain block length", "Could not obtain the last block",
893 "Blocklist for " + OLDLOGDIR + " has changed"};
894 private int count = 0;
895
896 public FSDataInputStream answer(InvocationOnMock invocation) throws Throwable {
897 if (count < 3) {
898 throw new IOException(errors[count++]);
899 }
900 return (FSDataInputStream)invocation.callRealMethod();
901 }
902 }).when(spiedFs).open(Mockito.<Path>any(), Mockito.anyInt());
903
904 try {
905 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, spiedFs, conf);
906 assertEquals(NUM_WRITERS, fs.listStatus(OLDLOGDIR).length);
907 assertFalse(fs.exists(HLOGDIR));
908 } catch (IOException e) {
909 fail("There shouldn't be any exception but: " + e.toString());
910 }
911 }
912
913 @Test (timeout=300000)
914 public void testTerminationAskedByReporter() throws IOException, CorruptedLogFileException {
915 generateHLogs(1, 10, -1);
916 FileStatus logfile = fs.listStatus(HLOGDIR)[0];
917 fs.initialize(fs.getUri(), conf);
918
919 final AtomicInteger count = new AtomicInteger();
920
921 CancelableProgressable localReporter
922 = new CancelableProgressable() {
923 @Override
924 public boolean progress() {
925 count.getAndIncrement();
926 return false;
927 }
928 };
929
930 FileSystem spiedFs = Mockito.spy(fs);
931 Mockito.doAnswer(new Answer<FSDataInputStream>() {
932 public FSDataInputStream answer(InvocationOnMock invocation) throws Throwable {
933 Thread.sleep(1500);
934 return (FSDataInputStream)invocation.callRealMethod();
935 }
936 }).when(spiedFs).open(Mockito.<Path>any(), Mockito.anyInt());
937
938 try {
939 conf.setInt("hbase.splitlog.report.period", 1000);
940 boolean ret = HLogSplitter.splitLogFile(
941 HBASEDIR, logfile, spiedFs, conf, localReporter, null, null);
942 assertFalse("Log splitting should failed", ret);
943 assertTrue(count.get() > 0);
944 } catch (IOException e) {
945 fail("There shouldn't be any exception but: " + e.toString());
946 } finally {
947
948 conf.setInt("hbase.splitlog.report.period", 59000);
949 }
950 }
951
952
953
954
955
956 @Test (timeout=300000)
957 public void testThreading() throws Exception {
958 doTestThreading(20000, 128*1024*1024, 0);
959 }
960
961
962
963
964
965 @Test (timeout=300000)
966 public void testThreadingSlowWriterSmallBuffer() throws Exception {
967 doTestThreading(200, 1024, 50);
968 }
969
970
971
972
973
974
975
976
977
978
979
980
981
982 private void doTestThreading(final int numFakeEdits,
983 final int bufferSize,
984 final int writerSlowness) throws Exception {
985
986 Configuration localConf = new Configuration(conf);
987 localConf.setInt("hbase.regionserver.hlog.splitlog.buffersize", bufferSize);
988
989
990 Path logPath = new Path(HLOGDIR, HLOG_FILE_PREFIX + ".fake");
991 FSDataOutputStream out = fs.create(logPath);
992 out.close();
993
994
995 final List<String> regions = ImmutableList.of("r0", "r1", "r2", "r3", "r4");
996 makeRegionDirs(fs, regions);
997
998
999 HLogSplitter logSplitter = new HLogSplitter(
1000 localConf, HBASEDIR, fs, null, null) {
1001
1002
1003 protected HLog.Writer createWriter(FileSystem fs, Path logfile, Configuration conf)
1004 throws IOException {
1005 HLog.Writer mockWriter = Mockito.mock(HLog.Writer.class);
1006 Mockito.doAnswer(new Answer<Void>() {
1007 int expectedIndex = 0;
1008
1009 @Override
1010 public Void answer(InvocationOnMock invocation) {
1011 if (writerSlowness > 0) {
1012 try {
1013 Thread.sleep(writerSlowness);
1014 } catch (InterruptedException ie) {
1015 Thread.currentThread().interrupt();
1016 }
1017 }
1018 HLog.Entry entry = (Entry) invocation.getArguments()[0];
1019 WALEdit edit = entry.getEdit();
1020 List<KeyValue> keyValues = edit.getKeyValues();
1021 assertEquals(1, keyValues.size());
1022 KeyValue kv = keyValues.get(0);
1023
1024
1025 assertEquals(expectedIndex, Bytes.toInt(kv.getRow()));
1026 expectedIndex++;
1027 return null;
1028 }
1029 }).when(mockWriter).append(Mockito.<HLog.Entry>any());
1030 return mockWriter;
1031 }
1032
1033
1034 protected Reader getReader(FileSystem fs, Path curLogFile,
1035 Configuration conf, CancelableProgressable reporter) throws IOException {
1036 Reader mockReader = Mockito.mock(Reader.class);
1037 Mockito.doAnswer(new Answer<HLog.Entry>() {
1038 int index = 0;
1039
1040 @Override
1041 public HLog.Entry answer(InvocationOnMock invocation) throws Throwable {
1042 if (index >= numFakeEdits) return null;
1043
1044
1045 int regionIdx = index % regions.size();
1046 byte region[] = new byte[] {(byte)'r', (byte) (0x30 + regionIdx)};
1047
1048 HLog.Entry ret = createTestEntry(TABLE_NAME, region,
1049 Bytes.toBytes((int)(index / regions.size())),
1050 FAMILY, QUALIFIER, VALUE, index);
1051 index++;
1052 return ret;
1053 }
1054 }).when(mockReader).next();
1055 return mockReader;
1056 }
1057 };
1058
1059 logSplitter.splitLogFile(fs.getFileStatus(logPath), null);
1060
1061
1062 Map<byte[], Long> outputCounts = logSplitter.outputSink.getOutputCounts();
1063 for (Map.Entry<byte[], Long> entry : outputCounts.entrySet()) {
1064 LOG.info("Got " + entry.getValue() + " output edits for region " +
1065 Bytes.toString(entry.getKey()));
1066 assertEquals((long)entry.getValue(), numFakeEdits / regions.size());
1067 }
1068 assertEquals(regions.size(), outputCounts.size());
1069 }
1070
1071
1072
1073
1074 @Test (timeout=300000)
1075 @Ignore("Need HADOOP-6886, HADOOP-6840, & HDFS-617 for this. HDFS 0.20.205.1+ should have this")
1076 public void testLogRollAfterSplitStart() throws IOException {
1077 HLog log = null;
1078 String logName = "testLogRollAfterSplitStart";
1079 Path thisTestsDir = new Path(HBASEDIR, logName);
1080
1081 try {
1082
1083 TableName tableName =
1084 TableName.valueOf(this.getClass().getName());
1085 HRegionInfo regioninfo = new HRegionInfo(tableName,
1086 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
1087 log = HLogFactory.createHLog(fs, HBASEDIR, logName, conf);
1088 final AtomicLong sequenceId = new AtomicLong(1);
1089
1090 final int total = 20;
1091 for (int i = 0; i < total; i++) {
1092 WALEdit kvs = new WALEdit();
1093 kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
1094 HTableDescriptor htd = new HTableDescriptor(tableName);
1095 htd.addFamily(new HColumnDescriptor("column"));
1096 log.append(regioninfo, tableName, kvs, System.currentTimeMillis(), htd, sequenceId);
1097 }
1098
1099 log.sync();
1100 ((FSHLog) log).cleanupCurrentWriter(log.getFilenum());
1101
1102
1103
1104
1105
1106 Path rsSplitDir = new Path(thisTestsDir.getParent(),
1107 thisTestsDir.getName() + "-splitting");
1108 fs.rename(thisTestsDir, rsSplitDir);
1109 LOG.debug("Renamed region directory: " + rsSplitDir);
1110
1111
1112 HLogSplitter.split(HBASEDIR, rsSplitDir, OLDLOGDIR, fs, conf);
1113
1114
1115 try {
1116 log.rollWriter();
1117 Assert.fail("rollWriter() did not throw any exception.");
1118 } catch (IOException ioe) {
1119 if (ioe.getCause().getMessage().contains("FileNotFound")) {
1120 LOG.info("Got the expected exception: ", ioe.getCause());
1121 } else {
1122 Assert.fail("Unexpected exception: " + ioe);
1123 }
1124 }
1125 } finally {
1126 if (log != null) {
1127 log.close();
1128 }
1129 if (fs.exists(thisTestsDir)) {
1130 fs.delete(thisTestsDir, true);
1131 }
1132 }
1133 }
1134
1135
1136
1137
1138
1139
1140 class ZombieNewLogWriterRegionServer extends Thread {
1141 AtomicBoolean stop;
1142 CountDownLatch latch;
1143 public ZombieNewLogWriterRegionServer(CountDownLatch latch, AtomicBoolean stop) {
1144 super("ZombieNewLogWriterRegionServer");
1145 this.latch = latch;
1146 this.stop = stop;
1147 }
1148
1149 @Override
1150 public void run() {
1151 if (stop.get()) {
1152 return;
1153 }
1154 Path tableDir = FSUtils.getTableDir(HBASEDIR, TABLE_NAME);
1155 Path regionDir = new Path(tableDir, REGIONS.get(0));
1156 Path recoveredEdits = new Path(regionDir, HConstants.RECOVERED_EDITS_DIR);
1157 String region = "juliet";
1158 Path julietLog = new Path(HLOGDIR, HLOG_FILE_PREFIX + ".juliet");
1159 try {
1160
1161 while (!fs.exists(recoveredEdits) && !stop.get()) {
1162 LOG.info("Juliet: split not started, sleeping a bit...");
1163 Threads.sleep(10);
1164 }
1165
1166 fs.mkdirs(new Path(tableDir, region));
1167 HLog.Writer writer = HLogFactory.createWALWriter(fs,
1168 julietLog, conf);
1169 appendEntry(writer, TableName.valueOf("juliet"), ("juliet").getBytes(),
1170 ("r").getBytes(), FAMILY, QUALIFIER, VALUE, 0);
1171 writer.close();
1172 LOG.info("Juliet file creator: created file " + julietLog);
1173 latch.countDown();
1174 } catch (IOException e1) {
1175 LOG.error("Failed to create file " + julietLog, e1);
1176 assertTrue("Failed to create file " + julietLog, false);
1177 }
1178 }
1179 }
1180
1181 @Test (timeout=300000)
1182 public void testSplitLogFileWithOneRegion() throws IOException {
1183 LOG.info("testSplitLogFileWithOneRegion");
1184 final String REGION = "region__1";
1185 REGIONS.removeAll(REGIONS);
1186 REGIONS.add(REGION);
1187
1188 generateHLogs(1, 10, -1);
1189 fs.initialize(fs.getUri(), conf);
1190 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
1191
1192 Path originalLog = (fs.listStatus(OLDLOGDIR))[0].getPath();
1193 Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, REGION);
1194 assertEquals(1, splitLog.length);
1195
1196 assertEquals(true, logsAreEqual(originalLog, splitLog[0]));
1197 }
1198
1199 @Test (timeout=300000)
1200 public void testSplitLogFileDeletedRegionDir() throws IOException {
1201 LOG.info("testSplitLogFileDeletedRegionDir");
1202 final String REGION = "region__1";
1203 REGIONS.removeAll(REGIONS);
1204 REGIONS.add(REGION);
1205
1206 generateHLogs(1, 10, -1);
1207 fs.initialize(fs.getUri(), conf);
1208
1209 Path regiondir = new Path(TABLEDIR, REGION);
1210 LOG.info("Region directory is" + regiondir);
1211 fs.delete(regiondir, true);
1212
1213 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
1214
1215 assertTrue(!fs.exists(regiondir));
1216 assertTrue(true);
1217 }
1218
1219 @Test (timeout=300000)
1220 public void testSplitLogFileEmpty() throws IOException {
1221 LOG.info("testSplitLogFileEmpty");
1222 injectEmptyFile(".empty", true);
1223
1224 fs.initialize(fs.getUri(), conf);
1225
1226 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
1227 Path tdir = FSUtils.getTableDir(HBASEDIR, TABLE_NAME);
1228 assertFalse(fs.exists(tdir));
1229
1230 assertEquals(0, countHLog(fs.listStatus(OLDLOGDIR)[0].getPath(), fs, conf));
1231 }
1232
1233 @Test (timeout=300000)
1234 public void testSplitLogFileMultipleRegions() throws IOException {
1235 LOG.info("testSplitLogFileMultipleRegions");
1236 generateHLogs(1, 10, -1);
1237 fs.initialize(fs.getUri(), conf);
1238
1239 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
1240 for (String region : REGIONS) {
1241 Path[] recovered = getLogForRegion(HBASEDIR, TABLE_NAME, region);
1242 assertEquals(1, recovered.length);
1243 assertEquals(10, countHLog(recovered[0], fs, conf));
1244 }
1245 }
1246
1247 @Test (timeout=300000)
1248 public void testSplitLogFileFirstLineCorruptionLog()
1249 throws IOException {
1250 conf.setBoolean(HBASE_SKIP_ERRORS, true);
1251 generateHLogs(1, 10, -1);
1252 FileStatus logfile = fs.listStatus(HLOGDIR)[0];
1253
1254 corruptHLog(logfile.getPath(),
1255 Corruptions.INSERT_GARBAGE_ON_FIRST_LINE, true, fs);
1256
1257 fs.initialize(fs.getUri(), conf);
1258 HLogSplitter.split(HBASEDIR, HLOGDIR, OLDLOGDIR, fs, conf);
1259
1260 final Path corruptDir = new Path(FSUtils.getRootDir(conf), conf.get(
1261 "hbase.regionserver.hlog.splitlog.corrupt.dir", HConstants.CORRUPT_DIR_NAME));
1262 assertEquals(1, fs.listStatus(corruptDir).length);
1263 }
1264
1265
1266
1267
1268
1269 @Test (timeout=300000)
1270 public void testConcurrentSplitLogAndReplayRecoverEdit() throws IOException {
1271 LOG.info("testConcurrentSplitLogAndReplayRecoverEdit");
1272
1273 String regionName = "r0";
1274 final Path regiondir = new Path(TABLEDIR, regionName);
1275 REGIONS = new ArrayList<String>();
1276 REGIONS.add(regionName);
1277 generateHLogs(-1);
1278
1279 HLogFactory.createHLog(fs, regiondir, regionName, conf);
1280 FileStatus[] logfiles = fs.listStatus(HLOGDIR);
1281 assertTrue("There should be some log file",
1282 logfiles != null && logfiles.length > 0);
1283
1284 HLogSplitter logSplitter = new HLogSplitter(
1285 conf, HBASEDIR, fs, null, null) {
1286 protected HLog.Writer createWriter(FileSystem fs, Path logfile, Configuration conf)
1287 throws IOException {
1288 HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, logfile, conf);
1289
1290
1291
1292 NavigableSet<Path> files = HLogUtil.getSplitEditFilesSorted(fs, regiondir);
1293 if (files != null && !files.isEmpty()) {
1294 for (Path file : files) {
1295 if (!this.fs.delete(file, false)) {
1296 LOG.error("Failed delete of " + file);
1297 } else {
1298 LOG.debug("Deleted recovered.edits file=" + file);
1299 }
1300 }
1301 }
1302 return writer;
1303 }
1304 };
1305 try{
1306 logSplitter.splitLogFile(logfiles[0], null);
1307 } catch (IOException e) {
1308 LOG.info(e);
1309 Assert.fail("Throws IOException when spliting "
1310 + "log, it is most likely because writing file does not "
1311 + "exist which is caused by concurrent replayRecoveredEditsIfAny()");
1312 }
1313 if (fs.exists(CORRUPTDIR)) {
1314 if (fs.listStatus(CORRUPTDIR).length > 0) {
1315 Assert.fail("There are some corrupt logs, "
1316 + "it is most likely caused by concurrent replayRecoveredEditsIfAny()");
1317 }
1318 }
1319 }
1320
1321 private static void flushToConsole(String s) {
1322 System.out.println(s);
1323 System.out.flush();
1324 }
1325
1326
1327 private HLog.Writer [] generateHLogs(int leaveOpen) throws IOException {
1328 return generateHLogs(NUM_WRITERS, ENTRIES, leaveOpen);
1329 }
1330
1331 private HLog.Writer [] generateHLogs(final int writers, final int entries, final int leaveOpen) throws IOException {
1332 return generateHLogs((DistributedFileSystem)this.fs, writers, entries, leaveOpen);
1333 }
1334
1335 private static void makeRegionDirs(FileSystem fs, List<String> regions) throws IOException {
1336 for (String region : regions) {
1337 flushToConsole("Creating dir for region " + region);
1338 fs.mkdirs(new Path(TABLEDIR, region));
1339 }
1340 }
1341
1342 private static HLog.Writer [] generateHLogs(final DistributedFileSystem dfs, int writers, int entries, int leaveOpen)
1343 throws IOException {
1344 makeRegionDirs(dfs, REGIONS);
1345 dfs.mkdirs(HLOGDIR);
1346 HLog.Writer [] ws = new HLog.Writer[writers];
1347 int seq = 0;
1348 for (int i = 0; i < writers; i++) {
1349 ws[i] = HLogFactory.createWALWriter(dfs, new Path(HLOGDIR, HLOG_FILE_PREFIX + i), dfs.getConf());
1350 for (int j = 0; j < entries; j++) {
1351 int prefix = 0;
1352 for (String region : REGIONS) {
1353 String row_key = region + prefix++ + i + j;
1354 appendEntry(ws[i], TABLE_NAME, region.getBytes(), row_key.getBytes(), FAMILY, QUALIFIER, VALUE, seq++);
1355 }
1356 }
1357 if (i != leaveOpen) {
1358 ws[i].close();
1359 LOG.info("Closing writer " + i);
1360 }
1361 }
1362 return ws;
1363 }
1364
1365 private Path[] getLogForRegion(Path rootdir, TableName table, String region)
1366 throws IOException {
1367 Path tdir = FSUtils.getTableDir(rootdir, table);
1368 @SuppressWarnings("deprecation")
1369 Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(HRegion.getRegionDir(tdir,
1370 Bytes.toString(region.getBytes())));
1371 FileStatus [] files = this.fs.listStatus(editsdir);
1372 Path[] paths = new Path[files.length];
1373 for (int i = 0; i < files.length; i++) {
1374 paths[i] = files[i].getPath();
1375 }
1376 return paths;
1377 }
1378
1379 private void corruptHLog(Path path, Corruptions corruption, boolean close,
1380 FileSystem fs) throws IOException {
1381
1382 FSDataOutputStream out;
1383 int fileSize = (int) fs.listStatus(path)[0].getLen();
1384
1385 FSDataInputStream in = fs.open(path);
1386 byte[] corrupted_bytes = new byte[fileSize];
1387 in.readFully(0, corrupted_bytes, 0, fileSize);
1388 in.close();
1389
1390 switch (corruption) {
1391 case APPEND_GARBAGE:
1392 fs.delete(path, false);
1393 out = fs.create(path);
1394 out.write(corrupted_bytes);
1395 out.write("-----".getBytes());
1396 closeOrFlush(close, out);
1397 break;
1398
1399 case INSERT_GARBAGE_ON_FIRST_LINE:
1400 fs.delete(path, false);
1401 out = fs.create(path);
1402 out.write(0);
1403 out.write(corrupted_bytes);
1404 closeOrFlush(close, out);
1405 break;
1406
1407 case INSERT_GARBAGE_IN_THE_MIDDLE:
1408 fs.delete(path, false);
1409 out = fs.create(path);
1410 int middle = (int) Math.floor(corrupted_bytes.length / 2);
1411 out.write(corrupted_bytes, 0, middle);
1412 out.write(0);
1413 out.write(corrupted_bytes, middle, corrupted_bytes.length - middle);
1414 closeOrFlush(close, out);
1415 break;
1416
1417 case TRUNCATE:
1418 fs.delete(path, false);
1419 out = fs.create(path);
1420 out.write(corrupted_bytes, 0, fileSize
1421 - (32 + ProtobufLogReader.PB_WAL_COMPLETE_MAGIC.length + Bytes.SIZEOF_INT));
1422 closeOrFlush(close, out);
1423 break;
1424
1425 case TRUNCATE_TRAILER:
1426 fs.delete(path, false);
1427 out = fs.create(path);
1428 out.write(corrupted_bytes, 0, fileSize - Bytes.SIZEOF_INT);
1429 closeOrFlush(close, out);
1430 break;
1431 }
1432 }
1433
1434 private void closeOrFlush(boolean close, FSDataOutputStream out)
1435 throws IOException {
1436 if (close) {
1437 out.close();
1438 } else {
1439 Method syncMethod = null;
1440 try {
1441 syncMethod = out.getClass().getMethod("hflush", new Class<?> []{});
1442 } catch (NoSuchMethodException e) {
1443 try {
1444 syncMethod = out.getClass().getMethod("sync", new Class<?> []{});
1445 } catch (NoSuchMethodException ex) {
1446 throw new IOException("This version of Hadoop supports " +
1447 "neither Syncable.sync() nor Syncable.hflush().");
1448 }
1449 }
1450 try {
1451 syncMethod.invoke(out, new Object[]{});
1452 } catch (Exception e) {
1453 throw new IOException(e);
1454 }
1455
1456 }
1457 }
1458
1459 @SuppressWarnings("unused")
1460 private void dumpHLog(Path log, FileSystem fs, Configuration conf) throws IOException {
1461 HLog.Entry entry;
1462 HLog.Reader in = HLogFactory.createReader(fs, log, conf);
1463 while ((entry = in.next()) != null) {
1464 System.out.println(entry);
1465 }
1466 }
1467
1468 private int countHLog(Path log, FileSystem fs, Configuration conf) throws IOException {
1469 int count = 0;
1470 HLog.Reader in = HLogFactory.createReader(fs, log, conf);
1471 while (in.next() != null) {
1472 count++;
1473 }
1474 return count;
1475 }
1476
1477
1478 public static long appendEntry(HLog.Writer writer, TableName table, byte[] region,
1479 byte[] row, byte[] family, byte[] qualifier,
1480 byte[] value, long seq)
1481 throws IOException {
1482 LOG.info(Thread.currentThread().getName() + " append");
1483 writer.append(createTestEntry(table, region, row, family, qualifier, value, seq));
1484 LOG.info(Thread.currentThread().getName() + " sync");
1485 writer.sync();
1486 return seq;
1487 }
1488
1489 private static HLog.Entry createTestEntry(
1490 TableName table, byte[] region,
1491 byte[] row, byte[] family, byte[] qualifier,
1492 byte[] value, long seq) {
1493 long time = System.nanoTime();
1494 WALEdit edit = new WALEdit();
1495 seq++;
1496 edit.add(new KeyValue(row, family, qualifier, time, KeyValue.Type.Put, value));
1497 return new HLog.Entry(new HLogKey(region, table, seq, time,
1498 HConstants.DEFAULT_CLUSTER_ID), edit);
1499 }
1500
1501
1502 private void injectEmptyFile(String suffix, boolean closeFile)
1503 throws IOException {
1504 HLog.Writer writer = HLogFactory.createWALWriter(
1505 fs, new Path(HLOGDIR, HLOG_FILE_PREFIX + suffix), conf);
1506 if (closeFile) writer.close();
1507 }
1508
1509 @SuppressWarnings("unused")
1510 private void listLogs(FileSystem fs, Path dir) throws IOException {
1511 for (FileStatus file : fs.listStatus(dir)) {
1512 System.out.println(file.getPath());
1513 }
1514
1515 }
1516
1517 private int compareHLogSplitDirs(Path p1, Path p2) throws IOException {
1518 FileStatus[] f1 = fs.listStatus(p1);
1519 FileStatus[] f2 = fs.listStatus(p2);
1520 assertNotNull("Path " + p1 + " doesn't exist", f1);
1521 assertNotNull("Path " + p2 + " doesn't exist", f2);
1522
1523 System.out.println("Files in " + p1 + ": " +
1524 Joiner.on(",").join(FileUtil.stat2Paths(f1)));
1525 System.out.println("Files in " + p2 + ": " +
1526 Joiner.on(",").join(FileUtil.stat2Paths(f2)));
1527 assertEquals(f1.length, f2.length);
1528
1529 for (int i = 0; i < f1.length; i++) {
1530
1531
1532 Path rd1 = HLogUtil.getRegionDirRecoveredEditsDir(f1[i].getPath());
1533 FileStatus[] rd1fs = fs.listStatus(rd1);
1534 assertEquals(1, rd1fs.length);
1535 Path rd2 = HLogUtil.getRegionDirRecoveredEditsDir(f2[i].getPath());
1536 FileStatus[] rd2fs = fs.listStatus(rd2);
1537 assertEquals(1, rd2fs.length);
1538 if (!logsAreEqual(rd1fs[0].getPath(), rd2fs[0].getPath())) {
1539 return -1;
1540 }
1541 }
1542 return 0;
1543 }
1544
1545 private boolean logsAreEqual(Path p1, Path p2) throws IOException {
1546 HLog.Reader in1, in2;
1547 in1 = HLogFactory.createReader(fs, p1, conf);
1548 in2 = HLogFactory.createReader(fs, p2, conf);
1549 HLog.Entry entry1;
1550 HLog.Entry entry2;
1551 while ((entry1 = in1.next()) != null) {
1552 entry2 = in2.next();
1553 if ((entry1.getKey().compareTo(entry2.getKey()) != 0) ||
1554 (!entry1.getEdit().toString().equals(entry2.getEdit().toString()))) {
1555 return false;
1556 }
1557 }
1558 return true;
1559 }
1560 }