1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23 import static org.junit.Assert.fail;
24 import static org.mockito.Matchers.any;
25 import static org.mockito.Matchers.eq;
26 import static org.mockito.Mockito.doAnswer;
27 import static org.mockito.Mockito.spy;
28
29 import java.io.FilterInputStream;
30 import java.io.IOException;
31 import java.io.InputStream;
32 import java.lang.reflect.Field;
33 import java.security.PrivilegedExceptionAction;
34 import java.util.ArrayList;
35 import java.util.List;
36 import java.util.SortedSet;
37 import java.util.concurrent.atomic.AtomicBoolean;
38 import java.util.concurrent.atomic.AtomicInteger;
39 import java.util.concurrent.atomic.AtomicLong;
40
41 import org.apache.commons.logging.Log;
42 import org.apache.commons.logging.LogFactory;
43 import org.apache.hadoop.conf.Configuration;
44 import org.apache.hadoop.fs.FSDataInputStream;
45 import org.apache.hadoop.fs.FileStatus;
46 import org.apache.hadoop.fs.FileSystem;
47 import org.apache.hadoop.fs.Path;
48 import org.apache.hadoop.hbase.Cell;
49 import org.apache.hadoop.hbase.HBaseConfiguration;
50 import org.apache.hadoop.hbase.HBaseTestingUtility;
51 import org.apache.hadoop.hbase.HColumnDescriptor;
52 import org.apache.hadoop.hbase.HConstants;
53 import org.apache.hadoop.hbase.HRegionInfo;
54 import org.apache.hadoop.hbase.HTableDescriptor;
55 import org.apache.hadoop.hbase.KeyValue;
56 import org.apache.hadoop.hbase.MasterNotRunningException;
57 import org.apache.hadoop.hbase.MiniHBaseCluster;
58 import org.apache.hadoop.hbase.ServerName;
59 import org.apache.hadoop.hbase.TableName;
60 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
61 import org.apache.hadoop.hbase.client.Delete;
62 import org.apache.hadoop.hbase.client.Get;
63 import org.apache.hadoop.hbase.client.HTable;
64 import org.apache.hadoop.hbase.client.Put;
65 import org.apache.hadoop.hbase.client.Result;
66 import org.apache.hadoop.hbase.client.ResultScanner;
67 import org.apache.hadoop.hbase.client.Scan;
68 import org.apache.hadoop.hbase.master.HMaster;
69 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
70 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
71 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
72 import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
73 import org.apache.hadoop.hbase.regionserver.FlushRequester;
74 import org.apache.hadoop.hbase.regionserver.HRegion;
75 import org.apache.hadoop.hbase.regionserver.HRegionServer;
76 import org.apache.hadoop.hbase.regionserver.RegionScanner;
77 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
78 import org.apache.hadoop.hbase.regionserver.Store;
79 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
80 import org.apache.hadoop.hbase.security.User;
81 import org.apache.hadoop.hbase.testclassification.MediumTests;
82 import org.apache.hadoop.hbase.util.Bytes;
83 import org.apache.hadoop.hbase.util.EnvironmentEdge;
84 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
85 import org.apache.hadoop.hbase.util.FSUtils;
86 import org.apache.hadoop.hbase.util.HFileTestUtil;
87 import org.apache.hadoop.hbase.util.Pair;
88 import org.junit.After;
89 import org.junit.AfterClass;
90 import org.junit.Before;
91 import org.junit.BeforeClass;
92 import org.junit.Test;
93 import org.junit.experimental.categories.Category;
94 import org.mockito.Mockito;
95 import org.mockito.invocation.InvocationOnMock;
96 import org.mockito.stubbing.Answer;
97
98
99
100
101 @Category(MediumTests.class)
102 public class TestWALReplay {
103 public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
104 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
105 private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
106 private Path hbaseRootDir = null;
107 private String logName;
108 private Path oldLogDir;
109 private Path logDir;
110 private FileSystem fs;
111 private Configuration conf;
112 private RecoveryMode mode;
113
114
115 @BeforeClass
116 public static void setUpBeforeClass() throws Exception {
117 Configuration conf = TEST_UTIL.getConfiguration();
118 conf.setBoolean("dfs.support.append", true);
119
120 conf.setInt("dfs.client.block.recovery.retries", 2);
121 TEST_UTIL.startMiniCluster(3);
122 Path hbaseRootDir =
123 TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
124 LOG.info("hbase.rootdir=" + hbaseRootDir);
125 FSUtils.setRootDir(conf, hbaseRootDir);
126 }
127
128 @AfterClass
129 public static void tearDownAfterClass() throws Exception {
130 TEST_UTIL.shutdownMiniCluster();
131 }
132
133 @Before
134 public void setUp() throws Exception {
135 this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
136 this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
137 this.hbaseRootDir = FSUtils.getRootDir(this.conf);
138 this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
139 this.logName = HConstants.HREGION_LOGDIR_NAME;
140 this.logDir = new Path(this.hbaseRootDir, logName);
141 if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
142 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
143 }
144 this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
145 RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
146 }
147
148 @After
149 public void tearDown() throws Exception {
150 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
151 }
152
153
154
155
156 private void deleteDir(final Path p) throws IOException {
157 if (this.fs.exists(p)) {
158 if (!this.fs.delete(p, true)) {
159 throw new IOException("Failed remove of " + p);
160 }
161 }
162 }
163
164
165
166
167
168 @Test
169 public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
170 final TableName tableName =
171 TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
172 byte[] family1 = Bytes.toBytes("cf1");
173 byte[] family2 = Bytes.toBytes("cf2");
174 byte[] qualifier = Bytes.toBytes("q");
175 byte[] value = Bytes.toBytes("testV");
176 byte[][] familys = { family1, family2 };
177 TEST_UTIL.createTable(tableName, familys);
178 HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
179 Put put = new Put(Bytes.toBytes("r1"));
180 put.add(family1, qualifier, value);
181 htable.put(put);
182 ResultScanner resultScanner = htable.getScanner(new Scan());
183 int count = 0;
184 while (resultScanner.next() != null) {
185 count++;
186 }
187 resultScanner.close();
188 assertEquals(1, count);
189
190 MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
191 List<HRegion> regions = hbaseCluster.getRegions(tableName);
192 assertEquals(1, regions.size());
193
194
195 HRegion destRegion = regions.get(0);
196 int originServerNum = hbaseCluster
197 .getServerWith(destRegion.getRegionName());
198 assertTrue("Please start more than 1 regionserver", hbaseCluster
199 .getRegionServerThreads().size() > 1);
200 int destServerNum = 0;
201 while (destServerNum == originServerNum) {
202 destServerNum++;
203 }
204 HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
205 HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
206
207 moveRegionAndWait(destRegion, destServer);
208
209
210 Delete del = new Delete(Bytes.toBytes("r1"));
211 htable.delete(del);
212 resultScanner = htable.getScanner(new Scan());
213 count = 0;
214 while (resultScanner.next() != null) {
215 count++;
216 }
217 resultScanner.close();
218 assertEquals(0, count);
219
220
221 destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
222
223 for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
224 .getStores().values()) {
225 store.triggerMajorCompaction();
226 }
227 destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
228
229
230 moveRegionAndWait(destRegion, originServer);
231
232 originServer.abort("testing");
233
234
235 Result result = htable.get(new Get(Bytes.toBytes("r1")));
236 if (result != null) {
237 assertTrue("Row is deleted, but we get" + result.toString(),
238 (result == null) || result.isEmpty());
239 }
240 resultScanner.close();
241 }
242
243 private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
244 throws InterruptedException, MasterNotRunningException,
245 ZooKeeperConnectionException, IOException {
246 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
247 TEST_UTIL.getHBaseAdmin().move(
248 destRegion.getRegionInfo().getEncodedNameAsBytes(),
249 Bytes.toBytes(destServer.getServerName().getServerName()));
250 while (true) {
251 ServerName serverName = master.getAssignmentManager()
252 .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
253 if (serverName != null && serverName.equals(destServer.getServerName())) {
254 TEST_UTIL.assertRegionOnServer(
255 destRegion.getRegionInfo(), serverName, 200);
256 break;
257 }
258 Thread.sleep(10);
259 }
260 }
261
262
263
264
265
266
267 @Test
268 public void test2727() throws Exception {
269
270
271 final TableName tableName =
272 TableName.valueOf("test2727");
273 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
274 Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
275 deleteDir(basedir);
276
277 HTableDescriptor htd = createBasic3FamilyHTD(tableName);
278 HRegion region2 = HRegion.createHRegion(hri,
279 hbaseRootDir, this.conf, htd);
280 HRegion.closeHRegion(region2);
281 final byte [] rowName = tableName.getName();
282
283 HLog wal1 = createWAL(this.conf);
284
285 final int countPerFamily = 1000;
286 final AtomicLong sequenceId = new AtomicLong(1);
287 for (HColumnDescriptor hcd: htd.getFamilies()) {
288 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
289 wal1, htd, sequenceId);
290 }
291 wal1.close();
292 runWALSplit(this.conf);
293
294 HLog wal2 = createWAL(this.conf);
295
296 for (HColumnDescriptor hcd: htd.getFamilies()) {
297 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
298 ee, wal2, htd, sequenceId);
299 }
300 wal2.close();
301 runWALSplit(this.conf);
302
303 HLog wal3 = createWAL(this.conf);
304 try {
305 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
306 long seqid = region.getOpenSeqNum();
307
308
309 assertTrue(seqid > sequenceId.get());
310 assertEquals(seqid - 1, sequenceId.get());
311 LOG.debug("region.getOpenSeqNum(): " + region.getOpenSeqNum() + ", wal3.id: "
312 + sequenceId.get());
313
314
315 region.close();
316 } finally {
317 wal3.closeAndDelete();
318 }
319 }
320
321
322
323
324
325
326
327
328
329
330 @Test
331 public void testRegionMadeOfBulkLoadedFilesOnly()
332 throws IOException, SecurityException, IllegalArgumentException,
333 NoSuchFieldException, IllegalAccessException, InterruptedException {
334 final TableName tableName =
335 TableName.valueOf("testRegionMadeOfBulkLoadedFilesOnly");
336 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
337 final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
338 deleteDir(basedir);
339 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
340 HRegion region2 = HRegion.createHRegion(hri,
341 hbaseRootDir, this.conf, htd);
342 HRegion.closeHRegion(region2);
343 HLog wal = createWAL(this.conf);
344 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
345
346 byte [] family = htd.getFamilies().iterator().next().getName();
347 Path f = new Path(basedir, "hfile");
348 HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""),
349 Bytes.toBytes("z"), 10);
350 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
351 hfs.add(Pair.newPair(family, f.toString()));
352 region.bulkLoadHFiles(hfs, true);
353
354
355 byte [] row = tableName.getName();
356 region.put((new Put(row)).add(family, family, family));
357 wal.sync();
358 final int rowsInsertedCount = 11;
359
360 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
361
362
363 final Configuration newConf = HBaseConfiguration.create(this.conf);
364 User user = HBaseTestingUtility.getDifferentUser(newConf,
365 tableName.getNameAsString());
366 user.runAs(new PrivilegedExceptionAction() {
367 public Object run() throws Exception {
368 runWALSplit(newConf);
369 HLog wal2 = createWAL(newConf);
370
371 HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
372 hbaseRootDir, hri, htd, wal2);
373 long seqid2 = region2.getOpenSeqNum();
374 assertTrue(seqid2 > -1);
375 assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
376
377
378 region2.close();
379 wal2.closeAndDelete();
380 return null;
381 }
382 });
383 }
384
385
386
387
388
389
390
391
392
393
394
395
396 @Test
397 public void testCompactedBulkLoadedFiles()
398 throws IOException, SecurityException, IllegalArgumentException,
399 NoSuchFieldException, IllegalAccessException, InterruptedException {
400 final TableName tableName =
401 TableName.valueOf("testCompactedBulkLoadedFiles");
402 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
403 final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
404 deleteDir(basedir);
405 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
406 HRegion region2 = HRegion.createHRegion(hri,
407 hbaseRootDir, this.conf, htd);
408 HRegion.closeHRegion(region2);
409 HLog wal = createWAL(this.conf);
410 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
411
412
413 byte [] row = tableName.getName();
414 byte [] family = htd.getFamilies().iterator().next().getName();
415 region.put((new Put(row)).add(family, family, family));
416 wal.sync();
417
418 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
419 for (int i = 0; i < 3; i++) {
420 Path f = new Path(basedir, "hfile"+i);
421 HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(i + "00"),
422 Bytes.toBytes(i + "50"), 10);
423 hfs.add(Pair.newPair(family, f.toString()));
424 }
425 region.bulkLoadHFiles(hfs, true);
426 final int rowsInsertedCount = 31;
427 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
428
429
430 region.compactStores(true);
431 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
432
433
434 final Configuration newConf = HBaseConfiguration.create(this.conf);
435 User user = HBaseTestingUtility.getDifferentUser(newConf,
436 tableName.getNameAsString());
437 user.runAs(new PrivilegedExceptionAction() {
438 public Object run() throws Exception {
439 runWALSplit(newConf);
440 HLog wal2 = createWAL(newConf);
441
442 HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
443 hbaseRootDir, hri, htd, wal2);
444 long seqid2 = region2.getOpenSeqNum();
445 assertTrue(seqid2 > -1);
446 assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
447
448
449 region2.close();
450 wal2.closeAndDelete();
451 return null;
452 }
453 });
454 }
455
456
457
458
459
460
461
462
463
464
465
466 @Test
467 public void testReplayEditsWrittenViaHRegion()
468 throws IOException, SecurityException, IllegalArgumentException,
469 NoSuchFieldException, IllegalAccessException, InterruptedException {
470 final TableName tableName =
471 TableName.valueOf("testReplayEditsWrittenViaHRegion");
472 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
473 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
474 deleteDir(basedir);
475 final byte[] rowName = tableName.getName();
476 final int countPerFamily = 10;
477 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
478 HRegion region3 = HRegion.createHRegion(hri,
479 hbaseRootDir, this.conf, htd);
480 HRegion.closeHRegion(region3);
481
482
483
484 HLog wal = createWAL(this.conf);
485 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
486 long seqid = region.getOpenSeqNum();
487 boolean first = true;
488 for (HColumnDescriptor hcd: htd.getFamilies()) {
489 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
490 if (first) {
491
492 region.flushcache();
493 first = false;
494 }
495 }
496
497 final Get g = new Get(rowName);
498 Result result = region.get(g);
499 assertEquals(countPerFamily * htd.getFamilies().size(),
500 result.size());
501
502
503
504 region.close(true);
505 wal.close();
506 runWALSplit(this.conf);
507 HLog wal2 = createWAL(this.conf);
508 HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
509 long seqid2 = region2.getOpenSeqNum();
510 assertTrue(seqid + result.size() < seqid2);
511 final Result result1b = region2.get(g);
512 assertEquals(result.size(), result1b.size());
513
514
515
516
517 for (HColumnDescriptor hcd: htd.getFamilies()) {
518 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
519 }
520
521 final Result result2 = region2.get(g);
522 assertEquals(2 * result.size(), result2.size());
523 wal2.sync();
524
525
526 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal2).getOutputStream(), 1);
527 final Configuration newConf = HBaseConfiguration.create(this.conf);
528 User user = HBaseTestingUtility.getDifferentUser(newConf,
529 tableName.getNameAsString());
530 user.runAs(new PrivilegedExceptionAction() {
531 public Object run() throws Exception {
532 runWALSplit(newConf);
533 FileSystem newFS = FileSystem.get(newConf);
534
535 HLog wal3 = createWAL(newConf);
536 final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
537 HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
538 @Override
539 protected boolean restoreEdit(Store s, KeyValue kv) {
540 boolean b = super.restoreEdit(s, kv);
541 countOfRestoredEdits.incrementAndGet();
542 return b;
543 }
544 };
545 long seqid3 = region3.initialize();
546 Result result3 = region3.get(g);
547
548 assertEquals(result2.size(), result3.size());
549 assertEquals(htd.getFamilies().size() * countPerFamily,
550 countOfRestoredEdits.get());
551
552
553 region3.close();
554 wal3.closeAndDelete();
555 return null;
556 }
557 });
558 }
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578 @Test
579 public void testReplayEditsAfterPartialFlush()
580 throws IOException, SecurityException, IllegalArgumentException,
581 NoSuchFieldException, IllegalAccessException, InterruptedException {
582 final TableName tableName =
583 TableName.valueOf("testReplayEditsWrittenViaHRegion");
584 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
585 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
586 deleteDir(basedir);
587 final byte[] rowName = tableName.getName();
588 final int countPerFamily = 10;
589 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
590 HRegion region3 = HRegion.createHRegion(hri,
591 hbaseRootDir, this.conf, htd);
592 HRegion.closeHRegion(region3);
593
594
595
596 HLog wal = createWAL(this.conf);
597 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
598 long seqid = region.getOpenSeqNum();
599 for (HColumnDescriptor hcd: htd.getFamilies()) {
600 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
601 }
602
603
604 final Get g = new Get(rowName);
605 Result result = region.get(g);
606 assertEquals(countPerFamily * htd.getFamilies().size(),
607 result.size());
608
609
610 region.flushcache();
611 region.close(true);
612 wal.close();
613
614
615
616
617
618 int cf_count = 0;
619 for (HColumnDescriptor hcd: htd.getFamilies()) {
620 cf_count++;
621 if (cf_count == 2) {
622 region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
623 }
624 }
625
626
627
628 runWALSplit(this.conf);
629 HLog wal2 = createWAL(this.conf);
630 HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
631 long seqid2 = region2.getOpenSeqNum();
632 assertTrue(seqid + result.size() < seqid2);
633
634 final Result result1b = region2.get(g);
635 assertEquals(result.size(), result1b.size());
636 }
637
638
639
640
641 public static class CustomStoreFlusher extends DefaultStoreFlusher {
642
643 static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
644
645 public CustomStoreFlusher(Configuration conf, Store store) {
646 super(conf, store);
647 }
648 @Override
649 public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
650 TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status)
651 throws IOException {
652 if (throwExceptionWhenFlushing.get()) {
653 throw new IOException("Simulated exception by tests");
654 }
655 return super.flushSnapshot(snapshot, cacheFlushId, snapshotTimeRangeTracker,
656 flushedSize, status);
657 }
658
659 };
660
661
662
663
664
665
666
667 @Test
668 public void testReplayEditsAfterAbortingFlush() throws IOException {
669 final TableName tableName =
670 TableName.valueOf("testReplayEditsAfterAbortingFlush");
671 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
672 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
673 deleteDir(basedir);
674 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
675 HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
676 region3.close();
677 region3.getLog().closeAndDelete();
678
679
680
681 HLog wal = createWAL(this.conf);
682 RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
683 Mockito.doReturn(false).when(rsServices).isAborted();
684 Configuration customConf = new Configuration(this.conf);
685 customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
686 CustomStoreFlusher.class.getName());
687 HRegion region =
688 HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
689 int writtenRowCount = 10;
690 List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
691 htd.getFamilies());
692 for (int i = 0; i < writtenRowCount; i++) {
693 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
694 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
695 Bytes.toBytes("val"));
696 region.put(put);
697 }
698
699
700 RegionScanner scanner = region.getScanner(new Scan());
701 assertEquals(writtenRowCount, getScannedCount(scanner));
702
703
704 CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
705 try {
706 region.flushcache();
707 fail("Injected exception hasn't been thrown");
708 } catch (Throwable t) {
709 LOG.info("Expected simulated exception when flushing region,"
710 + t.getMessage());
711
712 Mockito.doReturn(true).when(rsServices).isAborted();
713 region.setClosing(false);
714
715 }
716
717 int moreRow = 10;
718 for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
719 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
720 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
721 Bytes.toBytes("val"));
722 region.put(put);
723 }
724 writtenRowCount += moreRow;
725
726 CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
727 try {
728 region.flushcache();
729 } catch (IOException t) {
730 LOG.info("Expected exception when flushing region because server is stopped,"
731 + t.getMessage());
732 }
733
734 region.close(true);
735 wal.close();
736
737
738 runWALSplit(this.conf);
739 HLog wal2 = createWAL(this.conf);
740 Mockito.doReturn(false).when(rsServices).isAborted();
741 HRegion region2 =
742 HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
743 scanner = region2.getScanner(new Scan());
744 assertEquals(writtenRowCount, getScannedCount(scanner));
745 }
746
747 private int getScannedCount(RegionScanner scanner) throws IOException {
748 int scannedCount = 0;
749 List<Cell> results = new ArrayList<Cell>();
750 while (true) {
751 boolean existMore = scanner.next(results);
752 if (!results.isEmpty())
753 scannedCount++;
754 if (!existMore)
755 break;
756 results.clear();
757 }
758 return scannedCount;
759 }
760
761
762
763
764
765
766 @Test
767 public void testReplayEditsWrittenIntoWAL() throws Exception {
768 final TableName tableName =
769 TableName.valueOf("testReplayEditsWrittenIntoWAL");
770 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
771 final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
772 deleteDir(basedir);
773
774 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
775 HRegion region2 = HRegion.createHRegion(hri,
776 hbaseRootDir, this.conf, htd);
777 HRegion.closeHRegion(region2);
778 final HLog wal = createWAL(this.conf);
779 final byte[] rowName = tableName.getName();
780 final byte[] regionName = hri.getEncodedNameAsBytes();
781 final AtomicLong sequenceId = new AtomicLong(1);
782
783
784 final int countPerFamily = 1000;
785 for (HColumnDescriptor hcd: htd.getFamilies()) {
786 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
787 ee, wal, htd, sequenceId);
788 }
789
790
791 wal.startCacheFlush(regionName);
792 wal.completeCacheFlush(regionName);
793
794
795 WALEdit edit = new WALEdit();
796 long now = ee.currentTimeMillis();
797 edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
798 now, rowName));
799 wal.append(hri, tableName, edit, now, htd, sequenceId);
800
801
802 edit = new WALEdit();
803 now = ee.currentTimeMillis();
804 edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
805 KeyValue.Type.DeleteFamily));
806 wal.append(hri, tableName, edit, now, htd, sequenceId);
807
808
809 wal.sync();
810
811
812 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
813
814
815 final Configuration newConf = HBaseConfiguration.create(this.conf);
816 User user = HBaseTestingUtility.getDifferentUser(newConf,
817 ".replay.wal.secondtime");
818 user.runAs(new PrivilegedExceptionAction<Void>() {
819 @Override
820 public Void run() throws Exception {
821 runWALSplit(newConf);
822 FileSystem newFS = FileSystem.get(newConf);
823
824 newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
825
826 HLog newWal = createWAL(newConf);
827 final AtomicInteger flushcount = new AtomicInteger(0);
828 try {
829 final HRegion region =
830 new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
831 protected FlushResult internalFlushcache(
832 final HLog wal, final long myseqid, MonitoredTask status)
833 throws IOException {
834 LOG.info("InternalFlushCache Invoked");
835 FlushResult fs = super.internalFlushcache(wal, myseqid,
836 Mockito.mock(MonitoredTask.class));
837 flushcount.incrementAndGet();
838 return fs;
839 };
840 };
841 long seqid = region.initialize();
842
843 assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
844 assertTrue(seqid - 1 == sequenceId.get());
845
846 Get get = new Get(rowName);
847 Result result = region.get(get);
848
849 assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
850 result.size());
851 region.close();
852 } finally {
853 newWal.closeAndDelete();
854 }
855 return null;
856 }
857 });
858 }
859
860 @Test
861
862 public void testSequentialEditLogSeqNum() throws IOException {
863 final TableName tableName =
864 TableName.valueOf("testSequentialEditLogSeqNum");
865 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
866 final Path basedir =
867 FSUtils.getTableDir(this.hbaseRootDir, tableName);
868 deleteDir(basedir);
869 final byte[] rowName = tableName.getName();
870 final int countPerFamily = 10;
871 final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
872
873
874 MockHLog wal = createMockWAL(this.conf);
875
876 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
877 for (HColumnDescriptor hcd : htd.getFamilies()) {
878 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
879 }
880
881
882
883 region.flushcache();
884 for (HColumnDescriptor hcd : htd.getFamilies()) {
885 addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
886 }
887 long lastestSeqNumber = region.getSequenceId().get();
888
889 wal.doCompleteCacheFlush = true;
890
891
892 wal.completeCacheFlush(hri.getEncodedNameAsBytes());
893 wal.close();
894 FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
895 HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0],
896 this.fs, this.conf, null, null, null, mode);
897 FileStatus[] listStatus1 = this.fs.listStatus(
898 new Path(FSUtils.getTableDir(hbaseRootDir, tableName),
899 new Path(hri.getEncodedName(), "recovered.edits")));
900 int editCount = 0;
901 for (FileStatus fileStatus : listStatus1) {
902 editCount = Integer.parseInt(fileStatus.getPath().getName());
903 }
904
905 assertEquals(
906 "The sequence number of the recoverd.edits and the current edit seq should be same",
907 lastestSeqNumber, editCount);
908 }
909
910
911
912
913 @Test
914 public void testDatalossWhenInputError() throws IOException, InstantiationException,
915 IllegalAccessException {
916 final TableName tableName = TableName.valueOf("testDatalossWhenInputError");
917 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
918 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
919 deleteDir(basedir);
920 final byte[] rowName = tableName.getName();
921 final int countPerFamily = 10;
922 final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
923 HRegion region1 = HRegion.createHRegion(hri,
924 hbaseRootDir, this.conf, htd);
925 Path regionDir = region1.getRegionFileSystem().getRegionDir();
926 HRegion.closeHRegion(region1);
927
928 HLog wal = createWAL(this.conf);
929 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
930 for (HColumnDescriptor hcd : htd.getFamilies()) {
931 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
932 }
933
934 final Get g = new Get(rowName);
935 Result result = region.get(g);
936 assertEquals(countPerFamily * htd.getFamilies().size(), result.size());
937
938
939 region.close(true);
940 wal.close();
941
942 runWALSplit(this.conf);
943
944
945 Path editFile = HLogUtil.getSplitEditFilesSorted(this.fs, regionDir).first();
946 FSDataInputStream stream = fs.open(editFile);
947 stream.seek(ProtobufLogReader.PB_WAL_MAGIC.length);
948 Class<? extends HLog.Reader> logReaderClass =
949 conf.getClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class,
950 HLog.Reader.class);
951 HLog.Reader reader = logReaderClass.newInstance();
952 reader.init(this.fs, editFile, conf, stream);
953 final long headerLength = stream.getPos();
954 reader.close();
955 FileSystem spyFs = spy(this.fs);
956 doAnswer(new Answer<FSDataInputStream>() {
957
958 @Override
959 public FSDataInputStream answer(InvocationOnMock invocation) throws Throwable {
960 FSDataInputStream stream = (FSDataInputStream) invocation.callRealMethod();
961 Field field = FilterInputStream.class.getDeclaredField("in");
962 field.setAccessible(true);
963 final InputStream in = (InputStream) field.get(stream);
964 InputStream spyIn = spy(in);
965 doAnswer(new Answer<Integer>() {
966
967 private long pos;
968
969 @Override
970 public Integer answer(InvocationOnMock invocation) throws Throwable {
971 if (pos >= headerLength) {
972 throw new IOException("read over limit");
973 }
974 int b = (Integer) invocation.callRealMethod();
975 if (b > 0) {
976 pos += b;
977 }
978 return b;
979 }
980 }).when(spyIn).read(any(byte[].class), any(int.class), any(int.class));
981 doAnswer(new Answer<Void>() {
982
983 @Override
984 public Void answer(InvocationOnMock invocation) throws Throwable {
985 invocation.callRealMethod();
986 in.close();
987 return null;
988 }
989 }).when(spyIn).close();
990 field.set(stream, spyIn);
991 return stream;
992 }
993 }).when(spyFs).open(eq(editFile));
994
995 HLog wal2 = createWAL(this.conf);
996 HRegion region2;
997 try {
998
999 region2 = HRegion.openHRegion(conf, spyFs, hbaseRootDir, hri, htd, wal2);
1000 assertEquals(result.size(), region2.get(g).size());
1001 } catch (IOException e) {
1002 assertEquals("read over limit", e.getMessage());
1003 }
1004 region2 = HRegion.openHRegion(conf, fs, hbaseRootDir, hri, htd, wal2);
1005 assertEquals(result.size(), region2.get(g).size());
1006 }
1007
1008 static class MockHLog extends FSHLog {
1009 boolean doCompleteCacheFlush = false;
1010
1011 public MockHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) throws IOException {
1012 super(fs, rootDir, logName, conf);
1013 }
1014
1015 @Override
1016 public void completeCacheFlush(byte[] encodedRegionName) {
1017 if (!doCompleteCacheFlush) {
1018 return;
1019 }
1020 super.completeCacheFlush(encodedRegionName);
1021 }
1022 }
1023
1024 private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
1025 HTableDescriptor htd = new HTableDescriptor(tableName);
1026 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
1027 htd.addFamily(a);
1028 return htd;
1029 }
1030
1031 private MockHLog createMockWAL(Configuration conf) throws IOException {
1032 MockHLog wal = new MockHLog(FileSystem.get(conf), hbaseRootDir, logName, conf);
1033
1034
1035 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
1036 return wal;
1037 }
1038
1039
1040
1041 class TestFlusher implements FlushRequester {
1042 private HRegion r;
1043
1044 @Override
1045 public void requestFlush(HRegion region) {
1046 try {
1047 r.flushcache();
1048 } catch (IOException e) {
1049 throw new RuntimeException("Exception flushing", e);
1050 }
1051 }
1052
1053 @Override
1054 public void requestDelayedFlush(HRegion region, long when) {
1055
1056
1057 }
1058 }
1059
1060 private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
1061 final byte[] family, final int count, EnvironmentEdge ee, final HLog wal,
1062 final HTableDescriptor htd, final AtomicLong sequenceId)
1063 throws IOException {
1064 String familyStr = Bytes.toString(family);
1065 for (int j = 0; j < count; j++) {
1066 byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
1067 byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
1068 WALEdit edit = new WALEdit();
1069 edit.add(new KeyValue(rowName, family, qualifierBytes,
1070 ee.currentTimeMillis(), columnBytes));
1071 wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd, sequenceId);
1072 }
1073 }
1074
1075 private void addRegionEdits (final byte [] rowName, final byte [] family,
1076 final int count, EnvironmentEdge ee, final HRegion r,
1077 final String qualifierPrefix)
1078 throws IOException {
1079 for (int j = 0; j < count; j++) {
1080 byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
1081 Put p = new Put(rowName);
1082 p.add(family, qualifier, ee.currentTimeMillis(), rowName);
1083 r.put(p);
1084 }
1085 }
1086
1087
1088
1089
1090
1091
1092 private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
1093 return new HRegionInfo(tableName, null, null, false);
1094 }
1095
1096
1097
1098
1099
1100
1101
1102 private Path runWALSplit(final Configuration c) throws IOException {
1103 List<Path> splits = HLogSplitter.split(
1104 hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c);
1105
1106 assertEquals("splits=" + splits, 1, splits.size());
1107
1108 assertTrue(fs.exists(splits.get(0)));
1109 LOG.info("Split file=" + splits.get(0));
1110 return splits.get(0);
1111 }
1112
1113
1114
1115
1116
1117
1118 private HLog createWAL(final Configuration c) throws IOException {
1119 HLog wal = HLogFactory.createHLog(FileSystem.get(c),
1120 hbaseRootDir, logName, c);
1121
1122
1123 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
1124 return wal;
1125 }
1126
1127 private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
1128 HTableDescriptor htd = new HTableDescriptor(tableName);
1129 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
1130 htd.addFamily(a);
1131 HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
1132 htd.addFamily(b);
1133 HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
1134 htd.addFamily(c);
1135 return htd;
1136 }
1137
1138 }
1139