1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertNotNull;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25 import static org.mockito.Mockito.when;
26
27 import java.io.IOException;
28 import java.security.PrivilegedExceptionAction;
29 import java.util.ArrayList;
30 import java.util.List;
31 import java.util.concurrent.atomic.AtomicBoolean;
32 import java.util.concurrent.atomic.AtomicInteger;
33 import java.util.concurrent.atomic.AtomicLong;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.fs.PathFilter;
42 import org.apache.hadoop.hbase.Cell;
43 import org.apache.hadoop.hbase.HBaseConfiguration;
44 import org.apache.hadoop.hbase.HBaseTestingUtility;
45 import org.apache.hadoop.hbase.HColumnDescriptor;
46 import org.apache.hadoop.hbase.HConstants;
47 import org.apache.hadoop.hbase.HRegionInfo;
48 import org.apache.hadoop.hbase.HTableDescriptor;
49 import org.apache.hadoop.hbase.KeyValue;
50 import org.apache.hadoop.hbase.MasterNotRunningException;
51 import org.apache.hadoop.hbase.testclassification.MediumTests;
52 import org.apache.hadoop.hbase.MiniHBaseCluster;
53 import org.apache.hadoop.hbase.ServerName;
54 import org.apache.hadoop.hbase.TableName;
55 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
56 import org.apache.hadoop.hbase.client.Delete;
57 import org.apache.hadoop.hbase.client.Get;
58 import org.apache.hadoop.hbase.client.HTable;
59 import org.apache.hadoop.hbase.client.Put;
60 import org.apache.hadoop.hbase.client.Result;
61 import org.apache.hadoop.hbase.client.ResultScanner;
62 import org.apache.hadoop.hbase.client.Scan;
63 import org.apache.hadoop.hbase.client.Table;
64 import org.apache.hadoop.hbase.master.HMaster;
65 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
66 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
67 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
68 import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
69 import org.apache.hadoop.hbase.regionserver.FlushRequestListener;
70 import org.apache.hadoop.hbase.regionserver.FlushRequester;
71 import org.apache.hadoop.hbase.regionserver.HRegion;
72 import org.apache.hadoop.hbase.regionserver.HRegionServer;
73 import org.apache.hadoop.hbase.regionserver.MemStoreSnapshot;
74 import org.apache.hadoop.hbase.regionserver.RegionScanner;
75 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
76 import org.apache.hadoop.hbase.regionserver.Store;
77 import org.apache.hadoop.hbase.security.User;
78 import org.apache.hadoop.hbase.util.Bytes;
79 import org.apache.hadoop.hbase.util.EnvironmentEdge;
80 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
81 import org.apache.hadoop.hbase.util.FSUtils;
82 import org.apache.hadoop.hbase.util.HFileTestUtil;
83 import org.apache.hadoop.hbase.util.Pair;
84 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
85 import org.apache.hadoop.hbase.wal.WAL;
86 import org.apache.hadoop.hbase.wal.WALKey;
87 import org.apache.hadoop.hbase.wal.WALFactory;
88 import org.apache.hadoop.hbase.wal.WALSplitter;
89 import org.junit.After;
90 import org.junit.AfterClass;
91 import org.junit.Before;
92 import org.junit.BeforeClass;
93 import org.junit.Test;
94 import org.junit.Rule;
95 import org.junit.rules.TestName;
96 import org.junit.experimental.categories.Category;
97 import org.mockito.Mockito;
98
99
100
101
102 @Category(MediumTests.class)
103 public class TestWALReplay {
104 public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
105 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
106 private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
107 private Path hbaseRootDir = null;
108 private String logName;
109 private Path oldLogDir;
110 private Path logDir;
111 private FileSystem fs;
112 private Configuration conf;
113 private RecoveryMode mode;
114 private WALFactory wals;
115
116 @Rule
117 public final TestName currentTest = new TestName();
118
119
120 @BeforeClass
121 public static void setUpBeforeClass() throws Exception {
122 Configuration conf = TEST_UTIL.getConfiguration();
123 conf.setBoolean("dfs.support.append", true);
124
125 conf.setInt("dfs.client.block.recovery.retries", 2);
126 TEST_UTIL.startMiniCluster(3);
127 Path hbaseRootDir =
128 TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
129 LOG.info("hbase.rootdir=" + hbaseRootDir);
130 FSUtils.setRootDir(conf, hbaseRootDir);
131 }
132
133 @AfterClass
134 public static void tearDownAfterClass() throws Exception {
135 TEST_UTIL.shutdownMiniCluster();
136 }
137
138 @Before
139 public void setUp() throws Exception {
140 this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
141 this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
142 this.hbaseRootDir = FSUtils.getRootDir(this.conf);
143 this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
144 this.logName = DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName() + "-manual");
145 this.logDir = new Path(this.hbaseRootDir, logName);
146 if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
147 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
148 }
149 this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
150 RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
151 this.wals = new WALFactory(conf, null, currentTest.getMethodName());
152 }
153
154 @After
155 public void tearDown() throws Exception {
156 this.wals.close();
157 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
158 }
159
160
161
162
163 private void deleteDir(final Path p) throws IOException {
164 if (this.fs.exists(p)) {
165 if (!this.fs.delete(p, true)) {
166 throw new IOException("Failed remove of " + p);
167 }
168 }
169 }
170
171
172
173
174
175 @Test
176 public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
177 final TableName tableName =
178 TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
179 byte[] family1 = Bytes.toBytes("cf1");
180 byte[] family2 = Bytes.toBytes("cf2");
181 byte[] qualifier = Bytes.toBytes("q");
182 byte[] value = Bytes.toBytes("testV");
183 byte[][] familys = { family1, family2 };
184 TEST_UTIL.createTable(tableName, familys);
185 Table htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
186 Put put = new Put(Bytes.toBytes("r1"));
187 put.add(family1, qualifier, value);
188 htable.put(put);
189 ResultScanner resultScanner = htable.getScanner(new Scan());
190 int count = 0;
191 while (resultScanner.next() != null) {
192 count++;
193 }
194 resultScanner.close();
195 assertEquals(1, count);
196
197 MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
198 List<HRegion> regions = hbaseCluster.getRegions(tableName);
199 assertEquals(1, regions.size());
200
201
202 HRegion destRegion = regions.get(0);
203 int originServerNum = hbaseCluster
204 .getServerWith(destRegion.getRegionName());
205 assertTrue("Please start more than 1 regionserver", hbaseCluster
206 .getRegionServerThreads().size() > 1);
207 int destServerNum = 0;
208 while (destServerNum == originServerNum) {
209 destServerNum++;
210 }
211 HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
212 HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
213
214 moveRegionAndWait(destRegion, destServer);
215
216
217 Delete del = new Delete(Bytes.toBytes("r1"));
218 htable.delete(del);
219 resultScanner = htable.getScanner(new Scan());
220 count = 0;
221 while (resultScanner.next() != null) {
222 count++;
223 }
224 resultScanner.close();
225 assertEquals(0, count);
226
227
228 destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
229
230 for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
231 .getStores().values()) {
232 store.triggerMajorCompaction();
233 }
234 destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
235
236
237 moveRegionAndWait(destRegion, originServer);
238
239 originServer.abort("testing");
240
241
242 Result result = htable.get(new Get(Bytes.toBytes("r1")));
243 if (result != null) {
244 assertTrue("Row is deleted, but we get" + result.toString(),
245 (result == null) || result.isEmpty());
246 }
247 resultScanner.close();
248 }
249
250 private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
251 throws InterruptedException, MasterNotRunningException,
252 ZooKeeperConnectionException, IOException {
253 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
254 TEST_UTIL.getHBaseAdmin().move(
255 destRegion.getRegionInfo().getEncodedNameAsBytes(),
256 Bytes.toBytes(destServer.getServerName().getServerName()));
257 while (true) {
258 ServerName serverName = master.getAssignmentManager()
259 .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
260 if (serverName != null && serverName.equals(destServer.getServerName())) {
261 TEST_UTIL.assertRegionOnServer(
262 destRegion.getRegionInfo(), serverName, 200);
263 break;
264 }
265 Thread.sleep(10);
266 }
267 }
268
269
270
271
272
273
274 @Test
275 public void test2727() throws Exception {
276
277
278 final TableName tableName =
279 TableName.valueOf("test2727");
280 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
281 Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
282 deleteDir(basedir);
283
284 HTableDescriptor htd = createBasic3FamilyHTD(tableName);
285 HRegion region2 = HRegion.createHRegion(hri,
286 hbaseRootDir, this.conf, htd);
287 HRegion.closeHRegion(region2);
288 final byte [] rowName = tableName.getName();
289
290 WAL wal1 = createWAL(this.conf);
291
292 final int countPerFamily = 1000;
293 final AtomicLong sequenceId = new AtomicLong(1);
294 for (HColumnDescriptor hcd: htd.getFamilies()) {
295 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
296 wal1, htd, sequenceId);
297 }
298 wal1.shutdown();
299 runWALSplit(this.conf);
300
301 WAL wal2 = createWAL(this.conf);
302
303 for (HColumnDescriptor hcd: htd.getFamilies()) {
304 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
305 ee, wal2, htd, sequenceId);
306 }
307 wal2.shutdown();
308 runWALSplit(this.conf);
309
310 WAL wal3 = createWAL(this.conf);
311 try {
312 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
313 long seqid = region.getOpenSeqNum();
314
315
316 assertTrue(seqid > sequenceId.get());
317 assertEquals(seqid - 1, sequenceId.get());
318 LOG.debug("region.getOpenSeqNum(): " + region.getOpenSeqNum() + ", wal3.id: "
319 + sequenceId.get());
320
321
322 region.close();
323 } finally {
324 wal3.close();
325 }
326 }
327
328
329
330
331
332
333
334
335
336
337 @Test
338 public void testRegionMadeOfBulkLoadedFilesOnly()
339 throws IOException, SecurityException, IllegalArgumentException,
340 NoSuchFieldException, IllegalAccessException, InterruptedException {
341 final TableName tableName =
342 TableName.valueOf("testRegionMadeOfBulkLoadedFilesOnly");
343 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
344 final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
345 deleteDir(basedir);
346 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
347 HRegion region2 = HRegion.createHRegion(hri,
348 hbaseRootDir, this.conf, htd);
349 HRegion.closeHRegion(region2);
350 WAL wal = createWAL(this.conf);
351 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
352
353 byte [] family = htd.getFamilies().iterator().next().getName();
354 Path f = new Path(basedir, "hfile");
355 HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""),
356 Bytes.toBytes("z"), 10);
357 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
358 hfs.add(Pair.newPair(family, f.toString()));
359 region.bulkLoadHFiles(hfs, true);
360
361
362 byte [] row = tableName.getName();
363 region.put((new Put(row)).add(family, family, family));
364 wal.sync();
365 final int rowsInsertedCount = 11;
366
367 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
368
369
370 final Configuration newConf = HBaseConfiguration.create(this.conf);
371 User user = HBaseTestingUtility.getDifferentUser(newConf,
372 tableName.getNameAsString());
373 user.runAs(new PrivilegedExceptionAction() {
374 @Override
375 public Object run() throws Exception {
376 runWALSplit(newConf);
377 WAL wal2 = createWAL(newConf);
378
379 HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
380 hbaseRootDir, hri, htd, wal2);
381 long seqid2 = region2.getOpenSeqNum();
382 assertTrue(seqid2 > -1);
383 assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
384
385
386 region2.close();
387 wal2.close();
388 return null;
389 }
390 });
391 }
392
393
394
395
396
397
398
399
400
401
402
403
404 @Test
405 public void testCompactedBulkLoadedFiles()
406 throws IOException, SecurityException, IllegalArgumentException,
407 NoSuchFieldException, IllegalAccessException, InterruptedException {
408 final TableName tableName =
409 TableName.valueOf("testCompactedBulkLoadedFiles");
410 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
411 final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
412 deleteDir(basedir);
413 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
414 HRegion region2 = HRegion.createHRegion(hri,
415 hbaseRootDir, this.conf, htd);
416 HRegion.closeHRegion(region2);
417 WAL wal = createWAL(this.conf);
418 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
419
420
421 byte [] row = tableName.getName();
422 byte [] family = htd.getFamilies().iterator().next().getName();
423 region.put((new Put(row)).add(family, family, family));
424 wal.sync();
425
426 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
427 for (int i = 0; i < 3; i++) {
428 Path f = new Path(basedir, "hfile"+i);
429 HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(i + "00"),
430 Bytes.toBytes(i + "50"), 10);
431 hfs.add(Pair.newPair(family, f.toString()));
432 }
433 region.bulkLoadHFiles(hfs, true);
434 final int rowsInsertedCount = 31;
435 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
436
437
438 region.compactStores(true);
439 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
440
441
442 final Configuration newConf = HBaseConfiguration.create(this.conf);
443 User user = HBaseTestingUtility.getDifferentUser(newConf,
444 tableName.getNameAsString());
445 user.runAs(new PrivilegedExceptionAction() {
446 @Override
447 public Object run() throws Exception {
448 runWALSplit(newConf);
449 WAL wal2 = createWAL(newConf);
450
451 HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
452 hbaseRootDir, hri, htd, wal2);
453 long seqid2 = region2.getOpenSeqNum();
454 assertTrue(seqid2 > -1);
455 assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
456
457
458 region2.close();
459 wal2.close();
460 return null;
461 }
462 });
463 }
464
465
466
467
468
469
470
471
472
473
474
475 @Test
476 public void testReplayEditsWrittenViaHRegion()
477 throws IOException, SecurityException, IllegalArgumentException,
478 NoSuchFieldException, IllegalAccessException, InterruptedException {
479 final TableName tableName =
480 TableName.valueOf("testReplayEditsWrittenViaHRegion");
481 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
482 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
483 deleteDir(basedir);
484 final byte[] rowName = tableName.getName();
485 final int countPerFamily = 10;
486 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
487 HRegion region3 = HRegion.createHRegion(hri,
488 hbaseRootDir, this.conf, htd);
489 HRegion.closeHRegion(region3);
490
491
492
493 WAL wal = createWAL(this.conf);
494 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
495 long seqid = region.getOpenSeqNum();
496 boolean first = true;
497 for (HColumnDescriptor hcd: htd.getFamilies()) {
498 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
499 if (first ) {
500
501 region.flushcache();
502 first = false;
503 }
504 }
505
506 final Get g = new Get(rowName);
507 Result result = region.get(g);
508 assertEquals(countPerFamily * htd.getFamilies().size(),
509 result.size());
510
511
512
513 region.close(true);
514 wal.shutdown();
515 runWALSplit(this.conf);
516 WAL wal2 = createWAL(this.conf);
517 HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
518 long seqid2 = region2.getOpenSeqNum();
519 assertTrue(seqid + result.size() < seqid2);
520 final Result result1b = region2.get(g);
521 assertEquals(result.size(), result1b.size());
522
523
524
525
526 for (HColumnDescriptor hcd: htd.getFamilies()) {
527 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
528 }
529
530 final Result result2 = region2.get(g);
531 assertEquals(2 * result.size(), result2.size());
532 wal2.sync();
533 final Configuration newConf = HBaseConfiguration.create(this.conf);
534 User user = HBaseTestingUtility.getDifferentUser(newConf,
535 tableName.getNameAsString());
536 user.runAs(new PrivilegedExceptionAction() {
537 @Override
538 public Object run() throws Exception {
539 runWALSplit(newConf);
540 FileSystem newFS = FileSystem.get(newConf);
541
542 WAL wal3 = createWAL(newConf);
543 final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
544 HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
545 @Override
546 protected boolean restoreEdit(Store s, Cell cell) {
547 boolean b = super.restoreEdit(s, cell);
548 countOfRestoredEdits.incrementAndGet();
549 return b;
550 }
551 };
552 long seqid3 = region3.initialize();
553 Result result3 = region3.get(g);
554
555 assertEquals(result2.size(), result3.size());
556 assertEquals(htd.getFamilies().size() * countPerFamily,
557 countOfRestoredEdits.get());
558
559
560 region3.close();
561 wal3.close();
562 return null;
563 }
564 });
565 }
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585 @Test
586 public void testReplayEditsAfterPartialFlush()
587 throws IOException, SecurityException, IllegalArgumentException,
588 NoSuchFieldException, IllegalAccessException, InterruptedException {
589 final TableName tableName =
590 TableName.valueOf("testReplayEditsWrittenViaHRegion");
591 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
592 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
593 deleteDir(basedir);
594 final byte[] rowName = tableName.getName();
595 final int countPerFamily = 10;
596 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
597 HRegion region3 = HRegion.createHRegion(hri,
598 hbaseRootDir, this.conf, htd);
599 HRegion.closeHRegion(region3);
600
601
602
603 WAL wal = createWAL(this.conf);
604 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
605 long seqid = region.getOpenSeqNum();
606 for (HColumnDescriptor hcd: htd.getFamilies()) {
607 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
608 }
609
610
611 final Get g = new Get(rowName);
612 Result result = region.get(g);
613 assertEquals(countPerFamily * htd.getFamilies().size(),
614 result.size());
615
616
617 region.flushcache();
618 region.close(true);
619 wal.shutdown();
620
621
622
623
624
625 int cf_count = 0;
626 for (HColumnDescriptor hcd: htd.getFamilies()) {
627 cf_count++;
628 if (cf_count == 2) {
629 region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
630 }
631 }
632
633
634
635 runWALSplit(this.conf);
636 WAL wal2 = createWAL(this.conf);
637 HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
638 long seqid2 = region2.getOpenSeqNum();
639 assertTrue(seqid + result.size() < seqid2);
640
641 final Result result1b = region2.get(g);
642 assertEquals(result.size(), result1b.size());
643 }
644
645
646
647
648 public static class CustomStoreFlusher extends DefaultStoreFlusher {
649
650 static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
651
652 public CustomStoreFlusher(Configuration conf, Store store) {
653 super(conf, store);
654 }
655 @Override
656 public List<Path> flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushId,
657 MonitoredTask status) throws IOException {
658 if (throwExceptionWhenFlushing.get()) {
659 throw new IOException("Simulated exception by tests");
660 }
661 return super.flushSnapshot(snapshot, cacheFlushId, status);
662 }
663
664 };
665
666
667
668
669
670
671
672 @Test
673 public void testReplayEditsAfterAbortingFlush() throws IOException {
674 final TableName tableName =
675 TableName.valueOf("testReplayEditsAfterAbortingFlush");
676 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
677 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
678 deleteDir(basedir);
679 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
680 HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
681 region3.close();
682 region3.getWAL().close();
683
684
685
686 WAL wal = createWAL(this.conf);
687 RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
688 Mockito.doReturn(false).when(rsServices).isAborted();
689 when(rsServices.getServerName()).thenReturn(ServerName.valueOf("foo", 10, 10));
690 Configuration customConf = new Configuration(this.conf);
691 customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
692 CustomStoreFlusher.class.getName());
693 HRegion region =
694 HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
695 int writtenRowCount = 10;
696 List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
697 htd.getFamilies());
698 for (int i = 0; i < writtenRowCount; i++) {
699 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
700 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
701 Bytes.toBytes("val"));
702 region.put(put);
703 }
704
705
706 RegionScanner scanner = region.getScanner(new Scan());
707 assertEquals(writtenRowCount, getScannedCount(scanner));
708
709
710 CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
711 try {
712 region.flushcache();
713 fail("Injected exception hasn't been thrown");
714 } catch (Throwable t) {
715 LOG.info("Expected simulated exception when flushing region,"
716 + t.getMessage());
717
718 Mockito.doReturn(true).when(rsServices).isAborted();
719 }
720
721 int moreRow = 10;
722 for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
723 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
724 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
725 Bytes.toBytes("val"));
726 region.put(put);
727 }
728 writtenRowCount += moreRow;
729
730 CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
731 try {
732 region.flushcache();
733 } catch (IOException t) {
734 LOG.info("Expected exception when flushing region because server is stopped,"
735 + t.getMessage());
736 }
737
738 region.close(true);
739 wal.shutdown();
740
741
742 runWALSplit(this.conf);
743 WAL wal2 = createWAL(this.conf);
744 Mockito.doReturn(false).when(rsServices).isAborted();
745 HRegion region2 =
746 HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
747 scanner = region2.getScanner(new Scan());
748 assertEquals(writtenRowCount, getScannedCount(scanner));
749 }
750
751 private int getScannedCount(RegionScanner scanner) throws IOException {
752 int scannedCount = 0;
753 List<Cell> results = new ArrayList<Cell>();
754 while (true) {
755 boolean existMore = scanner.next(results);
756 if (!results.isEmpty())
757 scannedCount++;
758 if (!existMore)
759 break;
760 results.clear();
761 }
762 return scannedCount;
763 }
764
765
766
767
768
769
770 @Test
771 public void testReplayEditsWrittenIntoWAL() throws Exception {
772 final TableName tableName =
773 TableName.valueOf("testReplayEditsWrittenIntoWAL");
774 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
775 final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
776 deleteDir(basedir);
777
778 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
779 HRegion region2 = HRegion.createHRegion(hri,
780 hbaseRootDir, this.conf, htd);
781 HRegion.closeHRegion(region2);
782 final WAL wal = createWAL(this.conf);
783 final byte[] rowName = tableName.getName();
784 final byte[] regionName = hri.getEncodedNameAsBytes();
785 final AtomicLong sequenceId = new AtomicLong(1);
786
787
788 final int countPerFamily = 1000;
789 for (HColumnDescriptor hcd: htd.getFamilies()) {
790 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
791 ee, wal, htd, sequenceId);
792 }
793
794
795 wal.startCacheFlush(regionName);
796 wal.completeCacheFlush(regionName);
797
798
799 WALEdit edit = new WALEdit();
800 long now = ee.currentTime();
801 edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
802 now, rowName));
803 wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now), edit, sequenceId,
804 true, null);
805
806
807 edit = new WALEdit();
808 now = ee.currentTime();
809 edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
810 KeyValue.Type.DeleteFamily));
811 wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now), edit, sequenceId,
812 true, null);
813
814
815 wal.sync();
816
817
818 final Configuration newConf = HBaseConfiguration.create(this.conf);
819 User user = HBaseTestingUtility.getDifferentUser(newConf,
820 ".replay.wal.secondtime");
821 user.runAs(new PrivilegedExceptionAction() {
822 @Override
823 public Object run() throws Exception {
824 runWALSplit(newConf);
825 FileSystem newFS = FileSystem.get(newConf);
826
827 newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
828
829 WAL newWal = createWAL(newConf);
830 final AtomicInteger flushcount = new AtomicInteger(0);
831 try {
832 final HRegion region =
833 new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
834 @Override
835 protected FlushResult internalFlushcache(
836 final WAL wal, final long myseqid, MonitoredTask status)
837 throws IOException {
838 LOG.info("InternalFlushCache Invoked");
839 FlushResult fs = super.internalFlushcache(wal, myseqid,
840 Mockito.mock(MonitoredTask.class));
841 flushcount.incrementAndGet();
842 return fs;
843 };
844 };
845 long seqid = region.initialize();
846
847 assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
848 assertTrue(seqid - 1 == sequenceId.get());
849
850 Get get = new Get(rowName);
851 Result result = region.get(get);
852
853 assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
854 result.size());
855 region.close();
856 } finally {
857 newWal.close();
858 }
859 return null;
860 }
861 });
862 }
863
864 @Test
865
866 public void testSequentialEditLogSeqNum() throws IOException {
867 final TableName tableName = TableName.valueOf(currentTest.getMethodName());
868 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
869 final Path basedir =
870 FSUtils.getTableDir(this.hbaseRootDir, tableName);
871 deleteDir(basedir);
872 final byte[] rowName = tableName.getName();
873 final int countPerFamily = 10;
874 final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
875
876
877 MockWAL wal = createMockWAL();
878
879 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
880 for (HColumnDescriptor hcd : htd.getFamilies()) {
881 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
882 }
883
884
885
886 region.flushcache();
887 for (HColumnDescriptor hcd : htd.getFamilies()) {
888 addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
889 }
890 long lastestSeqNumber = region.getSequenceId().get();
891
892 wal.doCompleteCacheFlush = true;
893
894
895 wal.completeCacheFlush(hri.getEncodedNameAsBytes());
896 wal.shutdown();
897 FileStatus[] listStatus = wal.getFiles();
898 assertNotNull(listStatus);
899 assertTrue(listStatus.length > 0);
900 WALSplitter.splitLogFile(hbaseRootDir, listStatus[0],
901 this.fs, this.conf, null, null, null, mode, wals);
902 FileStatus[] listStatus1 = this.fs.listStatus(
903 new Path(FSUtils.getTableDir(hbaseRootDir, tableName), new Path(hri.getEncodedName(),
904 "recovered.edits")), new PathFilter() {
905 @Override
906 public boolean accept(Path p) {
907 if (WALSplitter.isSequenceIdFile(p)) {
908 return false;
909 }
910 return true;
911 }
912 });
913 int editCount = 0;
914 for (FileStatus fileStatus : listStatus1) {
915 editCount = Integer.parseInt(fileStatus.getPath().getName());
916 }
917
918 assertEquals(
919 "The sequence number of the recoverd.edits and the current edit seq should be same",
920 lastestSeqNumber, editCount);
921 }
922
923 static class MockWAL extends FSHLog {
924 boolean doCompleteCacheFlush = false;
925
926 public MockWAL(FileSystem fs, Path rootDir, String logName, Configuration conf)
927 throws IOException {
928 super(fs, rootDir, logName, HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null);
929 }
930
931 @Override
932 public void completeCacheFlush(byte[] encodedRegionName) {
933 if (!doCompleteCacheFlush) {
934 return;
935 }
936 super.completeCacheFlush(encodedRegionName);
937 }
938 }
939
940 private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
941 HTableDescriptor htd = new HTableDescriptor(tableName);
942 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
943 htd.addFamily(a);
944 return htd;
945 }
946
947 private MockWAL createMockWAL() throws IOException {
948 MockWAL wal = new MockWAL(fs, hbaseRootDir, logName, conf);
949
950
951 HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
952 return wal;
953 }
954
955
956
957 class TestFlusher implements FlushRequester {
958 private HRegion r;
959
960 @Override
961 public void requestFlush(HRegion region) {
962 try {
963 r.flushcache();
964 } catch (IOException e) {
965 throw new RuntimeException("Exception flushing", e);
966 }
967 }
968
969 @Override
970 public void requestDelayedFlush(HRegion region, long when) {
971
972
973 }
974
975 @Override
976 public void registerFlushRequestListener(FlushRequestListener listener) {
977
978 }
979
980 @Override
981 public boolean unregisterFlushRequestListener(FlushRequestListener listener) {
982 return false;
983 }
984
985 @Override
986 public void setGlobalMemstoreLimit(long globalMemStoreSize) {
987
988 }
989 }
990
991 private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
992 final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
993 final HTableDescriptor htd, final AtomicLong sequenceId)
994 throws IOException {
995 String familyStr = Bytes.toString(family);
996 for (int j = 0; j < count; j++) {
997 byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
998 byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
999 WALEdit edit = new WALEdit();
1000 edit.add(new KeyValue(rowName, family, qualifierBytes,
1001 ee.currentTime(), columnBytes));
1002 wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, ee.currentTime()),
1003 edit, sequenceId, true, null);
1004 }
1005 wal.sync();
1006 }
1007
1008 static List<Put> addRegionEdits (final byte [] rowName, final byte [] family,
1009 final int count, EnvironmentEdge ee, final HRegion r,
1010 final String qualifierPrefix)
1011 throws IOException {
1012 List<Put> puts = new ArrayList<Put>();
1013 for (int j = 0; j < count; j++) {
1014 byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
1015 Put p = new Put(rowName);
1016 p.add(family, qualifier, ee.currentTime(), rowName);
1017 r.put(p);
1018 puts.add(p);
1019 }
1020 return puts;
1021 }
1022
1023
1024
1025
1026
1027
1028 private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
1029 return new HRegionInfo(tableName, null, null, false);
1030 }
1031
1032
1033
1034
1035
1036
1037
1038 private Path runWALSplit(final Configuration c) throws IOException {
1039 List<Path> splits = WALSplitter.split(
1040 hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c, wals);
1041
1042 assertEquals("splits=" + splits, 1, splits.size());
1043
1044 assertTrue(fs.exists(splits.get(0)));
1045 LOG.info("Split file=" + splits.get(0));
1046 return splits.get(0);
1047 }
1048
1049
1050
1051
1052
1053
1054 private WAL createWAL(final Configuration c) throws IOException {
1055 FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c);
1056
1057
1058 HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
1059 return wal;
1060 }
1061
1062 private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
1063 HTableDescriptor htd = new HTableDescriptor(tableName);
1064 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
1065 htd.addFamily(a);
1066 HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
1067 htd.addFamily(b);
1068 HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
1069 htd.addFamily(c);
1070 return htd;
1071 }
1072 }