1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23 import static org.junit.Assert.fail;
24
25 import java.io.IOException;
26 import java.security.PrivilegedExceptionAction;
27 import java.util.ArrayList;
28 import java.util.List;
29 import java.util.SortedSet;
30 import java.util.concurrent.atomic.AtomicBoolean;
31 import java.util.concurrent.atomic.AtomicInteger;
32 import java.util.concurrent.atomic.AtomicLong;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.Cell;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HColumnDescriptor;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.KeyValue;
48 import org.apache.hadoop.hbase.MasterNotRunningException;
49 import org.apache.hadoop.hbase.MediumTests;
50 import org.apache.hadoop.hbase.MiniHBaseCluster;
51 import org.apache.hadoop.hbase.ServerName;
52 import org.apache.hadoop.hbase.TableName;
53 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
54 import org.apache.hadoop.hbase.client.Delete;
55 import org.apache.hadoop.hbase.client.Get;
56 import org.apache.hadoop.hbase.client.HTable;
57 import org.apache.hadoop.hbase.client.Put;
58 import org.apache.hadoop.hbase.client.Result;
59 import org.apache.hadoop.hbase.client.ResultScanner;
60 import org.apache.hadoop.hbase.client.Scan;
61 import org.apache.hadoop.hbase.io.hfile.HFile;
62 import org.apache.hadoop.hbase.master.HMaster;
63 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
64 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
65 import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
66 import org.apache.hadoop.hbase.regionserver.FlushRequester;
67 import org.apache.hadoop.hbase.regionserver.HRegion;
68 import org.apache.hadoop.hbase.regionserver.HRegionServer;
69 import org.apache.hadoop.hbase.regionserver.RegionScanner;
70 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
71 import org.apache.hadoop.hbase.regionserver.Store;
72 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
73 import org.apache.hadoop.hbase.security.User;
74 import org.apache.hadoop.hbase.util.Bytes;
75 import org.apache.hadoop.hbase.util.EnvironmentEdge;
76 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
77 import org.apache.hadoop.hbase.util.FSUtils;
78 import org.apache.hadoop.hbase.util.Pair;
79 import org.junit.After;
80 import org.junit.AfterClass;
81 import org.junit.Before;
82 import org.junit.BeforeClass;
83 import org.junit.Test;
84 import org.junit.experimental.categories.Category;
85 import org.mockito.Mockito;
86
87
88
89
90 @Category(MediumTests.class)
91 public class TestWALReplay {
92 public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
93 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
94 private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
95 private Path hbaseRootDir = null;
96 private String logName;
97 private Path oldLogDir;
98 private Path logDir;
99 private FileSystem fs;
100 private Configuration conf;
101
102 @BeforeClass
103 public static void setUpBeforeClass() throws Exception {
104 Configuration conf = TEST_UTIL.getConfiguration();
105 conf.setBoolean("dfs.support.append", true);
106
107 conf.setInt("dfs.client.block.recovery.retries", 2);
108 TEST_UTIL.startMiniCluster(3);
109 Path hbaseRootDir =
110 TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
111 LOG.info("hbase.rootdir=" + hbaseRootDir);
112 FSUtils.setRootDir(conf, hbaseRootDir);
113 }
114
115 @AfterClass
116 public static void tearDownAfterClass() throws Exception {
117 TEST_UTIL.shutdownMiniCluster();
118 }
119
120 @Before
121 public void setUp() throws Exception {
122 this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
123 this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
124 this.hbaseRootDir = FSUtils.getRootDir(this.conf);
125 this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
126 this.logName = HConstants.HREGION_LOGDIR_NAME;
127 this.logDir = new Path(this.hbaseRootDir, logName);
128 if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
129 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
130 }
131 }
132
133 @After
134 public void tearDown() throws Exception {
135 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
136 }
137
138
139
140
141 private void deleteDir(final Path p) throws IOException {
142 if (this.fs.exists(p)) {
143 if (!this.fs.delete(p, true)) {
144 throw new IOException("Failed remove of " + p);
145 }
146 }
147 }
148
149
150
151
152
153 @Test
154 public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
155 final TableName tableName =
156 TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
157 byte[] family1 = Bytes.toBytes("cf1");
158 byte[] family2 = Bytes.toBytes("cf2");
159 byte[] qualifier = Bytes.toBytes("q");
160 byte[] value = Bytes.toBytes("testV");
161 byte[][] familys = { family1, family2 };
162 TEST_UTIL.createTable(tableName, familys);
163 HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
164 Put put = new Put(Bytes.toBytes("r1"));
165 put.add(family1, qualifier, value);
166 htable.put(put);
167 ResultScanner resultScanner = htable.getScanner(new Scan());
168 int count = 0;
169 while (resultScanner.next() != null) {
170 count++;
171 }
172 resultScanner.close();
173 assertEquals(1, count);
174
175 MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
176 List<HRegion> regions = hbaseCluster.getRegions(tableName);
177 assertEquals(1, regions.size());
178
179
180 HRegion destRegion = regions.get(0);
181 int originServerNum = hbaseCluster
182 .getServerWith(destRegion.getRegionName());
183 assertTrue("Please start more than 1 regionserver", hbaseCluster
184 .getRegionServerThreads().size() > 1);
185 int destServerNum = 0;
186 while (destServerNum == originServerNum) {
187 destServerNum++;
188 }
189 HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
190 HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
191
192 moveRegionAndWait(destRegion, destServer);
193
194
195 Delete del = new Delete(Bytes.toBytes("r1"));
196 htable.delete(del);
197 resultScanner = htable.getScanner(new Scan());
198 count = 0;
199 while (resultScanner.next() != null) {
200 count++;
201 }
202 resultScanner.close();
203 assertEquals(0, count);
204
205
206 destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
207
208 for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
209 .getStores().values()) {
210 store.triggerMajorCompaction();
211 }
212 destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
213
214
215 moveRegionAndWait(destRegion, originServer);
216
217 originServer.abort("testing");
218
219
220 Result result = htable.get(new Get(Bytes.toBytes("r1")));
221 if (result != null) {
222 assertTrue("Row is deleted, but we get" + result.toString(),
223 (result == null) || result.isEmpty());
224 }
225 resultScanner.close();
226 }
227
228 private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
229 throws InterruptedException, MasterNotRunningException,
230 ZooKeeperConnectionException, IOException {
231 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
232 TEST_UTIL.getHBaseAdmin().move(
233 destRegion.getRegionInfo().getEncodedNameAsBytes(),
234 Bytes.toBytes(destServer.getServerName().getServerName()));
235 while (true) {
236 ServerName serverName = master.getAssignmentManager()
237 .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
238 if (serverName != null && serverName.equals(destServer.getServerName())) {
239 TEST_UTIL.assertRegionOnServer(
240 destRegion.getRegionInfo(), serverName, 200);
241 break;
242 }
243 Thread.sleep(10);
244 }
245 }
246
247
248
249
250
251
252 @Test
253 public void test2727() throws Exception {
254
255
256 final TableName tableName =
257 TableName.valueOf("test2727");
258 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
259 Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
260 deleteDir(basedir);
261
262 HTableDescriptor htd = createBasic3FamilyHTD(tableName);
263 HRegion region2 = HRegion.createHRegion(hri,
264 hbaseRootDir, this.conf, htd);
265 HRegion.closeHRegion(region2);
266 final byte [] rowName = tableName.getName();
267
268 HLog wal1 = createWAL(this.conf);
269
270 final int countPerFamily = 1000;
271 for (HColumnDescriptor hcd: htd.getFamilies()) {
272 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
273 wal1, htd);
274 }
275 wal1.close();
276 runWALSplit(this.conf);
277
278 HLog wal2 = createWAL(this.conf);
279
280 wal2.setSequenceNumber(wal1.getSequenceNumber());
281
282 for (HColumnDescriptor hcd: htd.getFamilies()) {
283 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
284 ee, wal2, htd);
285 }
286 wal2.close();
287 runWALSplit(this.conf);
288
289 HLog wal3 = createWAL(this.conf);
290 wal3.setSequenceNumber(wal2.getSequenceNumber());
291 try {
292 long wal3SeqId = wal3.getSequenceNumber();
293 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
294 long seqid = region.getOpenSeqNum();
295 assertTrue(seqid > wal3SeqId);
296
297
298 region.close();
299 } finally {
300 wal3.closeAndDelete();
301 }
302 }
303
304
305
306
307
308
309
310
311
312
313 @Test
314 public void testRegionMadeOfBulkLoadedFilesOnly()
315 throws IOException, SecurityException, IllegalArgumentException,
316 NoSuchFieldException, IllegalAccessException, InterruptedException {
317 final TableName tableName =
318 TableName.valueOf("testReplayEditsWrittenViaHRegion");
319 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
320 final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
321 deleteDir(basedir);
322 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
323 HRegion region2 = HRegion.createHRegion(hri,
324 hbaseRootDir, this.conf, htd);
325 HRegion.closeHRegion(region2);
326 HLog wal = createWAL(this.conf);
327 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
328 Path f = new Path(basedir, "hfile");
329 HFile.Writer writer =
330 HFile.getWriterFactoryNoCache(conf).withPath(fs, f).create();
331 byte [] family = htd.getFamilies().iterator().next().getName();
332 byte [] row = tableName.getName();
333 writer.append(new KeyValue(row, family, family, row));
334 writer.close();
335 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
336 hfs.add(Pair.newPair(family, f.toString()));
337 region.bulkLoadHFiles(hfs, true);
338
339 region.put((new Put(row)).add(family, family, family));
340 wal.sync();
341
342
343 final Configuration newConf = HBaseConfiguration.create(this.conf);
344 User user = HBaseTestingUtility.getDifferentUser(newConf,
345 tableName.getNameAsString());
346 user.runAs(new PrivilegedExceptionAction() {
347 public Object run() throws Exception {
348 runWALSplit(newConf);
349 HLog wal2 = createWAL(newConf);
350
351 HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
352 hbaseRootDir, hri, htd, wal2);
353 long seqid2 = region2.getOpenSeqNum();
354 assertTrue(seqid2 > -1);
355
356
357 region2.close();
358 wal2.closeAndDelete();
359 return null;
360 }
361 });
362 }
363
364
365
366
367
368
369
370
371
372
373 @Test
374 public void testReplayEditsWrittenViaHRegion()
375 throws IOException, SecurityException, IllegalArgumentException,
376 NoSuchFieldException, IllegalAccessException, InterruptedException {
377 final TableName tableName =
378 TableName.valueOf("testReplayEditsWrittenViaHRegion");
379 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
380 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
381 deleteDir(basedir);
382 final byte[] rowName = tableName.getName();
383 final int countPerFamily = 10;
384 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
385 HRegion region3 = HRegion.createHRegion(hri,
386 hbaseRootDir, this.conf, htd);
387 HRegion.closeHRegion(region3);
388
389
390
391 HLog wal = createWAL(this.conf);
392 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
393 long seqid = region.getOpenSeqNum();
394
395 wal.setSequenceNumber(seqid);
396 boolean first = true;
397 for (HColumnDescriptor hcd: htd.getFamilies()) {
398 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
399 if (first ) {
400
401 region.flushcache();
402 first = false;
403 }
404 }
405
406 final Get g = new Get(rowName);
407 Result result = region.get(g);
408 assertEquals(countPerFamily * htd.getFamilies().size(),
409 result.size());
410
411
412
413 region.close(true);
414 wal.close();
415 runWALSplit(this.conf);
416 HLog wal2 = createWAL(this.conf);
417 HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
418 long seqid2 = region2.getOpenSeqNum();
419
420 wal2.setSequenceNumber(seqid2);
421 assertTrue(seqid + result.size() < seqid2);
422 final Result result1b = region2.get(g);
423 assertEquals(result.size(), result1b.size());
424
425
426
427
428 for (HColumnDescriptor hcd: htd.getFamilies()) {
429 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
430 }
431
432 final Result result2 = region2.get(g);
433 assertEquals(2 * result.size(), result2.size());
434 wal2.sync();
435
436
437 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal2).getOutputStream(), 1);
438 final Configuration newConf = HBaseConfiguration.create(this.conf);
439 User user = HBaseTestingUtility.getDifferentUser(newConf,
440 tableName.getNameAsString());
441 user.runAs(new PrivilegedExceptionAction() {
442 public Object run() throws Exception {
443 runWALSplit(newConf);
444 FileSystem newFS = FileSystem.get(newConf);
445
446 HLog wal3 = createWAL(newConf);
447 final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
448 HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
449 @Override
450 protected boolean restoreEdit(Store s, KeyValue kv) {
451 boolean b = super.restoreEdit(s, kv);
452 countOfRestoredEdits.incrementAndGet();
453 return b;
454 }
455 };
456 long seqid3 = region3.initialize();
457
458 wal3.setSequenceNumber(seqid3);
459 Result result3 = region3.get(g);
460
461 assertEquals(result2.size(), result3.size());
462 assertEquals(htd.getFamilies().size() * countPerFamily,
463 countOfRestoredEdits.get());
464
465
466 region3.close();
467 wal3.closeAndDelete();
468 return null;
469 }
470 });
471 }
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491 @Test
492 public void testReplayEditsAfterPartialFlush()
493 throws IOException, SecurityException, IllegalArgumentException,
494 NoSuchFieldException, IllegalAccessException, InterruptedException {
495 final TableName tableName =
496 TableName.valueOf("testReplayEditsWrittenViaHRegion");
497 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
498 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
499 deleteDir(basedir);
500 final byte[] rowName = tableName.getName();
501 final int countPerFamily = 10;
502 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
503 HRegion region3 = HRegion.createHRegion(hri,
504 hbaseRootDir, this.conf, htd);
505 HRegion.closeHRegion(region3);
506
507
508
509 HLog wal = createWAL(this.conf);
510 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
511 long seqid = region.getOpenSeqNum();
512
513 wal.setSequenceNumber(seqid);
514 for (HColumnDescriptor hcd: htd.getFamilies()) {
515 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
516 }
517
518
519 final Get g = new Get(rowName);
520 Result result = region.get(g);
521 assertEquals(countPerFamily * htd.getFamilies().size(),
522 result.size());
523
524
525 region.flushcache();
526 region.close(true);
527 wal.close();
528
529
530
531
532
533 int cf_count = 0;
534 for (HColumnDescriptor hcd: htd.getFamilies()) {
535 cf_count++;
536 if (cf_count == 2) {
537 region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
538 }
539 }
540
541
542
543 runWALSplit(this.conf);
544 HLog wal2 = createWAL(this.conf);
545 HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
546 long seqid2 = region2.getOpenSeqNum();
547
548 wal2.setSequenceNumber(seqid2);
549 assertTrue(seqid + result.size() < seqid2);
550
551 final Result result1b = region2.get(g);
552 assertEquals(result.size(), result1b.size());
553 }
554
555
556
557
558 public static class CustomStoreFlusher extends DefaultStoreFlusher {
559
560 static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
561
562 public CustomStoreFlusher(Configuration conf, Store store) {
563 super(conf, store);
564 }
565 @Override
566 public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
567 TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status)
568 throws IOException {
569 if (throwExceptionWhenFlushing.get()) {
570 throw new IOException("Simulated exception by tests");
571 }
572 return super.flushSnapshot(snapshot, cacheFlushId, snapshotTimeRangeTracker,
573 flushedSize, status);
574 }
575
576 };
577
578
579
580
581
582
583
584 @Test
585 public void testReplayEditsAfterAbortingFlush() throws IOException {
586 final TableName tableName =
587 TableName.valueOf("testReplayEditsAfterAbortingFlush");
588 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
589 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
590 deleteDir(basedir);
591 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
592 HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
593 region3.close();
594 region3.getLog().closeAndDelete();
595
596
597
598 HLog wal = createWAL(this.conf);
599 RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
600 Mockito.doReturn(false).when(rsServices).isAborted();
601 Configuration customConf = new Configuration(this.conf);
602 customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
603 CustomStoreFlusher.class.getName());
604 HRegion region = new HRegion(basedir, wal, this.fs, customConf, hri, htd, rsServices);
605 long seqid = region.initialize();
606
607
608 wal.setSequenceNumber(seqid);
609
610 int writtenRowCount = 10;
611 List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
612 htd.getFamilies());
613 for (int i = 0; i < writtenRowCount; i++) {
614 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
615 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
616 Bytes.toBytes("val"));
617 region.put(put);
618 }
619
620
621 RegionScanner scanner = region.getScanner(new Scan());
622 assertEquals(writtenRowCount, getScannedCount(scanner));
623
624
625 CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
626 try {
627 region.flushcache();
628 fail("Injected exception hasn't been thrown");
629 } catch (Throwable t) {
630 LOG.info("Expected simulated exception when flushing region,"
631 + t.getMessage());
632
633 Mockito.doReturn(true).when(rsServices).isAborted();
634 }
635
636 int moreRow = 10;
637 for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
638 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
639 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
640 Bytes.toBytes("val"));
641 region.put(put);
642 }
643 writtenRowCount += moreRow;
644
645 CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
646 try {
647 region.flushcache();
648 } catch (IOException t) {
649 LOG.info("Expected exception when flushing region because server is stopped,"
650 + t.getMessage());
651 }
652
653 region.close(true);
654 wal.close();
655
656
657 runWALSplit(this.conf);
658 HLog wal2 = createWAL(this.conf);
659 Mockito.doReturn(false).when(rsServices).isAborted();
660 HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd,
661 rsServices);
662 long seqid2 = region2.initialize();
663
664
665 wal2.setSequenceNumber(seqid2);
666
667 scanner = region2.getScanner(new Scan());
668 assertEquals(writtenRowCount, getScannedCount(scanner));
669 }
670
671 private int getScannedCount(RegionScanner scanner) throws IOException {
672 int scannedCount = 0;
673 List<Cell> results = new ArrayList<Cell>();
674 while (true) {
675 boolean existMore = scanner.next(results);
676 if (!results.isEmpty())
677 scannedCount++;
678 if (!existMore)
679 break;
680 results.clear();
681 }
682 return scannedCount;
683 }
684
685
686
687
688
689
690 @Test
691 public void testReplayEditsWrittenIntoWAL() throws Exception {
692 final TableName tableName =
693 TableName.valueOf("testReplayEditsWrittenIntoWAL");
694 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
695 final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
696 deleteDir(basedir);
697
698 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
699 HRegion region2 = HRegion.createHRegion(hri,
700 hbaseRootDir, this.conf, htd);
701 HRegion.closeHRegion(region2);
702 final HLog wal = createWAL(this.conf);
703 final byte[] rowName = tableName.getName();
704 final byte[] regionName = hri.getEncodedNameAsBytes();
705
706
707 final int countPerFamily = 1000;
708 for (HColumnDescriptor hcd: htd.getFamilies()) {
709 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
710 ee, wal, htd);
711 }
712
713
714 wal.startCacheFlush(regionName);
715 wal.completeCacheFlush(regionName);
716
717
718 WALEdit edit = new WALEdit();
719 long now = ee.currentTimeMillis();
720 edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
721 now, rowName));
722 wal.append(hri, tableName, edit, now, htd);
723
724
725 edit = new WALEdit();
726 now = ee.currentTimeMillis();
727 edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
728 KeyValue.Type.DeleteFamily));
729 wal.append(hri, tableName, edit, now, htd);
730
731
732 wal.sync();
733
734
735 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
736
737
738 final Configuration newConf = HBaseConfiguration.create(this.conf);
739 User user = HBaseTestingUtility.getDifferentUser(newConf,
740 ".replay.wal.secondtime");
741 user.runAs(new PrivilegedExceptionAction() {
742 public Object run() throws Exception {
743 runWALSplit(newConf);
744 FileSystem newFS = FileSystem.get(newConf);
745
746 newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
747
748 HLog newWal = createWAL(newConf);
749 final AtomicInteger flushcount = new AtomicInteger(0);
750 try {
751 final HRegion region =
752 new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
753 protected boolean internalFlushcache(
754 final HLog wal, final long myseqid, MonitoredTask status)
755 throws IOException {
756 LOG.info("InternalFlushCache Invoked");
757 boolean b = super.internalFlushcache(wal, myseqid,
758 Mockito.mock(MonitoredTask.class));
759 flushcount.incrementAndGet();
760 return b;
761 };
762 };
763 long seqid = region.initialize();
764
765 assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
766 assertTrue(seqid > wal.getSequenceNumber());
767
768 Get get = new Get(rowName);
769 Result result = region.get(get);
770
771 assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
772 result.size());
773 region.close();
774 } finally {
775 newWal.closeAndDelete();
776 }
777 return null;
778 }
779 });
780 }
781
782 @Test
783
784 public void testSequentialEditLogSeqNum() throws IOException {
785 final TableName tableName =
786 TableName.valueOf("testSequentialEditLogSeqNum");
787 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
788 final Path basedir =
789 FSUtils.getTableDir(this.hbaseRootDir, tableName);
790 deleteDir(basedir);
791 final byte[] rowName = tableName.getName();
792 final int countPerFamily = 10;
793 final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
794
795
796 MockHLog wal = createMockWAL(this.conf);
797
798 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
799 long seqid = region.getOpenSeqNum();
800
801
802 wal.setSequenceNumber(seqid);
803 for (HColumnDescriptor hcd : htd.getFamilies()) {
804 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
805 }
806
807 long sequenceNumber = wal.getSequenceNumber();
808
809
810
811 region.flushcache();
812 for (HColumnDescriptor hcd : htd.getFamilies()) {
813 addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
814 }
815 long lastestSeqNumber = wal.getSequenceNumber();
816
817 wal.doCompleteCacheFlush = true;
818
819
820 wal.completeCacheFlush(hri.getEncodedNameAsBytes());
821 wal.close();
822 FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
823 HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0],
824 this.fs, this.conf, null, null, null);
825 FileStatus[] listStatus1 = this.fs.listStatus(
826 new Path(FSUtils.getTableDir(hbaseRootDir, tableName),
827 new Path(hri.getEncodedName(), "recovered.edits")));
828 int editCount = 0;
829 for (FileStatus fileStatus : listStatus1) {
830 editCount = Integer.parseInt(fileStatus.getPath().getName());
831 }
832
833 assertEquals(
834 "The sequence number of the recoverd.edits and the current edit seq should be same",
835 lastestSeqNumber, editCount);
836 }
837
838 static class MockHLog extends FSHLog {
839 boolean doCompleteCacheFlush = false;
840
841 public MockHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) throws IOException {
842 super(fs, rootDir, logName, conf);
843 }
844
845 @Override
846 public void completeCacheFlush(byte[] encodedRegionName) {
847 if (!doCompleteCacheFlush) {
848 return;
849 }
850 super.completeCacheFlush(encodedRegionName);
851 }
852 }
853
854 private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
855 HTableDescriptor htd = new HTableDescriptor(tableName);
856 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
857 htd.addFamily(a);
858 return htd;
859 }
860
861 private MockHLog createMockWAL(Configuration conf) throws IOException {
862 MockHLog wal = new MockHLog(FileSystem.get(conf), hbaseRootDir, logName, conf);
863
864
865 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
866 return wal;
867 }
868
869
870
871 class TestFlusher implements FlushRequester {
872 private HRegion r;
873
874 @Override
875 public void requestFlush(HRegion region) {
876 try {
877 r.flushcache();
878 } catch (IOException e) {
879 throw new RuntimeException("Exception flushing", e);
880 }
881 }
882
883 @Override
884 public void requestDelayedFlush(HRegion region, long when) {
885
886
887 }
888 }
889
890 private void addWALEdits (final TableName tableName, final HRegionInfo hri,
891 final byte [] rowName, final byte [] family,
892 final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd)
893 throws IOException {
894 String familyStr = Bytes.toString(family);
895 for (int j = 0; j < count; j++) {
896 byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
897 byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
898 WALEdit edit = new WALEdit();
899 edit.add(new KeyValue(rowName, family, qualifierBytes,
900 ee.currentTimeMillis(), columnBytes));
901 wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd);
902 }
903 }
904
905 private void addRegionEdits (final byte [] rowName, final byte [] family,
906 final int count, EnvironmentEdge ee, final HRegion r,
907 final String qualifierPrefix)
908 throws IOException {
909 for (int j = 0; j < count; j++) {
910 byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
911 Put p = new Put(rowName);
912 p.add(family, qualifier, ee.currentTimeMillis(), rowName);
913 r.put(p);
914 }
915 }
916
917
918
919
920
921
922 private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
923 return new HRegionInfo(tableName, null, null, false);
924 }
925
926
927
928
929
930
931
932 private Path runWALSplit(final Configuration c) throws IOException {
933 List<Path> splits = HLogSplitter.split(
934 hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c);
935
936 assertEquals("splits=" + splits, 1, splits.size());
937
938 assertTrue(fs.exists(splits.get(0)));
939 LOG.info("Split file=" + splits.get(0));
940 return splits.get(0);
941 }
942
943
944
945
946
947
948 private HLog createWAL(final Configuration c) throws IOException {
949 HLog wal = HLogFactory.createHLog(FileSystem.get(c),
950 hbaseRootDir, logName, c);
951
952
953 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
954 return wal;
955 }
956
957 private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
958 HTableDescriptor htd = new HTableDescriptor(tableName);
959 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
960 htd.addFamily(a);
961 HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
962 htd.addFamily(b);
963 HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
964 htd.addFamily(c);
965 return htd;
966 }
967
968 }
969