1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23 import static org.junit.Assert.fail;
24
25 import java.io.IOException;
26 import java.security.PrivilegedExceptionAction;
27 import java.util.ArrayList;
28 import java.util.List;
29 import java.util.SortedSet;
30 import java.util.concurrent.atomic.AtomicBoolean;
31 import java.util.concurrent.atomic.AtomicInteger;
32 import java.util.concurrent.atomic.AtomicLong;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.TableName;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HColumnDescriptor;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.KeyValue;
48 import org.apache.hadoop.hbase.MasterNotRunningException;
49 import org.apache.hadoop.hbase.MediumTests;
50 import org.apache.hadoop.hbase.MiniHBaseCluster;
51 import org.apache.hadoop.hbase.ServerName;
52 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
53 import org.apache.hadoop.hbase.client.Delete;
54 import org.apache.hadoop.hbase.client.Get;
55 import org.apache.hadoop.hbase.client.HTable;
56 import org.apache.hadoop.hbase.client.Put;
57 import org.apache.hadoop.hbase.client.Result;
58 import org.apache.hadoop.hbase.client.ResultScanner;
59 import org.apache.hadoop.hbase.client.Scan;
60 import org.apache.hadoop.hbase.io.hfile.HFile;
61 import org.apache.hadoop.hbase.master.HMaster;
62 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
63 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
64 import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
65 import org.apache.hadoop.hbase.regionserver.FlushRequester;
66 import org.apache.hadoop.hbase.regionserver.HRegion;
67 import org.apache.hadoop.hbase.regionserver.HRegionServer;
68 import org.apache.hadoop.hbase.regionserver.RegionScanner;
69 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
70 import org.apache.hadoop.hbase.regionserver.Store;
71 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
72 import org.apache.hadoop.hbase.security.User;
73 import org.apache.hadoop.hbase.util.Bytes;
74 import org.apache.hadoop.hbase.util.FSUtils;
75 import org.apache.hadoop.hbase.util.EnvironmentEdge;
76 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
77 import org.apache.hadoop.hbase.util.Pair;
78 import org.junit.After;
79 import org.junit.AfterClass;
80 import org.junit.Before;
81 import org.junit.BeforeClass;
82 import org.junit.Test;
83 import org.junit.experimental.categories.Category;
84 import org.mockito.Mockito;
85
86
87
88
89 @Category(MediumTests.class)
90 public class TestWALReplay {
91 public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
92 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
93 private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
94 private Path hbaseRootDir = null;
95 private String logName;
96 private Path oldLogDir;
97 private Path logDir;
98 private FileSystem fs;
99 private Configuration conf;
100
101 @BeforeClass
102 public static void setUpBeforeClass() throws Exception {
103 Configuration conf = TEST_UTIL.getConfiguration();
104 conf.setBoolean("dfs.support.append", true);
105
106 conf.setInt("dfs.client.block.recovery.retries", 2);
107 TEST_UTIL.startMiniCluster(3);
108 Path hbaseRootDir =
109 TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
110 LOG.info("hbase.rootdir=" + hbaseRootDir);
111 FSUtils.setRootDir(conf, hbaseRootDir);
112 }
113
114 @AfterClass
115 public static void tearDownAfterClass() throws Exception {
116 TEST_UTIL.shutdownMiniCluster();
117 }
118
119 @Before
120 public void setUp() throws Exception {
121 this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
122 this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
123 this.hbaseRootDir = FSUtils.getRootDir(this.conf);
124 this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
125 this.logName = HConstants.HREGION_LOGDIR_NAME;
126 this.logDir = new Path(this.hbaseRootDir, logName);
127 if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
128 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
129 }
130 }
131
132 @After
133 public void tearDown() throws Exception {
134 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
135 }
136
137
138
139
140 private void deleteDir(final Path p) throws IOException {
141 if (this.fs.exists(p)) {
142 if (!this.fs.delete(p, true)) {
143 throw new IOException("Failed remove of " + p);
144 }
145 }
146 }
147
148
149
150
151
152 @Test
153 public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
154 final TableName tableName =
155 TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
156 byte[] family1 = Bytes.toBytes("cf1");
157 byte[] family2 = Bytes.toBytes("cf2");
158 byte[] qualifier = Bytes.toBytes("q");
159 byte[] value = Bytes.toBytes("testV");
160 byte[][] familys = { family1, family2 };
161 TEST_UTIL.createTable(tableName, familys);
162 HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
163 Put put = new Put(Bytes.toBytes("r1"));
164 put.add(family1, qualifier, value);
165 htable.put(put);
166 ResultScanner resultScanner = htable.getScanner(new Scan());
167 int count = 0;
168 while (resultScanner.next() != null) {
169 count++;
170 }
171 resultScanner.close();
172 assertEquals(1, count);
173
174 MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
175 List<HRegion> regions = hbaseCluster.getRegions(tableName);
176 assertEquals(1, regions.size());
177
178
179 HRegion destRegion = regions.get(0);
180 int originServerNum = hbaseCluster
181 .getServerWith(destRegion.getRegionName());
182 assertTrue("Please start more than 1 regionserver", hbaseCluster
183 .getRegionServerThreads().size() > 1);
184 int destServerNum = 0;
185 while (destServerNum == originServerNum) {
186 destServerNum++;
187 }
188 HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
189 HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
190
191 moveRegionAndWait(destRegion, destServer);
192
193
194 Delete del = new Delete(Bytes.toBytes("r1"));
195 htable.delete(del);
196 resultScanner = htable.getScanner(new Scan());
197 count = 0;
198 while (resultScanner.next() != null) {
199 count++;
200 }
201 resultScanner.close();
202 assertEquals(0, count);
203
204
205 destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
206
207 for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
208 .getStores().values()) {
209 store.triggerMajorCompaction();
210 }
211 destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
212
213
214 moveRegionAndWait(destRegion, originServer);
215
216 originServer.abort("testing");
217
218
219 Result result = htable.get(new Get(Bytes.toBytes("r1")));
220 if (result != null) {
221 assertTrue("Row is deleted, but we get" + result.toString(),
222 (result == null) || result.isEmpty());
223 }
224 resultScanner.close();
225 }
226
227 private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
228 throws InterruptedException, MasterNotRunningException,
229 ZooKeeperConnectionException, IOException {
230 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
231 TEST_UTIL.getHBaseAdmin().move(
232 destRegion.getRegionInfo().getEncodedNameAsBytes(),
233 Bytes.toBytes(destServer.getServerName().getServerName()));
234 while (true) {
235 ServerName serverName = master.getAssignmentManager()
236 .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
237 if (serverName != null && serverName.equals(destServer.getServerName())) {
238 TEST_UTIL.assertRegionOnServer(
239 destRegion.getRegionInfo(), serverName, 200);
240 break;
241 }
242 Thread.sleep(10);
243 }
244 }
245
246
247
248
249
250
251 @Test
252 public void test2727() throws Exception {
253
254
255 final TableName tableName =
256 TableName.valueOf("test2727");
257 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
258 Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
259 deleteDir(basedir);
260
261 HTableDescriptor htd = createBasic3FamilyHTD(tableName);
262 HRegion region2 = HRegion.createHRegion(hri,
263 hbaseRootDir, this.conf, htd);
264 HRegion.closeHRegion(region2);
265 final byte [] rowName = tableName.getName();
266
267 HLog wal1 = createWAL(this.conf);
268
269 final int countPerFamily = 1000;
270 for (HColumnDescriptor hcd: htd.getFamilies()) {
271 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
272 wal1, htd);
273 }
274 wal1.close();
275 runWALSplit(this.conf);
276
277 HLog wal2 = createWAL(this.conf);
278
279 wal2.setSequenceNumber(wal1.getSequenceNumber());
280
281 for (HColumnDescriptor hcd: htd.getFamilies()) {
282 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
283 ee, wal2, htd);
284 }
285 wal2.close();
286 runWALSplit(this.conf);
287
288 HLog wal3 = createWAL(this.conf);
289 wal3.setSequenceNumber(wal2.getSequenceNumber());
290 try {
291 long wal3SeqId = wal3.getSequenceNumber();
292 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
293 long seqid = region.getOpenSeqNum();
294 assertTrue(seqid > wal3SeqId);
295
296
297 region.close();
298 } finally {
299 wal3.closeAndDelete();
300 }
301 }
302
303
304
305
306
307
308
309
310
311
312 @Test
313 public void testRegionMadeOfBulkLoadedFilesOnly()
314 throws IOException, SecurityException, IllegalArgumentException,
315 NoSuchFieldException, IllegalAccessException, InterruptedException {
316 final TableName tableName =
317 TableName.valueOf("testReplayEditsWrittenViaHRegion");
318 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
319 final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
320 deleteDir(basedir);
321 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
322 HRegion region2 = HRegion.createHRegion(hri,
323 hbaseRootDir, this.conf, htd);
324 HRegion.closeHRegion(region2);
325 HLog wal = createWAL(this.conf);
326 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
327 Path f = new Path(basedir, "hfile");
328 HFile.Writer writer =
329 HFile.getWriterFactoryNoCache(conf).withPath(fs, f).create();
330 byte [] family = htd.getFamilies().iterator().next().getName();
331 byte [] row = tableName.getName();
332 writer.append(new KeyValue(row, family, family, row));
333 writer.close();
334 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
335 hfs.add(Pair.newPair(family, f.toString()));
336 region.bulkLoadHFiles(hfs, true);
337
338 region.put((new Put(row)).add(family, family, family));
339 wal.sync();
340
341
342 final Configuration newConf = HBaseConfiguration.create(this.conf);
343 User user = HBaseTestingUtility.getDifferentUser(newConf,
344 tableName.getNameAsString());
345 user.runAs(new PrivilegedExceptionAction() {
346 public Object run() throws Exception {
347 runWALSplit(newConf);
348 HLog wal2 = createWAL(newConf);
349
350 HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
351 hbaseRootDir, hri, htd, wal2);
352 long seqid2 = region2.getOpenSeqNum();
353 assertTrue(seqid2 > -1);
354
355
356 region2.close();
357 wal2.closeAndDelete();
358 return null;
359 }
360 });
361 }
362
363
364
365
366
367
368
369
370
371
372 @Test
373 public void testReplayEditsWrittenViaHRegion()
374 throws IOException, SecurityException, IllegalArgumentException,
375 NoSuchFieldException, IllegalAccessException, InterruptedException {
376 final TableName tableName =
377 TableName.valueOf("testReplayEditsWrittenViaHRegion");
378 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
379 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
380 deleteDir(basedir);
381 final byte[] rowName = tableName.getName();
382 final int countPerFamily = 10;
383 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
384 HRegion region3 = HRegion.createHRegion(hri,
385 hbaseRootDir, this.conf, htd);
386 HRegion.closeHRegion(region3);
387
388
389
390 HLog wal = createWAL(this.conf);
391 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
392 long seqid = region.getOpenSeqNum();
393
394 wal.setSequenceNumber(seqid);
395 boolean first = true;
396 for (HColumnDescriptor hcd: htd.getFamilies()) {
397 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
398 if (first ) {
399
400 region.flushcache();
401 first = false;
402 }
403 }
404
405 final Get g = new Get(rowName);
406 Result result = region.get(g);
407 assertEquals(countPerFamily * htd.getFamilies().size(),
408 result.size());
409
410
411
412 region.close(true);
413 wal.close();
414 runWALSplit(this.conf);
415 HLog wal2 = createWAL(this.conf);
416 HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
417 long seqid2 = region2.getOpenSeqNum();
418
419 wal2.setSequenceNumber(seqid2);
420 assertTrue(seqid + result.size() < seqid2);
421 final Result result1b = region2.get(g);
422 assertEquals(result.size(), result1b.size());
423
424
425
426
427 for (HColumnDescriptor hcd: htd.getFamilies()) {
428 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
429 }
430
431 final Result result2 = region2.get(g);
432 assertEquals(2 * result.size(), result2.size());
433 wal2.sync();
434
435
436 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal2).getOutputStream(), 1);
437 final Configuration newConf = HBaseConfiguration.create(this.conf);
438 User user = HBaseTestingUtility.getDifferentUser(newConf,
439 tableName.getNameAsString());
440 user.runAs(new PrivilegedExceptionAction() {
441 public Object run() throws Exception {
442 runWALSplit(newConf);
443 FileSystem newFS = FileSystem.get(newConf);
444
445 HLog wal3 = createWAL(newConf);
446 final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
447 HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
448 @Override
449 protected boolean restoreEdit(Store s, KeyValue kv) {
450 boolean b = super.restoreEdit(s, kv);
451 countOfRestoredEdits.incrementAndGet();
452 return b;
453 }
454 };
455 long seqid3 = region3.initialize();
456
457 wal3.setSequenceNumber(seqid3);
458 Result result3 = region3.get(g);
459
460 assertEquals(result2.size(), result3.size());
461 assertEquals(htd.getFamilies().size() * countPerFamily,
462 countOfRestoredEdits.get());
463
464
465 region3.close();
466 wal3.closeAndDelete();
467 return null;
468 }
469 });
470 }
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490 @Test
491 public void testReplayEditsAfterPartialFlush()
492 throws IOException, SecurityException, IllegalArgumentException,
493 NoSuchFieldException, IllegalAccessException, InterruptedException {
494 final TableName tableName =
495 TableName.valueOf("testReplayEditsWrittenViaHRegion");
496 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
497 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
498 deleteDir(basedir);
499 final byte[] rowName = tableName.getName();
500 final int countPerFamily = 10;
501 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
502 HRegion region3 = HRegion.createHRegion(hri,
503 hbaseRootDir, this.conf, htd);
504 HRegion.closeHRegion(region3);
505
506
507
508 HLog wal = createWAL(this.conf);
509 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
510 long seqid = region.getOpenSeqNum();
511
512 wal.setSequenceNumber(seqid);
513 for (HColumnDescriptor hcd: htd.getFamilies()) {
514 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
515 }
516
517
518 final Get g = new Get(rowName);
519 Result result = region.get(g);
520 assertEquals(countPerFamily * htd.getFamilies().size(),
521 result.size());
522
523
524 region.flushcache();
525 region.close(true);
526 wal.close();
527
528
529
530
531
532 int cf_count = 0;
533 for (HColumnDescriptor hcd: htd.getFamilies()) {
534 cf_count++;
535 if (cf_count == 2) {
536 region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
537 }
538 }
539
540
541
542 runWALSplit(this.conf);
543 HLog wal2 = createWAL(this.conf);
544 HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
545 long seqid2 = region2.getOpenSeqNum();
546
547 wal2.setSequenceNumber(seqid2);
548 assertTrue(seqid + result.size() < seqid2);
549
550 final Result result1b = region2.get(g);
551 assertEquals(result.size(), result1b.size());
552 }
553
554
555
556
557 public static class CustomStoreFlusher extends DefaultStoreFlusher {
558
559 static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
560
561 public CustomStoreFlusher(Configuration conf, Store store) {
562 super(conf, store);
563 }
564 @Override
565 public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
566 TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status)
567 throws IOException {
568 if (throwExceptionWhenFlushing.get()) {
569 throw new IOException("Simulated exception by tests");
570 }
571 return super.flushSnapshot(snapshot, cacheFlushId, snapshotTimeRangeTracker,
572 flushedSize, status);
573 }
574
575 };
576
577
578
579
580
581
582
583 @Test
584 public void testReplayEditsAfterAbortingFlush() throws IOException {
585 final TableName tableName =
586 TableName.valueOf("testReplayEditsAfterAbortingFlush");
587 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
588 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
589 deleteDir(basedir);
590 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
591 HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
592 region3.close();
593 region3.getLog().closeAndDelete();
594
595
596
597 HLog wal = createWAL(this.conf);
598 RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
599 Mockito.doReturn(false).when(rsServices).isAborted();
600 Configuration customConf = new Configuration(this.conf);
601 customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
602 CustomStoreFlusher.class.getName());
603 HRegion region = new HRegion(basedir, wal, this.fs, customConf, hri, htd, rsServices);
604 long seqid = region.initialize();
605
606
607 wal.setSequenceNumber(seqid);
608
609 int writtenRowCount = 10;
610 List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
611 htd.getFamilies());
612 for (int i = 0; i < writtenRowCount; i++) {
613 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
614 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
615 Bytes.toBytes("val"));
616 region.put(put);
617 }
618
619
620 RegionScanner scanner = region.getScanner(new Scan());
621 assertEquals(writtenRowCount, getScannedCount(scanner));
622
623
624 CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
625 try {
626 region.flushcache();
627 fail("Injected exception hasn't been thrown");
628 } catch (Throwable t) {
629 LOG.info("Expected simulated exception when flushing region,"
630 + t.getMessage());
631
632 Mockito.doReturn(true).when(rsServices).isAborted();
633 }
634
635 int moreRow = 10;
636 for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
637 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
638 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
639 Bytes.toBytes("val"));
640 region.put(put);
641 }
642 writtenRowCount += moreRow;
643
644 CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
645 try {
646 region.flushcache();
647 } catch (IOException t) {
648 LOG.info("Expected exception when flushing region because server is stopped,"
649 + t.getMessage());
650 }
651
652 region.close(true);
653 wal.close();
654
655
656 runWALSplit(this.conf);
657 HLog wal2 = createWAL(this.conf);
658 Mockito.doReturn(false).when(rsServices).isAborted();
659 HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd,
660 rsServices);
661 long seqid2 = region2.initialize();
662
663
664 wal2.setSequenceNumber(seqid2);
665
666 scanner = region2.getScanner(new Scan());
667 assertEquals(writtenRowCount, getScannedCount(scanner));
668 }
669
670 private int getScannedCount(RegionScanner scanner) throws IOException {
671 int scannedCount = 0;
672 List<KeyValue> results = new ArrayList<KeyValue>();
673 while (true) {
674 boolean existMore = scanner.next(results);
675 if (!results.isEmpty())
676 scannedCount++;
677 if (!existMore)
678 break;
679 results.clear();
680 }
681 return scannedCount;
682 }
683
684
685
686
687
688
689 @Test
690 public void testReplayEditsWrittenIntoWAL() throws Exception {
691 final TableName tableName =
692 TableName.valueOf("testReplayEditsWrittenIntoWAL");
693 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
694 final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
695 deleteDir(basedir);
696
697 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
698 HRegion region2 = HRegion.createHRegion(hri,
699 hbaseRootDir, this.conf, htd);
700 HRegion.closeHRegion(region2);
701 final HLog wal = createWAL(this.conf);
702 final byte[] rowName = tableName.getName();
703 final byte[] regionName = hri.getEncodedNameAsBytes();
704
705
706 final int countPerFamily = 1000;
707 for (HColumnDescriptor hcd: htd.getFamilies()) {
708 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
709 ee, wal, htd);
710 }
711
712
713 wal.startCacheFlush(regionName);
714 wal.completeCacheFlush(regionName);
715
716
717 WALEdit edit = new WALEdit();
718 long now = ee.currentTimeMillis();
719 edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
720 now, rowName));
721 wal.append(hri, tableName, edit, now, htd);
722
723
724 edit = new WALEdit();
725 now = ee.currentTimeMillis();
726 edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
727 KeyValue.Type.DeleteFamily));
728 wal.append(hri, tableName, edit, now, htd);
729
730
731 wal.sync();
732
733
734 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
735
736
737 final Configuration newConf = HBaseConfiguration.create(this.conf);
738 User user = HBaseTestingUtility.getDifferentUser(newConf,
739 ".replay.wal.secondtime");
740 user.runAs(new PrivilegedExceptionAction() {
741 public Object run() throws Exception {
742 runWALSplit(newConf);
743 FileSystem newFS = FileSystem.get(newConf);
744
745 newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
746
747 HLog newWal = createWAL(newConf);
748 final AtomicInteger flushcount = new AtomicInteger(0);
749 try {
750 final HRegion region =
751 new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
752 protected boolean internalFlushcache(
753 final HLog wal, final long myseqid, MonitoredTask status)
754 throws IOException {
755 LOG.info("InternalFlushCache Invoked");
756 boolean b = super.internalFlushcache(wal, myseqid,
757 Mockito.mock(MonitoredTask.class));
758 flushcount.incrementAndGet();
759 return b;
760 };
761 };
762 long seqid = region.initialize();
763
764 assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
765 assertTrue(seqid > wal.getSequenceNumber());
766
767 Get get = new Get(rowName);
768 Result result = region.get(get);
769
770 assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
771 result.size());
772 region.close();
773 } finally {
774 newWal.closeAndDelete();
775 }
776 return null;
777 }
778 });
779 }
780
781 @Test
782
783 public void testSequentialEditLogSeqNum() throws IOException {
784 final TableName tableName =
785 TableName.valueOf("testSequentialEditLogSeqNum");
786 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
787 final Path basedir =
788 FSUtils.getTableDir(this.hbaseRootDir, tableName);
789 deleteDir(basedir);
790 final byte[] rowName = tableName.getName();
791 final int countPerFamily = 10;
792 final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
793
794
795 MockHLog wal = createMockWAL(this.conf);
796
797 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
798 long seqid = region.getOpenSeqNum();
799
800
801 wal.setSequenceNumber(seqid);
802 for (HColumnDescriptor hcd : htd.getFamilies()) {
803 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
804 }
805
806 long sequenceNumber = wal.getSequenceNumber();
807
808
809
810 region.flushcache();
811 for (HColumnDescriptor hcd : htd.getFamilies()) {
812 addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
813 }
814 long lastestSeqNumber = wal.getSequenceNumber();
815
816 wal.doCompleteCacheFlush = true;
817
818
819 wal.completeCacheFlush(hri.getEncodedNameAsBytes());
820 wal.close();
821 FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
822 HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0],
823 this.fs, this.conf, null, null, null);
824 FileStatus[] listStatus1 = this.fs.listStatus(
825 new Path(FSUtils.getTableDir(hbaseRootDir, tableName),
826 new Path(hri.getEncodedName(), "recovered.edits")));
827 int editCount = 0;
828 for (FileStatus fileStatus : listStatus1) {
829 editCount = Integer.parseInt(fileStatus.getPath().getName());
830 }
831
832 assertEquals(
833 "The sequence number of the recoverd.edits and the current edit seq should be same",
834 lastestSeqNumber, editCount);
835 }
836
837 static class MockHLog extends FSHLog {
838 boolean doCompleteCacheFlush = false;
839
840 public MockHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) throws IOException {
841 super(fs, rootDir, logName, conf);
842 }
843
844 @Override
845 public void completeCacheFlush(byte[] encodedRegionName) {
846 if (!doCompleteCacheFlush) {
847 return;
848 }
849 super.completeCacheFlush(encodedRegionName);
850 }
851 }
852
853 private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
854 HTableDescriptor htd = new HTableDescriptor(tableName);
855 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
856 htd.addFamily(a);
857 return htd;
858 }
859
860 private MockHLog createMockWAL(Configuration conf) throws IOException {
861 MockHLog wal = new MockHLog(FileSystem.get(conf), hbaseRootDir, logName, conf);
862
863
864 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
865 return wal;
866 }
867
868
869
870 class TestFlusher implements FlushRequester {
871 private HRegion r;
872
873 @Override
874 public void requestFlush(HRegion region) {
875 try {
876 r.flushcache();
877 } catch (IOException e) {
878 throw new RuntimeException("Exception flushing", e);
879 }
880 }
881
882 @Override
883 public void requestDelayedFlush(HRegion region, long when) {
884
885
886 }
887 }
888
889 private void addWALEdits (final TableName tableName, final HRegionInfo hri,
890 final byte [] rowName, final byte [] family,
891 final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd)
892 throws IOException {
893 String familyStr = Bytes.toString(family);
894 for (int j = 0; j < count; j++) {
895 byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
896 byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
897 WALEdit edit = new WALEdit();
898 edit.add(new KeyValue(rowName, family, qualifierBytes,
899 ee.currentTimeMillis(), columnBytes));
900 wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd);
901 }
902 }
903
904 private void addRegionEdits (final byte [] rowName, final byte [] family,
905 final int count, EnvironmentEdge ee, final HRegion r,
906 final String qualifierPrefix)
907 throws IOException {
908 for (int j = 0; j < count; j++) {
909 byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
910 Put p = new Put(rowName);
911 p.add(family, qualifier, ee.currentTimeMillis(), rowName);
912 r.put(p);
913 }
914 }
915
916
917
918
919
920
921 private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
922 return new HRegionInfo(tableName, null, null, false);
923 }
924
925
926
927
928
929
930
931 private Path runWALSplit(final Configuration c) throws IOException {
932 List<Path> splits = HLogSplitter.split(
933 hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c);
934
935 assertEquals("splits=" + splits, 1, splits.size());
936
937 assertTrue(fs.exists(splits.get(0)));
938 LOG.info("Split file=" + splits.get(0));
939 return splits.get(0);
940 }
941
942
943
944
945
946
947 private HLog createWAL(final Configuration c) throws IOException {
948 HLog wal = HLogFactory.createHLog(FileSystem.get(c),
949 hbaseRootDir, logName, c);
950
951
952 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
953 return wal;
954 }
955
956 private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
957 HTableDescriptor htd = new HTableDescriptor(tableName);
958 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
959 htd.addFamily(a);
960 HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
961 htd.addFamily(b);
962 HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
963 htd.addFamily(c);
964 return htd;
965 }
966
967 }
968