1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.regionserver.wal;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.security.PrivilegedExceptionAction;
28 import java.util.ArrayList;
29 import java.util.List;
30 import java.util.SortedSet;
31 import java.util.concurrent.atomic.AtomicBoolean;
32 import java.util.concurrent.atomic.AtomicInteger;
33 import java.util.concurrent.atomic.AtomicLong;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.Path;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HColumnDescriptor;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.KeyValue;
48 import org.apache.hadoop.hbase.MasterNotRunningException;
49 import org.apache.hadoop.hbase.MediumTests;
50 import org.apache.hadoop.hbase.MiniHBaseCluster;
51 import org.apache.hadoop.hbase.ServerName;
52 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
53 import org.apache.hadoop.hbase.client.Delete;
54 import org.apache.hadoop.hbase.client.Get;
55 import org.apache.hadoop.hbase.client.HTable;
56 import org.apache.hadoop.hbase.client.Put;
57 import org.apache.hadoop.hbase.client.Result;
58 import org.apache.hadoop.hbase.client.ResultScanner;
59 import org.apache.hadoop.hbase.client.Scan;
60 import org.apache.hadoop.hbase.master.HMaster;
61 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
62 import org.apache.hadoop.hbase.regionserver.FlushRequester;
63 import org.apache.hadoop.hbase.regionserver.HRegion;
64 import org.apache.hadoop.hbase.regionserver.HRegionServer;
65 import org.apache.hadoop.hbase.regionserver.RegionScanner;
66 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
67 import org.apache.hadoop.hbase.regionserver.Store;
68 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
69 import org.apache.hadoop.hbase.security.User;
70 import org.apache.hadoop.hbase.util.Bytes;
71 import org.apache.hadoop.hbase.util.EnvironmentEdge;
72 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
73 import org.apache.hadoop.hbase.util.HFileTestUtil;
74 import org.apache.hadoop.hbase.util.Pair;
75 import org.junit.After;
76 import org.junit.AfterClass;
77 import org.junit.Before;
78 import org.junit.BeforeClass;
79 import org.junit.Test;
80 import org.junit.experimental.categories.Category;
81 import org.mockito.Mockito;
82
83
84
85
86 @Category(MediumTests.class)
87 public class TestWALReplay {
88 public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
89 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
90 private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
91 private Path hbaseRootDir = null;
92 private Path oldLogDir;
93 private Path logDir;
94 private FileSystem fs;
95 private Configuration conf;
96
97 @BeforeClass
98 public static void setUpBeforeClass() throws Exception {
99 Configuration conf = TEST_UTIL.getConfiguration();
100 conf.setBoolean("dfs.support.append", true);
101
102 conf.setInt("dfs.client.block.recovery.retries", 2);
103 TEST_UTIL.startMiniCluster(3);
104 Path hbaseRootDir =
105 TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
106 LOG.info("hbase.rootdir=" + hbaseRootDir);
107 conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString());
108 }
109
110 @AfterClass
111 public static void tearDownAfterClass() throws Exception {
112 TEST_UTIL.shutdownMiniCluster();
113 }
114
115 @Before
116 public void setUp() throws Exception {
117 this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
118 this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
119 this.hbaseRootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
120 this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
121 this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME);
122 if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
123 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
124 }
125 }
126
127 @After
128 public void tearDown() throws Exception {
129 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
130 }
131
132
133
134
135 private void deleteDir(final Path p) throws IOException {
136 if (this.fs.exists(p)) {
137 if (!this.fs.delete(p, true)) {
138 throw new IOException("Failed remove of " + p);
139 }
140 }
141 }
142
143
144
145
146
147 @Test
148 public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
149 final byte[] tableName = Bytes
150 .toBytes("testReplayEditsAfterRegionMovedWithMultiCF");
151 byte[] family1 = Bytes.toBytes("cf1");
152 byte[] family2 = Bytes.toBytes("cf2");
153 byte[] qualifier = Bytes.toBytes("q");
154 byte[] value = Bytes.toBytes("testV");
155 byte[][] familys = { family1, family2 };
156 TEST_UTIL.createTable(tableName, familys);
157 HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
158 Put put = new Put(Bytes.toBytes("r1"));
159 put.add(family1, qualifier, value);
160 htable.put(put);
161 ResultScanner resultScanner = htable.getScanner(new Scan());
162 int count = 0;
163 while (resultScanner.next() != null) {
164 count++;
165 }
166 resultScanner.close();
167 assertEquals(1, count);
168
169 MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
170 List<HRegion> regions = hbaseCluster.getRegions(tableName);
171 assertEquals(1, regions.size());
172
173
174 HRegion destRegion = regions.get(0);
175 int originServerNum = hbaseCluster
176 .getServerWith(destRegion.getRegionName());
177 assertTrue("Please start more than 1 regionserver", hbaseCluster
178 .getRegionServerThreads().size() > 1);
179 int destServerNum = 0;
180 while (destServerNum == originServerNum) {
181 destServerNum++;
182 }
183 HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
184 HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
185
186 moveRegionAndWait(destRegion, destServer);
187
188
189 Delete del = new Delete(Bytes.toBytes("r1"));
190 htable.delete(del);
191 resultScanner = htable.getScanner(new Scan());
192 count = 0;
193 while (resultScanner.next() != null) {
194 count++;
195 }
196 resultScanner.close();
197 assertEquals(0, count);
198
199
200 destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
201
202 for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
203 .getStores().values()) {
204 store.triggerMajorCompaction();
205 }
206 destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
207
208
209 moveRegionAndWait(destRegion, originServer);
210
211 originServer.abort("testing");
212
213
214 Result result = htable.get(new Get(Bytes.toBytes("r1")));
215 if (result != null) {
216 assertTrue("Row is deleted, but we get" + result.toString(),
217 (result == null) || result.isEmpty());
218 }
219 resultScanner.close();
220 }
221
222 private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
223 throws InterruptedException, MasterNotRunningException,
224 ZooKeeperConnectionException, IOException {
225 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
226 TEST_UTIL.getHBaseAdmin().move(
227 destRegion.getRegionInfo().getEncodedNameAsBytes(),
228 Bytes.toBytes(destServer.getServerName().getServerName()));
229 while (true) {
230 ServerName serverName = master.getAssignmentManager()
231 .getRegionServerOfRegion(destRegion.getRegionInfo());
232 if (serverName != null && serverName.equals(destServer.getServerName())) break;
233 Thread.sleep(10);
234 }
235 }
236
237
238
239
240
241
242 @Test
243 public void test2727() throws Exception {
244
245
246 final String tableNameStr = "test2727";
247 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
248 Path basedir = new Path(hbaseRootDir, tableNameStr);
249 deleteDir(basedir);
250 fs.mkdirs(new Path(basedir, hri.getEncodedName()));
251
252 HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
253 HRegion region2 = HRegion.createHRegion(hri,
254 hbaseRootDir, this.conf, htd);
255 region2.close();
256 region2.getLog().closeAndDelete();
257 final byte [] tableName = Bytes.toBytes(tableNameStr);
258 final byte [] rowName = tableName;
259
260 HLog wal1 = createWAL(this.conf);
261
262 final int countPerFamily = 1000;
263 for (HColumnDescriptor hcd: htd.getFamilies()) {
264 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
265 wal1, htd);
266 }
267 wal1.close();
268 runWALSplit(this.conf);
269
270 HLog wal2 = createWAL(this.conf);
271
272 wal2.setSequenceNumber(wal1.getSequenceNumber());
273
274 for (HColumnDescriptor hcd: htd.getFamilies()) {
275 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
276 ee, wal2, htd);
277 }
278 wal2.close();
279 runWALSplit(this.conf);
280
281 HLog wal3 = createWAL(this.conf);
282 wal3.setSequenceNumber(wal2.getSequenceNumber());
283 try {
284 final HRegion region = new HRegion(basedir, wal3, this.fs, this.conf, hri,
285 htd, null);
286 long seqid = region.initialize();
287 assertTrue(seqid > wal3.getSequenceNumber());
288
289
290 region.close();
291 } finally {
292 wal3.closeAndDelete();
293 }
294 }
295
296
297
298
299
300
301
302
303
304
305 @Test
306 public void testRegionMadeOfBulkLoadedFilesOnly()
307 throws IOException, SecurityException, IllegalArgumentException,
308 NoSuchFieldException, IllegalAccessException, InterruptedException {
309 final String tableNameStr = "testRegionMadeOfBulkLoadedFilesOnly";
310 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
311 final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
312 deleteDir(basedir);
313 final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
314 HRegion region2 = HRegion.createHRegion(hri,
315 hbaseRootDir, this.conf, htd);
316 region2.close();
317 region2.getLog().closeAndDelete();
318 HLog wal = createWAL(this.conf);
319 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
320
321 byte [] family = htd.getFamilies().iterator().next().getName();
322 Path f = new Path(basedir, "hfile");
323 HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""),
324 Bytes.toBytes("z"), 10);
325 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
326 hfs.add(Pair.newPair(family, f.toString()));
327 region.bulkLoadHFiles(hfs, true);
328
329
330 byte [] row = Bytes.toBytes(tableNameStr);
331 region.put((new Put(row)).add(family, family, family));
332 wal.sync();
333 final int rowsInsertedCount = 11;
334
335 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
336
337
338 final Configuration newConf = HBaseConfiguration.create(this.conf);
339 User user = HBaseTestingUtility.getDifferentUser(newConf,
340 tableNameStr);
341 user.runAs(new PrivilegedExceptionAction() {
342 public Object run() throws Exception {
343 runWALSplit(newConf);
344 HLog wal2 = createWAL(newConf);
345 HRegion region2 = new HRegion(basedir, wal2, FileSystem.get(newConf),
346 newConf, hri, htd, null);
347 long seqid2 = region2.initialize();
348 assertTrue(seqid2 > -1);
349 assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
350
351
352 region2.close();
353 wal2.closeAndDelete();
354 return null;
355 }
356 });
357 }
358
359
360
361
362
363
364
365
366
367
368
369
370 @Test
371 public void testCompactedBulkLoadedFiles()
372 throws IOException, SecurityException, IllegalArgumentException,
373 NoSuchFieldException, IllegalAccessException, InterruptedException {
374 final String tableNameStr = "testCompactedBulkLoadedFiles";
375 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
376 final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
377 deleteDir(basedir);
378 final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
379 HRegion region2 = HRegion.createHRegion(hri,
380 hbaseRootDir, this.conf, htd);
381 region2.close();
382 region2.getLog().closeAndDelete();
383 HLog wal = createWAL(this.conf);
384 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
385
386
387 byte [] row = Bytes.toBytes(tableNameStr);
388 byte [] family = htd.getFamilies().iterator().next().getName();
389 region.put((new Put(row)).add(family, family, family));
390 wal.sync();
391
392 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
393 for (int i = 0; i < 3; i++) {
394 Path f = new Path(basedir, "hfile"+i);
395 HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(i + "00"),
396 Bytes.toBytes(i + "50"), 10);
397 hfs.add(Pair.newPair(family, f.toString()));
398 }
399 region.bulkLoadHFiles(hfs, true);
400 final int rowsInsertedCount = 31;
401 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
402
403
404 region.compactStores(true);
405 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
406
407
408 final Configuration newConf = HBaseConfiguration.create(this.conf);
409 User user = HBaseTestingUtility.getDifferentUser(newConf,
410 tableNameStr);
411 user.runAs(new PrivilegedExceptionAction() {
412 public Object run() throws Exception {
413 runWALSplit(newConf);
414 HLog wal2 = createWAL(newConf);
415 HRegion region2 = new HRegion(basedir, wal2, FileSystem.get(newConf),
416 newConf, hri, htd, null);
417 long seqid2 = region2.initialize();
418 assertTrue(seqid2 > -1);
419 assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
420
421
422 region2.close();
423 wal2.closeAndDelete();
424 return null;
425 }
426 });
427 }
428
429
430
431
432
433
434
435
436
437
438 @Test
439 public void testReplayEditsWrittenViaHRegion()
440 throws IOException, SecurityException, IllegalArgumentException,
441 NoSuchFieldException, IllegalAccessException, InterruptedException {
442 final String tableNameStr = "testReplayEditsWrittenViaHRegion";
443 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
444 final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
445 deleteDir(basedir);
446 final byte[] rowName = Bytes.toBytes(tableNameStr);
447 final int countPerFamily = 10;
448 final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
449 HRegion region3 = HRegion.createHRegion(hri,
450 hbaseRootDir, this.conf, htd);
451 region3.close();
452 region3.getLog().closeAndDelete();
453
454
455
456 HLog wal = createWAL(this.conf);
457 HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd, null);
458 long seqid = region.initialize();
459
460 wal.setSequenceNumber(seqid);
461 boolean first = true;
462 for (HColumnDescriptor hcd: htd.getFamilies()) {
463 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
464 if (first ) {
465
466 region.flushcache();
467 first = false;
468 }
469 }
470
471 final Get g = new Get(rowName);
472 Result result = region.get(g, null);
473 assertEquals(countPerFamily * htd.getFamilies().size(),
474 result.size());
475
476
477
478 region.close(true);
479 wal.close();
480 runWALSplit(this.conf);
481 HLog wal2 = createWAL(this.conf);
482 HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd, null);
483 long seqid2 = region2.initialize();
484
485 wal2.setSequenceNumber(seqid2);
486 assertTrue(seqid + result.size() < seqid2);
487 final Result result1b = region2.get(g, null);
488 assertEquals(result.size(), result1b.size());
489
490
491
492
493 for (HColumnDescriptor hcd: htd.getFamilies()) {
494 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
495 }
496
497 final Result result2 = region2.get(g, null);
498 assertEquals(2 * result.size(), result2.size());
499 wal2.sync();
500
501
502 HBaseTestingUtility.setMaxRecoveryErrorCount(wal2.getOutputStream(), 1);
503 final Configuration newConf = HBaseConfiguration.create(this.conf);
504 User user = HBaseTestingUtility.getDifferentUser(newConf,
505 tableNameStr);
506 user.runAs(new PrivilegedExceptionAction() {
507 public Object run() throws Exception {
508 runWALSplit(newConf);
509 FileSystem newFS = FileSystem.get(newConf);
510
511 HLog wal3 = createWAL(newConf);
512 final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
513 HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
514 @Override
515 protected boolean restoreEdit(Store s, KeyValue kv) {
516 boolean b = super.restoreEdit(s, kv);
517 countOfRestoredEdits.incrementAndGet();
518 return b;
519 }
520 };
521 long seqid3 = region3.initialize();
522
523 wal3.setSequenceNumber(seqid3);
524 Result result3 = region3.get(g, null);
525
526 assertEquals(result2.size(), result3.size());
527 assertEquals(htd.getFamilies().size() * countPerFamily,
528 countOfRestoredEdits.get());
529
530
531 region3.close();
532 wal3.closeAndDelete();
533 return null;
534 }
535 });
536 }
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556 @Test
557 public void testReplayEditsAfterPartialFlush()
558 throws IOException, SecurityException, IllegalArgumentException,
559 NoSuchFieldException, IllegalAccessException, InterruptedException {
560 final String tableNameStr = "testReplayEditsWrittenViaHRegion";
561 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
562 final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
563 deleteDir(basedir);
564 final byte[] rowName = Bytes.toBytes(tableNameStr);
565 final int countPerFamily = 10;
566 final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
567 HRegion region3 = HRegion.createHRegion(hri,
568 hbaseRootDir, this.conf, htd);
569 region3.close();
570 region3.getLog().closeAndDelete();
571
572
573
574 HLog wal = createWAL(this.conf);
575 HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd, null);
576 long seqid = region.initialize();
577
578 wal.setSequenceNumber(seqid);
579 for (HColumnDescriptor hcd: htd.getFamilies()) {
580 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
581 }
582
583
584 final Get g = new Get(rowName);
585 Result result = region.get(g, null);
586 assertEquals(countPerFamily * htd.getFamilies().size(),
587 result.size());
588
589
590 region.flushcache();
591 region.close(true);
592 wal.close();
593
594
595
596
597
598 int cf_count = 0;
599 for (HColumnDescriptor hcd: htd.getFamilies()) {
600 cf_count++;
601 if (cf_count == 2) {
602 this.fs.delete(new Path(region.getRegionDir(), Bytes.toString(hcd.getName()))
603 , true);
604 }
605 }
606
607
608
609 runWALSplit(this.conf);
610 HLog wal2 = createWAL(this.conf);
611 HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd, null);
612 long seqid2 = region2.initialize();
613
614 wal2.setSequenceNumber(seqid2);
615 assertTrue(seqid + result.size() < seqid2);
616
617 final Result result1b = region2.get(g, null);
618 assertEquals(result.size(), result1b.size());
619 }
620
621
622
623
624
625
626
627 @Test
628 public void testReplayEditsAfterAbortingFlush() throws IOException {
629 final String tableNameStr = "testReplayEditsAfterAbortingFlush";
630 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
631 final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
632 deleteDir(basedir);
633 final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
634 HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
635 region3.close();
636 region3.getLog().closeAndDelete();
637
638
639
640 HLog wal = createWAL(this.conf);
641 final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
642 RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
643 Mockito.doReturn(false).when(rsServices).isAborted();
644 HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd,
645 rsServices) {
646 @Override
647 protected Store instantiateHStore(Path tableDir, HColumnDescriptor c)
648 throws IOException {
649 return new Store(tableDir, this, c, fs, conf) {
650 @Override
651 protected Path flushCache(final long logCacheFlushId,
652 SortedSet<KeyValue> snapshot,
653 TimeRangeTracker snapshotTimeRangeTracker,
654 AtomicLong flushedSize, MonitoredTask status) throws IOException {
655 if (throwExceptionWhenFlushing.get()) {
656 throw new IOException("Simulated exception by tests");
657 }
658 return super.flushCache(logCacheFlushId, snapshot,
659 snapshotTimeRangeTracker, flushedSize, status);
660 }
661 };
662 }
663 };
664 long seqid = region.initialize();
665
666
667 wal.setSequenceNumber(seqid);
668
669 int writtenRowCount = 10;
670 List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
671 htd.getFamilies());
672 for (int i = 0; i < writtenRowCount; i++) {
673 Put put = new Put(Bytes.toBytes(tableNameStr + Integer.toString(i)));
674 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
675 Bytes.toBytes("val"));
676 region.put(put);
677 }
678
679
680 RegionScanner scanner = region.getScanner(new Scan());
681 assertEquals(writtenRowCount, getScannedCount(scanner));
682
683
684 throwExceptionWhenFlushing.set(true);
685 try {
686 region.flushcache();
687 fail("Injected exception hasn't been thrown");
688 } catch (Throwable t) {
689 LOG.info("Expected simulated exception when flushing region,"
690 + t.getMessage());
691
692 Mockito.doReturn(true).when(rsServices).isAborted();
693 }
694
695 int moreRow = 10;
696 for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
697 Put put = new Put(Bytes.toBytes(tableNameStr + Integer.toString(i)));
698 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
699 Bytes.toBytes("val"));
700 region.put(put);
701 }
702 writtenRowCount += moreRow;
703
704 throwExceptionWhenFlushing.set(false);
705 try {
706 region.flushcache();
707 } catch (IOException t) {
708 LOG.info("Expected exception when flushing region because server is stopped,"
709 + t.getMessage());
710 }
711
712 region.close(true);
713 wal.close();
714
715
716 runWALSplit(this.conf);
717 HLog wal2 = createWAL(this.conf);
718 Mockito.doReturn(false).when(rsServices).isAborted();
719 HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd,
720 rsServices);
721 long seqid2 = region2.initialize();
722
723
724 wal2.setSequenceNumber(seqid2);
725
726 scanner = region2.getScanner(new Scan());
727 assertEquals(writtenRowCount, getScannedCount(scanner));
728 }
729
730 private int getScannedCount(RegionScanner scanner) throws IOException {
731 int scannedCount = 0;
732 List<KeyValue> results = new ArrayList<KeyValue>();
733 while (true) {
734 boolean existMore = scanner.next(results);
735 if (!results.isEmpty())
736 scannedCount++;
737 if (!existMore)
738 break;
739 results.clear();
740 }
741 return scannedCount;
742 }
743
744
745
746
747
748
749 @Test
750 public void testReplayEditsWrittenIntoWAL() throws Exception {
751 final String tableNameStr = "testReplayEditsWrittenIntoWAL";
752 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
753 final Path basedir = new Path(hbaseRootDir, tableNameStr);
754 deleteDir(basedir);
755 fs.mkdirs(new Path(basedir, hri.getEncodedName()));
756 final HTableDescriptor htd = createBasic3FamilyHTD(tableNameStr);
757 HRegion region2 = HRegion.createHRegion(hri,
758 hbaseRootDir, this.conf, htd);
759 region2.close();
760 region2.getLog().closeAndDelete();
761 final HLog wal = createWAL(this.conf);
762 final byte[] tableName = Bytes.toBytes(tableNameStr);
763 final byte[] rowName = tableName;
764 final byte[] regionName = hri.getEncodedNameAsBytes();
765
766
767 final int countPerFamily = 1000;
768 for (HColumnDescriptor hcd: htd.getFamilies()) {
769 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
770 ee, wal, htd);
771 }
772
773
774 long logSeqId = wal.startCacheFlush(regionName);
775 wal.completeCacheFlush(regionName, tableName, logSeqId, hri.isMetaRegion());
776
777
778 WALEdit edit = new WALEdit();
779 long now = ee.currentTimeMillis();
780 edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
781 now, rowName));
782 wal.append(hri, tableName, edit, now, htd);
783
784
785 edit = new WALEdit();
786 now = ee.currentTimeMillis();
787 edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
788 KeyValue.Type.DeleteFamily));
789 wal.append(hri, tableName, edit, now, htd);
790
791
792 wal.sync();
793
794
795 HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
796
797
798 final Configuration newConf = HBaseConfiguration.create(this.conf);
799 User user = HBaseTestingUtility.getDifferentUser(newConf,
800 ".replay.wal.secondtime");
801 user.runAs(new PrivilegedExceptionAction() {
802 public Object run() throws Exception {
803 runWALSplit(newConf);
804 FileSystem newFS = FileSystem.get(newConf);
805
806 newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
807
808 HLog newWal = createWAL(newConf);
809 final AtomicInteger flushcount = new AtomicInteger(0);
810 try {
811 final HRegion region =
812 new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
813 protected FlushResult internalFlushcache(
814 final HLog wal, final long myseqid, MonitoredTask status)
815 throws IOException {
816 LOG.info("InternalFlushCache Invoked");
817 FlushResult fs = super.internalFlushcache(wal, myseqid,
818 Mockito.mock(MonitoredTask.class));
819 flushcount.incrementAndGet();
820 return fs;
821 };
822 };
823 long seqid = region.initialize();
824
825 assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
826 assertTrue(seqid > wal.getSequenceNumber());
827
828 Get get = new Get(rowName);
829 Result result = region.get(get, -1);
830
831 assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
832 result.size());
833 region.close();
834 } finally {
835 newWal.closeAndDelete();
836 }
837 return null;
838 }
839 });
840 }
841
842 @Test
843 public void testSequentialEditLogSeqNum() throws IOException {
844 final String tableNameStr = "testSequentialEditLogSeqNum";
845 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
846 final Path basedir = new Path(this.hbaseRootDir, tableNameStr);
847 deleteDir(basedir);
848 final byte[] rowName = Bytes.toBytes(tableNameStr);
849 final int countPerFamily = 10;
850 final HTableDescriptor htd = createBasic1FamilyHTD(tableNameStr);
851
852
853 MockHLog wal = createMockWAL(this.conf);
854
855 HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, htd, null);
856 long seqid = region.initialize();
857
858
859 wal.setSequenceNumber(seqid);
860 for (HColumnDescriptor hcd : htd.getFamilies()) {
861 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
862 }
863
864 long sequenceNumber = wal.getSequenceNumber();
865
866
867
868 region.flushcache();
869 for (HColumnDescriptor hcd : htd.getFamilies()) {
870 addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
871 }
872 long lastestSeqNumber = wal.getSequenceNumber();
873
874 wal.doCompleteCacheFlush = true;
875
876
877 wal.completeCacheFlush(hri.getEncodedNameAsBytes(), hri.getTableName(), sequenceNumber, false);
878 wal.close();
879 FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
880 HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0], this.fs, this.conf,
881 null);
882 FileStatus[] listStatus1 = this.fs.listStatus(new Path(hbaseRootDir + "/"
883 + tableNameStr + "/" + hri.getEncodedName() + "/recovered.edits"));
884 int editCount = 0;
885 for (FileStatus fileStatus : listStatus1) {
886 editCount = Integer.parseInt(fileStatus.getPath().getName());
887 }
888
889 assertEquals(
890 "The sequence number of the recoverd.edits and the current edit seq should be same",
891 lastestSeqNumber, editCount);
892 }
893
894 static class MockHLog extends HLog {
895 boolean doCompleteCacheFlush = false;
896
897 public MockHLog(FileSystem fs, Path dir, Path oldLogDir, Configuration conf) throws IOException {
898 super(fs, dir, oldLogDir, conf);
899 }
900
901 @Override
902 public void completeCacheFlush(byte[] encodedRegionName, byte[] tableName, long logSeqId,
903 boolean isMetaRegion) throws IOException {
904 if (!doCompleteCacheFlush) {
905 return;
906 }
907 super.completeCacheFlush(encodedRegionName, tableName, logSeqId, isMetaRegion);
908 }
909 }
910
911 private HTableDescriptor createBasic1FamilyHTD(final String tableName) {
912 HTableDescriptor htd = new HTableDescriptor(tableName);
913 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
914 htd.addFamily(a);
915 return htd;
916 }
917
918 private MockHLog createMockWAL(Configuration conf) throws IOException {
919 MockHLog wal = new MockHLog(FileSystem.get(conf), logDir, oldLogDir, conf);
920
921
922 HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
923 return wal;
924 }
925
926
927
928
929 class TestFlusher implements FlushRequester {
930 private int count = 0;
931 private HRegion r;
932
933 @Override
934 public void requestFlush(HRegion region) {
935 count++;
936 try {
937 r.flushcache();
938 } catch (IOException e) {
939 throw new RuntimeException("Exception flushing", e);
940 }
941 }
942
943 @Override
944 public void requestDelayedFlush(HRegion region, long when) {
945
946
947 }
948 }
949
950 private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
951 final byte [] rowName, final byte [] family,
952 final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd)
953 throws IOException {
954 String familyStr = Bytes.toString(family);
955 for (int j = 0; j < count; j++) {
956 byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
957 byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
958 WALEdit edit = new WALEdit();
959 edit.add(new KeyValue(rowName, family, qualifierBytes,
960 ee.currentTimeMillis(), columnBytes));
961 wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd);
962 }
963 }
964
965 private void addRegionEdits (final byte [] rowName, final byte [] family,
966 final int count, EnvironmentEdge ee, final HRegion r,
967 final String qualifierPrefix)
968 throws IOException {
969 for (int j = 0; j < count; j++) {
970 byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
971 Put p = new Put(rowName);
972 p.add(family, qualifier, ee.currentTimeMillis(), rowName);
973 r.put(p);
974 }
975 }
976
977
978
979
980
981
982 private HRegionInfo createBasic3FamilyHRegionInfo(final String tableName) {
983 return new HRegionInfo(Bytes.toBytes(tableName), null, null, false);
984 }
985
986
987
988
989
990
991
992 private Path runWALSplit(final Configuration c) throws IOException {
993 FileSystem fs = FileSystem.get(c);
994 HLogSplitter logSplitter = HLogSplitter.createLogSplitter(c,
995 this.hbaseRootDir, this.logDir, this.oldLogDir, fs);
996 List<Path> splits = logSplitter.splitLog();
997
998 assertEquals("splits=" + splits, 1, splits.size());
999
1000 assertTrue(fs.exists(splits.get(0)));
1001 LOG.info("Split file=" + splits.get(0));
1002 return splits.get(0);
1003 }
1004
1005
1006
1007
1008
1009
1010 private HLog createWAL(final Configuration c) throws IOException {
1011 HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c);
1012
1013
1014 HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
1015 return wal;
1016 }
1017
1018 private HTableDescriptor createBasic3FamilyHTD(final String tableName) {
1019 HTableDescriptor htd = new HTableDescriptor(tableName);
1020 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
1021 htd.addFamily(a);
1022 HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
1023 htd.addFamily(b);
1024 HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
1025 htd.addFamily(c);
1026 return htd;
1027 }
1028
1029 @org.junit.Rule
1030 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
1031 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
1032 }
1033