1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23 import static org.junit.Assert.fail;
24
25 import java.io.IOException;
26 import java.security.PrivilegedExceptionAction;
27 import java.util.ArrayList;
28 import java.util.List;
29 import java.util.SortedSet;
30 import java.util.concurrent.atomic.AtomicBoolean;
31 import java.util.concurrent.atomic.AtomicInteger;
32 import java.util.concurrent.atomic.AtomicLong;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.Cell;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HColumnDescriptor;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.KeyValue;
48 import org.apache.hadoop.hbase.MasterNotRunningException;
49 import org.apache.hadoop.hbase.MediumTests;
50 import org.apache.hadoop.hbase.MiniHBaseCluster;
51 import org.apache.hadoop.hbase.ServerName;
52 import org.apache.hadoop.hbase.TableName;
53 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
54 import org.apache.hadoop.hbase.client.Delete;
55 import org.apache.hadoop.hbase.client.Get;
56 import org.apache.hadoop.hbase.client.HTable;
57 import org.apache.hadoop.hbase.client.Put;
58 import org.apache.hadoop.hbase.client.Result;
59 import org.apache.hadoop.hbase.client.ResultScanner;
60 import org.apache.hadoop.hbase.client.Scan;
61 import org.apache.hadoop.hbase.master.HMaster;
62 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
63 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
64 import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
65 import org.apache.hadoop.hbase.regionserver.FlushRequester;
66 import org.apache.hadoop.hbase.regionserver.HRegion;
67 import org.apache.hadoop.hbase.regionserver.HRegionServer;
68 import org.apache.hadoop.hbase.regionserver.RegionScanner;
69 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
70 import org.apache.hadoop.hbase.regionserver.Store;
71 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
72 import org.apache.hadoop.hbase.security.User;
73 import org.apache.hadoop.hbase.util.Bytes;
74 import org.apache.hadoop.hbase.util.EnvironmentEdge;
75 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
76 import org.apache.hadoop.hbase.util.FSUtils;
77 import org.apache.hadoop.hbase.util.HFileTestUtil;
78 import org.apache.hadoop.hbase.util.Pair;
79 import org.junit.After;
80 import org.junit.AfterClass;
81 import org.junit.Before;
82 import org.junit.BeforeClass;
83 import org.junit.Test;
84 import org.junit.experimental.categories.Category;
85 import org.mockito.Mockito;
86
87
88
89
90 @Category(MediumTests.class)
91 public class TestWALReplay {
92 public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
93 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
94 private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
95 private Path hbaseRootDir = null;
96 private String logName;
97 private Path oldLogDir;
98 private Path logDir;
99 private FileSystem fs;
100 private Configuration conf;
101
102 @BeforeClass
103 public static void setUpBeforeClass() throws Exception {
104 Configuration conf = TEST_UTIL.getConfiguration();
105 conf.setBoolean("dfs.support.append", true);
106
107 conf.setInt("dfs.client.block.recovery.retries", 2);
108 TEST_UTIL.startMiniCluster(3);
109 Path hbaseRootDir =
110 TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
111 LOG.info("hbase.rootdir=" + hbaseRootDir);
112 FSUtils.setRootDir(conf, hbaseRootDir);
113 }
114
115 @AfterClass
116 public static void tearDownAfterClass() throws Exception {
117 TEST_UTIL.shutdownMiniCluster();
118 }
119
120 @Before
121 public void setUp() throws Exception {
122 this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
123 this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
124 this.hbaseRootDir = FSUtils.getRootDir(this.conf);
125 this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
126 this.logName = HConstants.HREGION_LOGDIR_NAME;
127 this.logDir = new Path(this.hbaseRootDir, logName);
128 if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
129 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
130 }
131 }
132
133 @After
134 public void tearDown() throws Exception {
135 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
136 }
137
138
139
140
141 private void deleteDir(final Path p) throws IOException {
142 if (this.fs.exists(p)) {
143 if (!this.fs.delete(p, true)) {
144 throw new IOException("Failed remove of " + p);
145 }
146 }
147 }
148
149
150
151
152
153 @Test
154 public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
155 final TableName tableName =
156 TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
157 byte[] family1 = Bytes.toBytes("cf1");
158 byte[] family2 = Bytes.toBytes("cf2");
159 byte[] qualifier = Bytes.toBytes("q");
160 byte[] value = Bytes.toBytes("testV");
161 byte[][] familys = { family1, family2 };
162 TEST_UTIL.createTable(tableName, familys);
163 HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
164 Put put = new Put(Bytes.toBytes("r1"));
165 put.add(family1, qualifier, value);
166 htable.put(put);
167 ResultScanner resultScanner = htable.getScanner(new Scan());
168 int count = 0;
169 while (resultScanner.next() != null) {
170 count++;
171 }
172 resultScanner.close();
173 assertEquals(1, count);
174
175 MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
176 List<HRegion> regions = hbaseCluster.getRegions(tableName);
177 assertEquals(1, regions.size());
178
179
180 HRegion destRegion = regions.get(0);
181 int originServerNum = hbaseCluster
182 .getServerWith(destRegion.getRegionName());
183 assertTrue("Please start more than 1 regionserver", hbaseCluster
184 .getRegionServerThreads().size() > 1);
185 int destServerNum = 0;
186 while (destServerNum == originServerNum) {
187 destServerNum++;
188 }
189 HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
190 HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
191
192 moveRegionAndWait(destRegion, destServer);
193
194
195 Delete del = new Delete(Bytes.toBytes("r1"));
196 htable.delete(del);
197 resultScanner = htable.getScanner(new Scan());
198 count = 0;
199 while (resultScanner.next() != null) {
200 count++;
201 }
202 resultScanner.close();
203 assertEquals(0, count);
204
205
206 destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
207
208 for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
209 .getStores().values()) {
210 store.triggerMajorCompaction();
211 }
212 destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
213
214
215 moveRegionAndWait(destRegion, originServer);
216
217 originServer.abort("testing");
218
219
220 Result result = htable.get(new Get(Bytes.toBytes("r1")));
221 if (result != null) {
222 assertTrue("Row is deleted, but we get" + result.toString(),
223 (result == null) || result.isEmpty());
224 }
225 resultScanner.close();
226 }
227
228 private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
229 throws InterruptedException, MasterNotRunningException,
230 ZooKeeperConnectionException, IOException {
231 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
232 TEST_UTIL.getHBaseAdmin().move(
233 destRegion.getRegionInfo().getEncodedNameAsBytes(),
234 Bytes.toBytes(destServer.getServerName().getServerName()));
235 while (true) {
236 ServerName serverName = master.getAssignmentManager()
237 .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
238 if (serverName != null && serverName.equals(destServer.getServerName())) {
239 TEST_UTIL.assertRegionOnServer(
240 destRegion.getRegionInfo(), serverName, 200);
241 break;
242 }
243 Thread.sleep(10);
244 }
245 }
246
247
248
249
250
251
252 @Test
253 public void test2727() throws Exception {
254
255
256 final TableName tableName =
257 TableName.valueOf("test2727");
258 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
259 Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
260 deleteDir(basedir);
261
262 HTableDescriptor htd = createBasic3FamilyHTD(tableName);
263 HRegion region2 = HRegion.createHRegion(hri,
264 hbaseRootDir, this.conf, htd);
265 HRegion.closeHRegion(region2);
266 final byte [] rowName = tableName.getName();
267
268 HLog wal1 = createWAL(this.conf);
269
270 final int countPerFamily = 1000;
271 final AtomicLong sequenceId = new AtomicLong(1);
272 for (HColumnDescriptor hcd: htd.getFamilies()) {
273 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
274 wal1, htd, sequenceId);
275 }
276 wal1.close();
277 runWALSplit(this.conf);
278
279 HLog wal2 = createWAL(this.conf);
280
281 for (HColumnDescriptor hcd: htd.getFamilies()) {
282 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
283 ee, wal2, htd, sequenceId);
284 }
285 wal2.close();
286 runWALSplit(this.conf);
287
288 HLog wal3 = createWAL(this.conf);
289 try {
290 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
291 long seqid = region.getOpenSeqNum();
292
293
294 assertTrue(seqid > sequenceId.get());
295 assertEquals(seqid - 1, sequenceId.get());
296 LOG.debug("region.getOpenSeqNum(): " + region.getOpenSeqNum() + ", wal3.id: "
297 + sequenceId.get());
298
299
300 region.close();
301 } finally {
302 wal3.closeAndDelete();
303 }
304 }
305
306
307
308
309
310
311
312
313
314
315 @Test
316 public void testRegionMadeOfBulkLoadedFilesOnly()
317 throws IOException, SecurityException, IllegalArgumentException,
318 NoSuchFieldException, IllegalAccessException, InterruptedException {
319 final TableName tableName =
320 TableName.valueOf("testRegionMadeOfBulkLoadedFilesOnly");
321 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
322 final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
323 deleteDir(basedir);
324 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
325 HRegion region2 = HRegion.createHRegion(hri,
326 hbaseRootDir, this.conf, htd);
327 HRegion.closeHRegion(region2);
328 HLog wal = createWAL(this.conf);
329 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
330
331 byte [] family = htd.getFamilies().iterator().next().getName();
332 Path f = new Path(basedir, "hfile");
333 HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""),
334 Bytes.toBytes("z"), 10);
335 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
336 hfs.add(Pair.newPair(family, f.toString()));
337 region.bulkLoadHFiles(hfs, true);
338
339
340 byte [] row = tableName.getName();
341 region.put((new Put(row)).add(family, family, family));
342 wal.sync();
343 final int rowsInsertedCount = 11;
344
345 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
346
347
348 final Configuration newConf = HBaseConfiguration.create(this.conf);
349 User user = HBaseTestingUtility.getDifferentUser(newConf,
350 tableName.getNameAsString());
351 user.runAs(new PrivilegedExceptionAction() {
352 public Object run() throws Exception {
353 runWALSplit(newConf);
354 HLog wal2 = createWAL(newConf);
355
356 HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
357 hbaseRootDir, hri, htd, wal2);
358 long seqid2 = region2.getOpenSeqNum();
359 assertTrue(seqid2 > -1);
360 assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
361
362
363 region2.close();
364 wal2.closeAndDelete();
365 return null;
366 }
367 });
368 }
369
370
371
372
373
374
375
376
377
378
379
380
381 @Test
382 public void testCompactedBulkLoadedFiles()
383 throws IOException, SecurityException, IllegalArgumentException,
384 NoSuchFieldException, IllegalAccessException, InterruptedException {
385 final TableName tableName =
386 TableName.valueOf("testCompactedBulkLoadedFiles");
387 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
388 final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
389 deleteDir(basedir);
390 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
391 HRegion region2 = HRegion.createHRegion(hri,
392 hbaseRootDir, this.conf, htd);
393 HRegion.closeHRegion(region2);
394 HLog wal = createWAL(this.conf);
395 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
396
397
398 byte [] row = tableName.getName();
399 byte [] family = htd.getFamilies().iterator().next().getName();
400 region.put((new Put(row)).add(family, family, family));
401 wal.sync();
402
403 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
404 for (int i = 0; i < 3; i++) {
405 Path f = new Path(basedir, "hfile"+i);
406 HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(i + "00"),
407 Bytes.toBytes(i + "50"), 10);
408 hfs.add(Pair.newPair(family, f.toString()));
409 }
410 region.bulkLoadHFiles(hfs, true);
411 final int rowsInsertedCount = 31;
412 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
413
414
415 region.compactStores(true);
416 assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
417
418
419 final Configuration newConf = HBaseConfiguration.create(this.conf);
420 User user = HBaseTestingUtility.getDifferentUser(newConf,
421 tableName.getNameAsString());
422 user.runAs(new PrivilegedExceptionAction() {
423 public Object run() throws Exception {
424 runWALSplit(newConf);
425 HLog wal2 = createWAL(newConf);
426
427 HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
428 hbaseRootDir, hri, htd, wal2);
429 long seqid2 = region2.getOpenSeqNum();
430 assertTrue(seqid2 > -1);
431 assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
432
433
434 region2.close();
435 wal2.closeAndDelete();
436 return null;
437 }
438 });
439 }
440
441
442
443
444
445
446
447
448
449
450
451 @Test
452 public void testReplayEditsWrittenViaHRegion()
453 throws IOException, SecurityException, IllegalArgumentException,
454 NoSuchFieldException, IllegalAccessException, InterruptedException {
455 final TableName tableName =
456 TableName.valueOf("testReplayEditsWrittenViaHRegion");
457 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
458 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
459 deleteDir(basedir);
460 final byte[] rowName = tableName.getName();
461 final int countPerFamily = 10;
462 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
463 HRegion region3 = HRegion.createHRegion(hri,
464 hbaseRootDir, this.conf, htd);
465 HRegion.closeHRegion(region3);
466
467
468
469 HLog wal = createWAL(this.conf);
470 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
471 long seqid = region.getOpenSeqNum();
472 boolean first = true;
473 for (HColumnDescriptor hcd: htd.getFamilies()) {
474 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
475 if (first ) {
476
477 region.flushcache();
478 first = false;
479 }
480 }
481
482 final Get g = new Get(rowName);
483 Result result = region.get(g);
484 assertEquals(countPerFamily * htd.getFamilies().size(),
485 result.size());
486
487
488
489 region.close(true);
490 wal.close();
491 runWALSplit(this.conf);
492 HLog wal2 = createWAL(this.conf);
493 HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
494 long seqid2 = region2.getOpenSeqNum();
495 assertTrue(seqid + result.size() < seqid2);
496 final Result result1b = region2.get(g);
497 assertEquals(result.size(), result1b.size());
498
499
500
501
502 for (HColumnDescriptor hcd: htd.getFamilies()) {
503 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
504 }
505
506 final Result result2 = region2.get(g);
507 assertEquals(2 * result.size(), result2.size());
508 wal2.sync();
509
510
511 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal2).getOutputStream(), 1);
512 final Configuration newConf = HBaseConfiguration.create(this.conf);
513 User user = HBaseTestingUtility.getDifferentUser(newConf,
514 tableName.getNameAsString());
515 user.runAs(new PrivilegedExceptionAction() {
516 public Object run() throws Exception {
517 runWALSplit(newConf);
518 FileSystem newFS = FileSystem.get(newConf);
519
520 HLog wal3 = createWAL(newConf);
521 final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
522 HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
523 @Override
524 protected boolean restoreEdit(Store s, KeyValue kv) {
525 boolean b = super.restoreEdit(s, kv);
526 countOfRestoredEdits.incrementAndGet();
527 return b;
528 }
529 };
530 long seqid3 = region3.initialize();
531 Result result3 = region3.get(g);
532
533 assertEquals(result2.size(), result3.size());
534 assertEquals(htd.getFamilies().size() * countPerFamily,
535 countOfRestoredEdits.get());
536
537
538 region3.close();
539 wal3.closeAndDelete();
540 return null;
541 }
542 });
543 }
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563 @Test
564 public void testReplayEditsAfterPartialFlush()
565 throws IOException, SecurityException, IllegalArgumentException,
566 NoSuchFieldException, IllegalAccessException, InterruptedException {
567 final TableName tableName =
568 TableName.valueOf("testReplayEditsWrittenViaHRegion");
569 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
570 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
571 deleteDir(basedir);
572 final byte[] rowName = tableName.getName();
573 final int countPerFamily = 10;
574 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
575 HRegion region3 = HRegion.createHRegion(hri,
576 hbaseRootDir, this.conf, htd);
577 HRegion.closeHRegion(region3);
578
579
580
581 HLog wal = createWAL(this.conf);
582 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
583 long seqid = region.getOpenSeqNum();
584 for (HColumnDescriptor hcd: htd.getFamilies()) {
585 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
586 }
587
588
589 final Get g = new Get(rowName);
590 Result result = region.get(g);
591 assertEquals(countPerFamily * htd.getFamilies().size(),
592 result.size());
593
594
595 region.flushcache();
596 region.close(true);
597 wal.close();
598
599
600
601
602
603 int cf_count = 0;
604 for (HColumnDescriptor hcd: htd.getFamilies()) {
605 cf_count++;
606 if (cf_count == 2) {
607 region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
608 }
609 }
610
611
612
613 runWALSplit(this.conf);
614 HLog wal2 = createWAL(this.conf);
615 HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
616 long seqid2 = region2.getOpenSeqNum();
617 assertTrue(seqid + result.size() < seqid2);
618
619 final Result result1b = region2.get(g);
620 assertEquals(result.size(), result1b.size());
621 }
622
623
624
625
626 public static class CustomStoreFlusher extends DefaultStoreFlusher {
627
628 static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
629
630 public CustomStoreFlusher(Configuration conf, Store store) {
631 super(conf, store);
632 }
633 @Override
634 public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
635 TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status)
636 throws IOException {
637 if (throwExceptionWhenFlushing.get()) {
638 throw new IOException("Simulated exception by tests");
639 }
640 return super.flushSnapshot(snapshot, cacheFlushId, snapshotTimeRangeTracker,
641 flushedSize, status);
642 }
643
644 };
645
646
647
648
649
650
651
652 @Test
653 public void testReplayEditsAfterAbortingFlush() throws IOException {
654 final TableName tableName =
655 TableName.valueOf("testReplayEditsAfterAbortingFlush");
656 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
657 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
658 deleteDir(basedir);
659 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
660 HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
661 region3.close();
662 region3.getLog().closeAndDelete();
663
664
665
666 HLog wal = createWAL(this.conf);
667 RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
668 Mockito.doReturn(false).when(rsServices).isAborted();
669 Configuration customConf = new Configuration(this.conf);
670 customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
671 CustomStoreFlusher.class.getName());
672 HRegion region =
673 HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
674 int writtenRowCount = 10;
675 List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
676 htd.getFamilies());
677 for (int i = 0; i < writtenRowCount; i++) {
678 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
679 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
680 Bytes.toBytes("val"));
681 region.put(put);
682 }
683
684
685 RegionScanner scanner = region.getScanner(new Scan());
686 assertEquals(writtenRowCount, getScannedCount(scanner));
687
688
689 CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
690 try {
691 region.flushcache();
692 fail("Injected exception hasn't been thrown");
693 } catch (Throwable t) {
694 LOG.info("Expected simulated exception when flushing region,"
695 + t.getMessage());
696
697 Mockito.doReturn(true).when(rsServices).isAborted();
698 }
699
700 int moreRow = 10;
701 for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
702 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
703 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
704 Bytes.toBytes("val"));
705 region.put(put);
706 }
707 writtenRowCount += moreRow;
708
709 CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
710 try {
711 region.flushcache();
712 } catch (IOException t) {
713 LOG.info("Expected exception when flushing region because server is stopped,"
714 + t.getMessage());
715 }
716
717 region.close(true);
718 wal.close();
719
720
721 runWALSplit(this.conf);
722 HLog wal2 = createWAL(this.conf);
723 Mockito.doReturn(false).when(rsServices).isAborted();
724 HRegion region2 =
725 HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
726 scanner = region2.getScanner(new Scan());
727 assertEquals(writtenRowCount, getScannedCount(scanner));
728 }
729
730 private int getScannedCount(RegionScanner scanner) throws IOException {
731 int scannedCount = 0;
732 List<Cell> results = new ArrayList<Cell>();
733 while (true) {
734 boolean existMore = scanner.next(results);
735 if (!results.isEmpty())
736 scannedCount++;
737 if (!existMore)
738 break;
739 results.clear();
740 }
741 return scannedCount;
742 }
743
744
745
746
747
748
749 @Test
750 public void testReplayEditsWrittenIntoWAL() throws Exception {
751 final TableName tableName =
752 TableName.valueOf("testReplayEditsWrittenIntoWAL");
753 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
754 final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
755 deleteDir(basedir);
756
757 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
758 HRegion region2 = HRegion.createHRegion(hri,
759 hbaseRootDir, this.conf, htd);
760 HRegion.closeHRegion(region2);
761 final HLog wal = createWAL(this.conf);
762 final byte[] rowName = tableName.getName();
763 final byte[] regionName = hri.getEncodedNameAsBytes();
764 final AtomicLong sequenceId = new AtomicLong(1);
765
766
767 final int countPerFamily = 1000;
768 for (HColumnDescriptor hcd: htd.getFamilies()) {
769 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
770 ee, wal, htd, sequenceId);
771 }
772
773
774 wal.startCacheFlush(regionName);
775 wal.completeCacheFlush(regionName);
776
777
778 WALEdit edit = new WALEdit();
779 long now = ee.currentTimeMillis();
780 edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
781 now, rowName));
782 wal.append(hri, tableName, edit, now, htd, sequenceId);
783
784
785 edit = new WALEdit();
786 now = ee.currentTimeMillis();
787 edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
788 KeyValue.Type.DeleteFamily));
789 wal.append(hri, tableName, edit, now, htd, sequenceId);
790
791
792 wal.sync();
793
794
795 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
796
797
798 final Configuration newConf = HBaseConfiguration.create(this.conf);
799 User user = HBaseTestingUtility.getDifferentUser(newConf,
800 ".replay.wal.secondtime");
801 user.runAs(new PrivilegedExceptionAction() {
802 public Object run() throws Exception {
803 runWALSplit(newConf);
804 FileSystem newFS = FileSystem.get(newConf);
805
806 newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
807
808 HLog newWal = createWAL(newConf);
809 final AtomicInteger flushcount = new AtomicInteger(0);
810 try {
811 final HRegion region =
812 new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
813 protected FlushResult internalFlushcache(
814 final HLog wal, final long myseqid, MonitoredTask status)
815 throws IOException {
816 LOG.info("InternalFlushCache Invoked");
817 FlushResult fs = super.internalFlushcache(wal, myseqid,
818 Mockito.mock(MonitoredTask.class));
819 flushcount.incrementAndGet();
820 return fs;
821 };
822 };
823 long seqid = region.initialize();
824
825 assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
826 assertTrue(seqid - 1 == sequenceId.get());
827
828 Get get = new Get(rowName);
829 Result result = region.get(get);
830
831 assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
832 result.size());
833 region.close();
834 } finally {
835 newWal.closeAndDelete();
836 }
837 return null;
838 }
839 });
840 }
841
842 @Test
843
844 public void testSequentialEditLogSeqNum() throws IOException {
845 final TableName tableName =
846 TableName.valueOf("testSequentialEditLogSeqNum");
847 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
848 final Path basedir =
849 FSUtils.getTableDir(this.hbaseRootDir, tableName);
850 deleteDir(basedir);
851 final byte[] rowName = tableName.getName();
852 final int countPerFamily = 10;
853 final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
854
855
856 MockHLog wal = createMockWAL(this.conf);
857
858 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
859 for (HColumnDescriptor hcd : htd.getFamilies()) {
860 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
861 }
862
863
864
865 region.flushcache();
866 for (HColumnDescriptor hcd : htd.getFamilies()) {
867 addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
868 }
869 long lastestSeqNumber = region.getSequenceId().get();
870
871 wal.doCompleteCacheFlush = true;
872
873
874 wal.completeCacheFlush(hri.getEncodedNameAsBytes());
875 wal.close();
876 FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
877 HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0],
878 this.fs, this.conf, null, null, null);
879 FileStatus[] listStatus1 = this.fs.listStatus(
880 new Path(FSUtils.getTableDir(hbaseRootDir, tableName),
881 new Path(hri.getEncodedName(), "recovered.edits")));
882 int editCount = 0;
883 for (FileStatus fileStatus : listStatus1) {
884 editCount = Integer.parseInt(fileStatus.getPath().getName());
885 }
886
887 assertEquals(
888 "The sequence number of the recoverd.edits and the current edit seq should be same",
889 lastestSeqNumber, editCount);
890 }
891
892 static class MockHLog extends FSHLog {
893 boolean doCompleteCacheFlush = false;
894
895 public MockHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) throws IOException {
896 super(fs, rootDir, logName, conf);
897 }
898
899 @Override
900 public void completeCacheFlush(byte[] encodedRegionName) {
901 if (!doCompleteCacheFlush) {
902 return;
903 }
904 super.completeCacheFlush(encodedRegionName);
905 }
906 }
907
908 private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
909 HTableDescriptor htd = new HTableDescriptor(tableName);
910 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
911 htd.addFamily(a);
912 return htd;
913 }
914
915 private MockHLog createMockWAL(Configuration conf) throws IOException {
916 MockHLog wal = new MockHLog(FileSystem.get(conf), hbaseRootDir, logName, conf);
917
918
919 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
920 return wal;
921 }
922
923
924
925 class TestFlusher implements FlushRequester {
926 private HRegion r;
927
928 @Override
929 public void requestFlush(HRegion region) {
930 try {
931 r.flushcache();
932 } catch (IOException e) {
933 throw new RuntimeException("Exception flushing", e);
934 }
935 }
936
937 @Override
938 public void requestDelayedFlush(HRegion region, long when) {
939
940
941 }
942 }
943
944 private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
945 final byte[] family, final int count, EnvironmentEdge ee, final HLog wal,
946 final HTableDescriptor htd, final AtomicLong sequenceId)
947 throws IOException {
948 String familyStr = Bytes.toString(family);
949 for (int j = 0; j < count; j++) {
950 byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
951 byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
952 WALEdit edit = new WALEdit();
953 edit.add(new KeyValue(rowName, family, qualifierBytes,
954 ee.currentTimeMillis(), columnBytes));
955 wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd, sequenceId);
956 }
957 }
958
959 private void addRegionEdits (final byte [] rowName, final byte [] family,
960 final int count, EnvironmentEdge ee, final HRegion r,
961 final String qualifierPrefix)
962 throws IOException {
963 for (int j = 0; j < count; j++) {
964 byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
965 Put p = new Put(rowName);
966 p.add(family, qualifier, ee.currentTimeMillis(), rowName);
967 r.put(p);
968 }
969 }
970
971
972
973
974
975
976 private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
977 return new HRegionInfo(tableName, null, null, false);
978 }
979
980
981
982
983
984
985
986 private Path runWALSplit(final Configuration c) throws IOException {
987 List<Path> splits = HLogSplitter.split(
988 hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c);
989
990 assertEquals("splits=" + splits, 1, splits.size());
991
992 assertTrue(fs.exists(splits.get(0)));
993 LOG.info("Split file=" + splits.get(0));
994 return splits.get(0);
995 }
996
997
998
999
1000
1001
1002 private HLog createWAL(final Configuration c) throws IOException {
1003 HLog wal = HLogFactory.createHLog(FileSystem.get(c),
1004 hbaseRootDir, logName, c);
1005
1006
1007 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
1008 return wal;
1009 }
1010
1011 private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
1012 HTableDescriptor htd = new HTableDescriptor(tableName);
1013 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
1014 htd.addFamily(a);
1015 HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
1016 htd.addFamily(b);
1017 HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
1018 htd.addFamily(c);
1019 return htd;
1020 }
1021
1022 }
1023