1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver.wal;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23 import static org.junit.Assert.fail;
24
25 import java.io.IOException;
26 import java.security.PrivilegedExceptionAction;
27 import java.util.ArrayList;
28 import java.util.List;
29 import java.util.SortedSet;
30 import java.util.concurrent.atomic.AtomicBoolean;
31 import java.util.concurrent.atomic.AtomicInteger;
32 import java.util.concurrent.atomic.AtomicLong;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.Cell;
41 import org.apache.hadoop.hbase.HBaseConfiguration;
42 import org.apache.hadoop.hbase.HBaseTestingUtility;
43 import org.apache.hadoop.hbase.HColumnDescriptor;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HTableDescriptor;
47 import org.apache.hadoop.hbase.KeyValue;
48 import org.apache.hadoop.hbase.MasterNotRunningException;
49 import org.apache.hadoop.hbase.MediumTests;
50 import org.apache.hadoop.hbase.MiniHBaseCluster;
51 import org.apache.hadoop.hbase.ServerName;
52 import org.apache.hadoop.hbase.TableName;
53 import org.apache.hadoop.hbase.ZooKeeperConnectionException;
54 import org.apache.hadoop.hbase.client.Delete;
55 import org.apache.hadoop.hbase.client.Get;
56 import org.apache.hadoop.hbase.client.HTable;
57 import org.apache.hadoop.hbase.client.Put;
58 import org.apache.hadoop.hbase.client.Result;
59 import org.apache.hadoop.hbase.client.ResultScanner;
60 import org.apache.hadoop.hbase.client.Scan;
61 import org.apache.hadoop.hbase.io.hfile.HFile;
62 import org.apache.hadoop.hbase.io.hfile.HFileContext;
63 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
64 import org.apache.hadoop.hbase.master.HMaster;
65 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
66 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
67 import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
68 import org.apache.hadoop.hbase.regionserver.FlushRequester;
69 import org.apache.hadoop.hbase.regionserver.HRegion;
70 import org.apache.hadoop.hbase.regionserver.HRegionServer;
71 import org.apache.hadoop.hbase.regionserver.RegionScanner;
72 import org.apache.hadoop.hbase.regionserver.RegionServerServices;
73 import org.apache.hadoop.hbase.regionserver.Store;
74 import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
75 import org.apache.hadoop.hbase.security.User;
76 import org.apache.hadoop.hbase.util.Bytes;
77 import org.apache.hadoop.hbase.util.EnvironmentEdge;
78 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
79 import org.apache.hadoop.hbase.util.FSUtils;
80 import org.apache.hadoop.hbase.util.Pair;
81 import org.junit.After;
82 import org.junit.AfterClass;
83 import org.junit.Before;
84 import org.junit.BeforeClass;
85 import org.junit.Test;
86 import org.junit.experimental.categories.Category;
87 import org.mockito.Mockito;
88
89
90
91
92 @Category(MediumTests.class)
93 public class TestWALReplay {
94 public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
95 static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
96 private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
97 private Path hbaseRootDir = null;
98 private String logName;
99 private Path oldLogDir;
100 private Path logDir;
101 private FileSystem fs;
102 private Configuration conf;
103
104 @BeforeClass
105 public static void setUpBeforeClass() throws Exception {
106 Configuration conf = TEST_UTIL.getConfiguration();
107 conf.setBoolean("dfs.support.append", true);
108
109 conf.setInt("dfs.client.block.recovery.retries", 2);
110 TEST_UTIL.startMiniCluster(3);
111 Path hbaseRootDir =
112 TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
113 LOG.info("hbase.rootdir=" + hbaseRootDir);
114 FSUtils.setRootDir(conf, hbaseRootDir);
115 }
116
117 @AfterClass
118 public static void tearDownAfterClass() throws Exception {
119 TEST_UTIL.shutdownMiniCluster();
120 }
121
122 @Before
123 public void setUp() throws Exception {
124 this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
125 this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
126 this.hbaseRootDir = FSUtils.getRootDir(this.conf);
127 this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
128 this.logName = HConstants.HREGION_LOGDIR_NAME;
129 this.logDir = new Path(this.hbaseRootDir, logName);
130 if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
131 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
132 }
133 }
134
135 @After
136 public void tearDown() throws Exception {
137 TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
138 }
139
140
141
142
143 private void deleteDir(final Path p) throws IOException {
144 if (this.fs.exists(p)) {
145 if (!this.fs.delete(p, true)) {
146 throw new IOException("Failed remove of " + p);
147 }
148 }
149 }
150
151
152
153
154
155 @Test
156 public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
157 final TableName tableName =
158 TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
159 byte[] family1 = Bytes.toBytes("cf1");
160 byte[] family2 = Bytes.toBytes("cf2");
161 byte[] qualifier = Bytes.toBytes("q");
162 byte[] value = Bytes.toBytes("testV");
163 byte[][] familys = { family1, family2 };
164 TEST_UTIL.createTable(tableName, familys);
165 HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
166 Put put = new Put(Bytes.toBytes("r1"));
167 put.add(family1, qualifier, value);
168 htable.put(put);
169 ResultScanner resultScanner = htable.getScanner(new Scan());
170 int count = 0;
171 while (resultScanner.next() != null) {
172 count++;
173 }
174 resultScanner.close();
175 assertEquals(1, count);
176
177 MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
178 List<HRegion> regions = hbaseCluster.getRegions(tableName);
179 assertEquals(1, regions.size());
180
181
182 HRegion destRegion = regions.get(0);
183 int originServerNum = hbaseCluster
184 .getServerWith(destRegion.getRegionName());
185 assertTrue("Please start more than 1 regionserver", hbaseCluster
186 .getRegionServerThreads().size() > 1);
187 int destServerNum = 0;
188 while (destServerNum == originServerNum) {
189 destServerNum++;
190 }
191 HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
192 HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
193
194 moveRegionAndWait(destRegion, destServer);
195
196
197 Delete del = new Delete(Bytes.toBytes("r1"));
198 htable.delete(del);
199 resultScanner = htable.getScanner(new Scan());
200 count = 0;
201 while (resultScanner.next() != null) {
202 count++;
203 }
204 resultScanner.close();
205 assertEquals(0, count);
206
207
208 destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
209
210 for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
211 .getStores().values()) {
212 store.triggerMajorCompaction();
213 }
214 destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
215
216
217 moveRegionAndWait(destRegion, originServer);
218
219 originServer.abort("testing");
220
221
222 Result result = htable.get(new Get(Bytes.toBytes("r1")));
223 if (result != null) {
224 assertTrue("Row is deleted, but we get" + result.toString(),
225 (result == null) || result.isEmpty());
226 }
227 resultScanner.close();
228 }
229
230 private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
231 throws InterruptedException, MasterNotRunningException,
232 ZooKeeperConnectionException, IOException {
233 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
234 TEST_UTIL.getHBaseAdmin().move(
235 destRegion.getRegionInfo().getEncodedNameAsBytes(),
236 Bytes.toBytes(destServer.getServerName().getServerName()));
237 while (true) {
238 ServerName serverName = master.getAssignmentManager()
239 .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
240 if (serverName != null && serverName.equals(destServer.getServerName())) {
241 TEST_UTIL.assertRegionOnServer(
242 destRegion.getRegionInfo(), serverName, 200);
243 break;
244 }
245 Thread.sleep(10);
246 }
247 }
248
249
250
251
252
253
254 @Test
255 public void test2727() throws Exception {
256
257
258 final TableName tableName =
259 TableName.valueOf("test2727");
260 HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
261 Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
262 deleteDir(basedir);
263
264 HTableDescriptor htd = createBasic3FamilyHTD(tableName);
265 HRegion region2 = HRegion.createHRegion(hri,
266 hbaseRootDir, this.conf, htd);
267 HRegion.closeHRegion(region2);
268 final byte [] rowName = tableName.getName();
269
270 HLog wal1 = createWAL(this.conf);
271
272 final int countPerFamily = 1000;
273 final AtomicLong sequenceId = new AtomicLong(1);
274 for (HColumnDescriptor hcd: htd.getFamilies()) {
275 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
276 wal1, htd, sequenceId);
277 }
278 wal1.close();
279 runWALSplit(this.conf);
280
281 HLog wal2 = createWAL(this.conf);
282
283 for (HColumnDescriptor hcd: htd.getFamilies()) {
284 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
285 ee, wal2, htd, sequenceId);
286 }
287 wal2.close();
288 runWALSplit(this.conf);
289
290 HLog wal3 = createWAL(this.conf);
291 try {
292 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
293 long seqid = region.getOpenSeqNum();
294
295
296 assertTrue(seqid > sequenceId.get());
297 assertEquals(seqid - 1, sequenceId.get());
298 LOG.debug("region.getOpenSeqNum(): " + region.getOpenSeqNum() + ", wal3.id: "
299 + sequenceId.get());
300
301
302 region.close();
303 } finally {
304 wal3.closeAndDelete();
305 }
306 }
307
308
309
310
311
312
313
314
315
316
317 @Test
318 public void testRegionMadeOfBulkLoadedFilesOnly()
319 throws IOException, SecurityException, IllegalArgumentException,
320 NoSuchFieldException, IllegalAccessException, InterruptedException {
321 final TableName tableName =
322 TableName.valueOf("testReplayEditsWrittenViaHRegion");
323 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
324 final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
325 deleteDir(basedir);
326 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
327 HRegion region2 = HRegion.createHRegion(hri,
328 hbaseRootDir, this.conf, htd);
329 HRegion.closeHRegion(region2);
330 HLog wal = createWAL(this.conf);
331 HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
332 Path f = new Path(basedir, "hfile");
333 HFileContext context = new HFileContextBuilder().build();
334 HFile.Writer writer =
335 HFile.getWriterFactoryNoCache(conf).withPath(fs, f)
336 .withFileContext(context).create();
337 byte [] family = htd.getFamilies().iterator().next().getName();
338 byte [] row = tableName.getName();
339 writer.append(new KeyValue(row, family, family, row));
340 writer.close();
341 List <Pair<byte[],String>> hfs= new ArrayList<Pair<byte[],String>>(1);
342 hfs.add(Pair.newPair(family, f.toString()));
343 region.bulkLoadHFiles(hfs, true);
344
345 region.put((new Put(row)).add(family, family, family));
346 wal.sync();
347
348
349 final Configuration newConf = HBaseConfiguration.create(this.conf);
350 User user = HBaseTestingUtility.getDifferentUser(newConf,
351 tableName.getNameAsString());
352 user.runAs(new PrivilegedExceptionAction() {
353 public Object run() throws Exception {
354 runWALSplit(newConf);
355 HLog wal2 = createWAL(newConf);
356
357 HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
358 hbaseRootDir, hri, htd, wal2);
359 long seqid2 = region2.getOpenSeqNum();
360 assertTrue(seqid2 > -1);
361
362
363 region2.close();
364 wal2.closeAndDelete();
365 return null;
366 }
367 });
368 }
369
370
371
372
373
374
375
376
377
378
379 @Test
380 public void testReplayEditsWrittenViaHRegion()
381 throws IOException, SecurityException, IllegalArgumentException,
382 NoSuchFieldException, IllegalAccessException, InterruptedException {
383 final TableName tableName =
384 TableName.valueOf("testReplayEditsWrittenViaHRegion");
385 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
386 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
387 deleteDir(basedir);
388 final byte[] rowName = tableName.getName();
389 final int countPerFamily = 10;
390 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
391 HRegion region3 = HRegion.createHRegion(hri,
392 hbaseRootDir, this.conf, htd);
393 HRegion.closeHRegion(region3);
394
395
396
397 HLog wal = createWAL(this.conf);
398 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
399 long seqid = region.getOpenSeqNum();
400 boolean first = true;
401 for (HColumnDescriptor hcd: htd.getFamilies()) {
402 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
403 if (first ) {
404
405 region.flushcache();
406 first = false;
407 }
408 }
409
410 final Get g = new Get(rowName);
411 Result result = region.get(g);
412 assertEquals(countPerFamily * htd.getFamilies().size(),
413 result.size());
414
415
416
417 region.close(true);
418 wal.close();
419 runWALSplit(this.conf);
420 HLog wal2 = createWAL(this.conf);
421 HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
422 long seqid2 = region2.getOpenSeqNum();
423 assertTrue(seqid + result.size() < seqid2);
424 final Result result1b = region2.get(g);
425 assertEquals(result.size(), result1b.size());
426
427
428
429
430 for (HColumnDescriptor hcd: htd.getFamilies()) {
431 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
432 }
433
434 final Result result2 = region2.get(g);
435 assertEquals(2 * result.size(), result2.size());
436 wal2.sync();
437
438
439 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal2).getOutputStream(), 1);
440 final Configuration newConf = HBaseConfiguration.create(this.conf);
441 User user = HBaseTestingUtility.getDifferentUser(newConf,
442 tableName.getNameAsString());
443 user.runAs(new PrivilegedExceptionAction() {
444 public Object run() throws Exception {
445 runWALSplit(newConf);
446 FileSystem newFS = FileSystem.get(newConf);
447
448 HLog wal3 = createWAL(newConf);
449 final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
450 HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
451 @Override
452 protected boolean restoreEdit(Store s, KeyValue kv) {
453 boolean b = super.restoreEdit(s, kv);
454 countOfRestoredEdits.incrementAndGet();
455 return b;
456 }
457 };
458 long seqid3 = region3.initialize();
459 Result result3 = region3.get(g);
460
461 assertEquals(result2.size(), result3.size());
462 assertEquals(htd.getFamilies().size() * countPerFamily,
463 countOfRestoredEdits.get());
464
465
466 region3.close();
467 wal3.closeAndDelete();
468 return null;
469 }
470 });
471 }
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491 @Test
492 public void testReplayEditsAfterPartialFlush()
493 throws IOException, SecurityException, IllegalArgumentException,
494 NoSuchFieldException, IllegalAccessException, InterruptedException {
495 final TableName tableName =
496 TableName.valueOf("testReplayEditsWrittenViaHRegion");
497 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
498 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
499 deleteDir(basedir);
500 final byte[] rowName = tableName.getName();
501 final int countPerFamily = 10;
502 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
503 HRegion region3 = HRegion.createHRegion(hri,
504 hbaseRootDir, this.conf, htd);
505 HRegion.closeHRegion(region3);
506
507
508
509 HLog wal = createWAL(this.conf);
510 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
511 long seqid = region.getOpenSeqNum();
512 for (HColumnDescriptor hcd: htd.getFamilies()) {
513 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
514 }
515
516
517 final Get g = new Get(rowName);
518 Result result = region.get(g);
519 assertEquals(countPerFamily * htd.getFamilies().size(),
520 result.size());
521
522
523 region.flushcache();
524 region.close(true);
525 wal.close();
526
527
528
529
530
531 int cf_count = 0;
532 for (HColumnDescriptor hcd: htd.getFamilies()) {
533 cf_count++;
534 if (cf_count == 2) {
535 region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
536 }
537 }
538
539
540
541 runWALSplit(this.conf);
542 HLog wal2 = createWAL(this.conf);
543 HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
544 long seqid2 = region2.getOpenSeqNum();
545 assertTrue(seqid + result.size() < seqid2);
546
547 final Result result1b = region2.get(g);
548 assertEquals(result.size(), result1b.size());
549 }
550
551
552
553
554 public static class CustomStoreFlusher extends DefaultStoreFlusher {
555
556 static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
557
558 public CustomStoreFlusher(Configuration conf, Store store) {
559 super(conf, store);
560 }
561 @Override
562 public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
563 TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status)
564 throws IOException {
565 if (throwExceptionWhenFlushing.get()) {
566 throw new IOException("Simulated exception by tests");
567 }
568 return super.flushSnapshot(snapshot, cacheFlushId, snapshotTimeRangeTracker,
569 flushedSize, status);
570 }
571
572 };
573
574
575
576
577
578
579
580 @Test
581 public void testReplayEditsAfterAbortingFlush() throws IOException {
582 final TableName tableName =
583 TableName.valueOf("testReplayEditsAfterAbortingFlush");
584 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
585 final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
586 deleteDir(basedir);
587 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
588 HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
589 region3.close();
590 region3.getLog().closeAndDelete();
591
592
593
594 HLog wal = createWAL(this.conf);
595 RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
596 Mockito.doReturn(false).when(rsServices).isAborted();
597 Configuration customConf = new Configuration(this.conf);
598 customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
599 CustomStoreFlusher.class.getName());
600 HRegion region =
601 HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
602 int writtenRowCount = 10;
603 List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
604 htd.getFamilies());
605 for (int i = 0; i < writtenRowCount; i++) {
606 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
607 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
608 Bytes.toBytes("val"));
609 region.put(put);
610 }
611
612
613 RegionScanner scanner = region.getScanner(new Scan());
614 assertEquals(writtenRowCount, getScannedCount(scanner));
615
616
617 CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
618 try {
619 region.flushcache();
620 fail("Injected exception hasn't been thrown");
621 } catch (Throwable t) {
622 LOG.info("Expected simulated exception when flushing region,"
623 + t.getMessage());
624
625 Mockito.doReturn(true).when(rsServices).isAborted();
626 }
627
628 int moreRow = 10;
629 for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
630 Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
631 put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
632 Bytes.toBytes("val"));
633 region.put(put);
634 }
635 writtenRowCount += moreRow;
636
637 CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
638 try {
639 region.flushcache();
640 } catch (IOException t) {
641 LOG.info("Expected exception when flushing region because server is stopped,"
642 + t.getMessage());
643 }
644
645 region.close(true);
646 wal.close();
647
648
649 runWALSplit(this.conf);
650 HLog wal2 = createWAL(this.conf);
651 Mockito.doReturn(false).when(rsServices).isAborted();
652 HRegion region2 =
653 HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
654 scanner = region2.getScanner(new Scan());
655 assertEquals(writtenRowCount, getScannedCount(scanner));
656 }
657
658 private int getScannedCount(RegionScanner scanner) throws IOException {
659 int scannedCount = 0;
660 List<Cell> results = new ArrayList<Cell>();
661 while (true) {
662 boolean existMore = scanner.next(results);
663 if (!results.isEmpty())
664 scannedCount++;
665 if (!existMore)
666 break;
667 results.clear();
668 }
669 return scannedCount;
670 }
671
672
673
674
675
676
677 @Test
678 public void testReplayEditsWrittenIntoWAL() throws Exception {
679 final TableName tableName =
680 TableName.valueOf("testReplayEditsWrittenIntoWAL");
681 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
682 final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
683 deleteDir(basedir);
684
685 final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
686 HRegion region2 = HRegion.createHRegion(hri,
687 hbaseRootDir, this.conf, htd);
688 HRegion.closeHRegion(region2);
689 final HLog wal = createWAL(this.conf);
690 final byte[] rowName = tableName.getName();
691 final byte[] regionName = hri.getEncodedNameAsBytes();
692 final AtomicLong sequenceId = new AtomicLong(1);
693
694
695 final int countPerFamily = 1000;
696 for (HColumnDescriptor hcd: htd.getFamilies()) {
697 addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
698 ee, wal, htd, sequenceId);
699 }
700
701
702 wal.startCacheFlush(regionName);
703 wal.completeCacheFlush(regionName);
704
705
706 WALEdit edit = new WALEdit();
707 long now = ee.currentTimeMillis();
708 edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
709 now, rowName));
710 wal.append(hri, tableName, edit, now, htd, sequenceId);
711
712
713 edit = new WALEdit();
714 now = ee.currentTimeMillis();
715 edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
716 KeyValue.Type.DeleteFamily));
717 wal.append(hri, tableName, edit, now, htd, sequenceId);
718
719
720 wal.sync();
721
722
723 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
724
725
726 final Configuration newConf = HBaseConfiguration.create(this.conf);
727 User user = HBaseTestingUtility.getDifferentUser(newConf,
728 ".replay.wal.secondtime");
729 user.runAs(new PrivilegedExceptionAction() {
730 public Object run() throws Exception {
731 runWALSplit(newConf);
732 FileSystem newFS = FileSystem.get(newConf);
733
734 newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
735
736 HLog newWal = createWAL(newConf);
737 final AtomicInteger flushcount = new AtomicInteger(0);
738 try {
739 final HRegion region =
740 new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
741 protected boolean internalFlushcache(
742 final HLog wal, final long myseqid, MonitoredTask status)
743 throws IOException {
744 LOG.info("InternalFlushCache Invoked");
745 boolean b = super.internalFlushcache(wal, myseqid,
746 Mockito.mock(MonitoredTask.class));
747 flushcount.incrementAndGet();
748 return b;
749 };
750 };
751 long seqid = region.initialize();
752
753 assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
754 assertTrue(seqid - 1 == sequenceId.get());
755
756 Get get = new Get(rowName);
757 Result result = region.get(get);
758
759 assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
760 result.size());
761 region.close();
762 } finally {
763 newWal.closeAndDelete();
764 }
765 return null;
766 }
767 });
768 }
769
770 @Test
771
772 public void testSequentialEditLogSeqNum() throws IOException {
773 final TableName tableName =
774 TableName.valueOf("testSequentialEditLogSeqNum");
775 final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
776 final Path basedir =
777 FSUtils.getTableDir(this.hbaseRootDir, tableName);
778 deleteDir(basedir);
779 final byte[] rowName = tableName.getName();
780 final int countPerFamily = 10;
781 final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
782
783
784 MockHLog wal = createMockWAL(this.conf);
785
786 HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
787 for (HColumnDescriptor hcd : htd.getFamilies()) {
788 addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
789 }
790
791
792
793 region.flushcache();
794 for (HColumnDescriptor hcd : htd.getFamilies()) {
795 addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
796 }
797 long lastestSeqNumber = region.getSequenceId().get();
798
799 wal.doCompleteCacheFlush = true;
800
801
802 wal.completeCacheFlush(hri.getEncodedNameAsBytes());
803 wal.close();
804 FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
805 HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0],
806 this.fs, this.conf, null, null, null);
807 FileStatus[] listStatus1 = this.fs.listStatus(
808 new Path(FSUtils.getTableDir(hbaseRootDir, tableName),
809 new Path(hri.getEncodedName(), "recovered.edits")));
810 int editCount = 0;
811 for (FileStatus fileStatus : listStatus1) {
812 editCount = Integer.parseInt(fileStatus.getPath().getName());
813 }
814
815 assertEquals(
816 "The sequence number of the recoverd.edits and the current edit seq should be same",
817 lastestSeqNumber, editCount);
818 }
819
820 static class MockHLog extends FSHLog {
821 boolean doCompleteCacheFlush = false;
822
823 public MockHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) throws IOException {
824 super(fs, rootDir, logName, conf);
825 }
826
827 @Override
828 public void completeCacheFlush(byte[] encodedRegionName) {
829 if (!doCompleteCacheFlush) {
830 return;
831 }
832 super.completeCacheFlush(encodedRegionName);
833 }
834 }
835
836 private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
837 HTableDescriptor htd = new HTableDescriptor(tableName);
838 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
839 htd.addFamily(a);
840 return htd;
841 }
842
843 private MockHLog createMockWAL(Configuration conf) throws IOException {
844 MockHLog wal = new MockHLog(FileSystem.get(conf), hbaseRootDir, logName, conf);
845
846
847 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
848 return wal;
849 }
850
851
852
853 class TestFlusher implements FlushRequester {
854 private HRegion r;
855
856 @Override
857 public void requestFlush(HRegion region) {
858 try {
859 r.flushcache();
860 } catch (IOException e) {
861 throw new RuntimeException("Exception flushing", e);
862 }
863 }
864
865 @Override
866 public void requestDelayedFlush(HRegion region, long when) {
867
868
869 }
870 }
871
872 private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
873 final byte[] family, final int count, EnvironmentEdge ee, final HLog wal,
874 final HTableDescriptor htd, final AtomicLong sequenceId)
875 throws IOException {
876 String familyStr = Bytes.toString(family);
877 for (int j = 0; j < count; j++) {
878 byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
879 byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
880 WALEdit edit = new WALEdit();
881 edit.add(new KeyValue(rowName, family, qualifierBytes,
882 ee.currentTimeMillis(), columnBytes));
883 wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd, sequenceId);
884 }
885 }
886
887 private void addRegionEdits (final byte [] rowName, final byte [] family,
888 final int count, EnvironmentEdge ee, final HRegion r,
889 final String qualifierPrefix)
890 throws IOException {
891 for (int j = 0; j < count; j++) {
892 byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
893 Put p = new Put(rowName);
894 p.add(family, qualifier, ee.currentTimeMillis(), rowName);
895 r.put(p);
896 }
897 }
898
899
900
901
902
903
904 private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
905 return new HRegionInfo(tableName, null, null, false);
906 }
907
908
909
910
911
912
913
914 private Path runWALSplit(final Configuration c) throws IOException {
915 List<Path> splits = HLogSplitter.split(
916 hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c);
917
918 assertEquals("splits=" + splits, 1, splits.size());
919
920 assertTrue(fs.exists(splits.get(0)));
921 LOG.info("Split file=" + splits.get(0));
922 return splits.get(0);
923 }
924
925
926
927
928
929
930 private HLog createWAL(final Configuration c) throws IOException {
931 HLog wal = HLogFactory.createHLog(FileSystem.get(c),
932 hbaseRootDir, logName, c);
933
934
935 HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
936 return wal;
937 }
938
939 private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
940 HTableDescriptor htd = new HTableDescriptor(tableName);
941 HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
942 htd.addFamily(a);
943 HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
944 htd.addFamily(b);
945 HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
946 htd.addFamily(c);
947 return htd;
948 }
949
950 }
951