View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  import static org.junit.Assert.fail;
24  
25  import java.io.IOException;
26  import java.security.PrivilegedExceptionAction;
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.SortedSet;
30  import java.util.concurrent.atomic.AtomicBoolean;
31  import java.util.concurrent.atomic.AtomicInteger;
32  import java.util.concurrent.atomic.AtomicLong;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileStatus;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.HBaseConfiguration;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.KeyValue;
48  import org.apache.hadoop.hbase.MasterNotRunningException;
49  import org.apache.hadoop.hbase.MediumTests;
50  import org.apache.hadoop.hbase.MiniHBaseCluster;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
53  import org.apache.hadoop.hbase.client.Delete;
54  import org.apache.hadoop.hbase.client.Get;
55  import org.apache.hadoop.hbase.client.HTable;
56  import org.apache.hadoop.hbase.client.Put;
57  import org.apache.hadoop.hbase.client.Result;
58  import org.apache.hadoop.hbase.client.ResultScanner;
59  import org.apache.hadoop.hbase.client.Scan;
60  import org.apache.hadoop.hbase.io.hfile.HFile;
61  import org.apache.hadoop.hbase.master.HMaster;
62  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
63  import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
64  import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
65  import org.apache.hadoop.hbase.regionserver.FlushRequester;
66  import org.apache.hadoop.hbase.regionserver.HRegion;
67  import org.apache.hadoop.hbase.regionserver.HRegionServer;
68  import org.apache.hadoop.hbase.regionserver.RegionScanner;
69  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
70  import org.apache.hadoop.hbase.regionserver.Store;
71  import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
72  import org.apache.hadoop.hbase.security.User;
73  import org.apache.hadoop.hbase.util.Bytes;
74  import org.apache.hadoop.hbase.util.FSUtils;
75  import org.apache.hadoop.hbase.util.EnvironmentEdge;
76  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
77  import org.apache.hadoop.hbase.util.Pair;
78  import org.junit.After;
79  import org.junit.AfterClass;
80  import org.junit.Before;
81  import org.junit.BeforeClass;
82  import org.junit.Test;
83  import org.junit.experimental.categories.Category;
84  import org.mockito.Mockito;
85  
86  /**
87   * Test replay of edits out of a WAL split.
88   */
89  @Category(MediumTests.class)
90  public class TestWALReplay {
91    public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
92    static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
93    private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
94    private Path hbaseRootDir = null;
95    private String logName;
96    private Path oldLogDir;
97    private Path logDir;
98    private FileSystem fs;
99    private Configuration conf;
100 
101   @BeforeClass
102   public static void setUpBeforeClass() throws Exception {
103     Configuration conf = TEST_UTIL.getConfiguration();
104     conf.setBoolean("dfs.support.append", true);
105     // The below config supported by 0.20-append and CDH3b2
106     conf.setInt("dfs.client.block.recovery.retries", 2);
107     TEST_UTIL.startMiniCluster(3);
108     Path hbaseRootDir =
109       TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
110     LOG.info("hbase.rootdir=" + hbaseRootDir);
111     FSUtils.setRootDir(conf, hbaseRootDir);
112   }
113 
114   @AfterClass
115   public static void tearDownAfterClass() throws Exception {
116     TEST_UTIL.shutdownMiniCluster();
117   }
118 
119   @Before
120   public void setUp() throws Exception {
121     this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
122     this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
123     this.hbaseRootDir = FSUtils.getRootDir(this.conf);
124     this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
125     this.logName = HConstants.HREGION_LOGDIR_NAME;
126     this.logDir = new Path(this.hbaseRootDir, logName);
127     if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
128       TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
129     }
130   }
131 
132   @After
133   public void tearDown() throws Exception {
134     TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
135   }
136 
137   /*
138    * @param p Directory to cleanup
139    */
140   private void deleteDir(final Path p) throws IOException {
141     if (this.fs.exists(p)) {
142       if (!this.fs.delete(p, true)) {
143         throw new IOException("Failed remove of " + p);
144       }
145     }
146   }
147 
148   /**
149    * 
150    * @throws Exception
151    */
152   @Test
153   public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
154     final TableName tableName =
155         TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
156     byte[] family1 = Bytes.toBytes("cf1");
157     byte[] family2 = Bytes.toBytes("cf2");
158     byte[] qualifier = Bytes.toBytes("q");
159     byte[] value = Bytes.toBytes("testV");
160     byte[][] familys = { family1, family2 };
161     TEST_UTIL.createTable(tableName, familys);
162     HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
163     Put put = new Put(Bytes.toBytes("r1"));
164     put.add(family1, qualifier, value);
165     htable.put(put);
166     ResultScanner resultScanner = htable.getScanner(new Scan());
167     int count = 0;
168     while (resultScanner.next() != null) {
169       count++;
170     }
171     resultScanner.close();
172     assertEquals(1, count);
173 
174     MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
175     List<HRegion> regions = hbaseCluster.getRegions(tableName);
176     assertEquals(1, regions.size());
177 
178     // move region to another regionserver
179     HRegion destRegion = regions.get(0);
180     int originServerNum = hbaseCluster
181         .getServerWith(destRegion.getRegionName());
182     assertTrue("Please start more than 1 regionserver", hbaseCluster
183         .getRegionServerThreads().size() > 1);
184     int destServerNum = 0;
185     while (destServerNum == originServerNum) {
186       destServerNum++;
187     }
188     HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
189     HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
190     // move region to destination regionserver
191     moveRegionAndWait(destRegion, destServer);
192 
193     // delete the row
194     Delete del = new Delete(Bytes.toBytes("r1"));
195     htable.delete(del);
196     resultScanner = htable.getScanner(new Scan());
197     count = 0;
198     while (resultScanner.next() != null) {
199       count++;
200     }
201     resultScanner.close();
202     assertEquals(0, count);
203 
204     // flush region and make major compaction
205     destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
206     // wait to complete major compaction
207     for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
208         .getStores().values()) {
209       store.triggerMajorCompaction();
210     }
211     destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
212 
213     // move region to origin regionserver
214     moveRegionAndWait(destRegion, originServer);
215     // abort the origin regionserver
216     originServer.abort("testing");
217 
218     // see what we get
219     Result result = htable.get(new Get(Bytes.toBytes("r1")));
220     if (result != null) {
221       assertTrue("Row is deleted, but we get" + result.toString(),
222           (result == null) || result.isEmpty());
223     }
224     resultScanner.close();
225   }
226 
227   private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
228       throws InterruptedException, MasterNotRunningException,
229       ZooKeeperConnectionException, IOException {
230     HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
231     TEST_UTIL.getHBaseAdmin().move(
232         destRegion.getRegionInfo().getEncodedNameAsBytes(),
233         Bytes.toBytes(destServer.getServerName().getServerName()));
234     while (true) {
235       ServerName serverName = master.getAssignmentManager()
236         .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
237       if (serverName != null && serverName.equals(destServer.getServerName())) {
238         TEST_UTIL.assertRegionOnServer(
239           destRegion.getRegionInfo(), serverName, 200);
240         break;
241       }
242       Thread.sleep(10);
243     }
244   }
245 
246   /**
247    * Tests for hbase-2727.
248    * @throws Exception
249    * @see https://issues.apache.org/jira/browse/HBASE-2727
250    */
251   @Test
252   public void test2727() throws Exception {
253     // Test being able to have > 1 set of edits in the recovered.edits directory.
254     // Ensure edits are replayed properly.
255     final TableName tableName =
256         TableName.valueOf("test2727");
257     HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
258     Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
259     deleteDir(basedir);
260 
261     HTableDescriptor htd = createBasic3FamilyHTD(tableName);
262     HRegion region2 = HRegion.createHRegion(hri,
263         hbaseRootDir, this.conf, htd);
264     HRegion.closeHRegion(region2);
265     final byte [] rowName = tableName.getName();
266 
267     HLog wal1 = createWAL(this.conf);
268     // Add 1k to each family.
269     final int countPerFamily = 1000;
270     for (HColumnDescriptor hcd: htd.getFamilies()) {
271       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
272           wal1, htd);
273     }
274     wal1.close();
275     runWALSplit(this.conf);
276 
277     HLog wal2 = createWAL(this.conf);
278     // Up the sequenceid so that these edits are after the ones added above.
279     wal2.setSequenceNumber(wal1.getSequenceNumber());
280     // Add 1k to each family.
281     for (HColumnDescriptor hcd: htd.getFamilies()) {
282       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
283           ee, wal2, htd);
284     }
285     wal2.close();
286     runWALSplit(this.conf);
287 
288     HLog wal3 = createWAL(this.conf);
289     wal3.setSequenceNumber(wal2.getSequenceNumber());
290     try {
291       long wal3SeqId = wal3.getSequenceNumber();
292       HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
293       long seqid = region.getOpenSeqNum();
294       assertTrue(seqid > wal3SeqId);
295 
296       // TODO: Scan all.
297       region.close();
298     } finally {
299       wal3.closeAndDelete();
300     }
301   }
302 
303   /**
304    * Test case of HRegion that is only made out of bulk loaded files.  Assert
305    * that we don't 'crash'.
306    * @throws IOException
307    * @throws IllegalAccessException
308    * @throws NoSuchFieldException
309    * @throws IllegalArgumentException
310    * @throws SecurityException
311    */
312   @Test
313   public void testRegionMadeOfBulkLoadedFilesOnly()
314   throws IOException, SecurityException, IllegalArgumentException,
315       NoSuchFieldException, IllegalAccessException, InterruptedException {
316     final TableName tableName =
317         TableName.valueOf("testReplayEditsWrittenViaHRegion");
318     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
319     final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
320     deleteDir(basedir);
321     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
322     HRegion region2 = HRegion.createHRegion(hri,
323         hbaseRootDir, this.conf, htd);
324     HRegion.closeHRegion(region2);
325     HLog wal = createWAL(this.conf);
326     HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
327     Path f =  new Path(basedir, "hfile");
328     HFile.Writer writer =
329       HFile.getWriterFactoryNoCache(conf).withPath(fs, f).create();
330     byte [] family = htd.getFamilies().iterator().next().getName();
331     byte [] row = tableName.getName();
332     writer.append(new KeyValue(row, family, family, row));
333     writer.close();
334     List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
335     hfs.add(Pair.newPair(family, f.toString()));
336     region.bulkLoadHFiles(hfs, true);
337     // Add an edit so something in the WAL
338     region.put((new Put(row)).add(family, family, family));
339     wal.sync();
340 
341     // Now 'crash' the region by stealing its wal
342     final Configuration newConf = HBaseConfiguration.create(this.conf);
343     User user = HBaseTestingUtility.getDifferentUser(newConf,
344         tableName.getNameAsString());
345     user.runAs(new PrivilegedExceptionAction() {
346       public Object run() throws Exception {
347         runWALSplit(newConf);
348         HLog wal2 = createWAL(newConf);
349 
350         HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
351           hbaseRootDir, hri, htd, wal2);
352         long seqid2 = region2.getOpenSeqNum();
353         assertTrue(seqid2 > -1);
354 
355         // I can't close wal1.  Its been appropriated when we split.
356         region2.close();
357         wal2.closeAndDelete();
358         return null;
359       }
360     });
361   }
362 
363   /**
364    * Test writing edits into an HRegion, closing it, splitting logs, opening
365    * Region again.  Verify seqids.
366    * @throws IOException
367    * @throws IllegalAccessException
368    * @throws NoSuchFieldException
369    * @throws IllegalArgumentException
370    * @throws SecurityException
371    */
372   @Test
373   public void testReplayEditsWrittenViaHRegion()
374   throws IOException, SecurityException, IllegalArgumentException,
375       NoSuchFieldException, IllegalAccessException, InterruptedException {
376     final TableName tableName =
377         TableName.valueOf("testReplayEditsWrittenViaHRegion");
378     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
379     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
380     deleteDir(basedir);
381     final byte[] rowName = tableName.getName();
382     final int countPerFamily = 10;
383     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
384     HRegion region3 = HRegion.createHRegion(hri,
385             hbaseRootDir, this.conf, htd);
386     HRegion.closeHRegion(region3);
387     // Write countPerFamily edits into the three families.  Do a flush on one
388     // of the families during the load of edits so its seqid is not same as
389     // others to test we do right thing when different seqids.
390     HLog wal = createWAL(this.conf);
391     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
392     long seqid = region.getOpenSeqNum();
393     // HRegionServer usually does this. It knows the largest seqid across all regions.
394     wal.setSequenceNumber(seqid);
395     boolean first = true;
396     for (HColumnDescriptor hcd: htd.getFamilies()) {
397       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
398       if (first ) {
399         // If first, so we have at least one family w/ different seqid to rest.
400         region.flushcache();
401         first = false;
402       }
403     }
404     // Now assert edits made it in.
405     final Get g = new Get(rowName);
406     Result result = region.get(g);
407     assertEquals(countPerFamily * htd.getFamilies().size(),
408       result.size());
409     // Now close the region (without flush), split the log, reopen the region and assert that
410     // replay of log has the correct effect, that our seqids are calculated correctly so
411     // all edits in logs are seen as 'stale'/old.
412     region.close(true);
413     wal.close();
414     runWALSplit(this.conf);
415     HLog wal2 = createWAL(this.conf);
416     HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
417     long seqid2 = region2.getOpenSeqNum();
418     // HRegionServer usually does this. It knows the largest seqid across all regions.
419     wal2.setSequenceNumber(seqid2);
420     assertTrue(seqid + result.size() < seqid2);
421     final Result result1b = region2.get(g);
422     assertEquals(result.size(), result1b.size());
423 
424     // Next test.  Add more edits, then 'crash' this region by stealing its wal
425     // out from under it and assert that replay of the log adds the edits back
426     // correctly when region is opened again.
427     for (HColumnDescriptor hcd: htd.getFamilies()) {
428       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
429     }
430     // Get count of edits.
431     final Result result2 = region2.get(g);
432     assertEquals(2 * result.size(), result2.size());
433     wal2.sync();
434     // Set down maximum recovery so we dfsclient doesn't linger retrying something
435     // long gone.
436     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal2).getOutputStream(), 1);
437     final Configuration newConf = HBaseConfiguration.create(this.conf);
438     User user = HBaseTestingUtility.getDifferentUser(newConf,
439       tableName.getNameAsString());
440     user.runAs(new PrivilegedExceptionAction() {
441       public Object run() throws Exception {
442         runWALSplit(newConf);
443         FileSystem newFS = FileSystem.get(newConf);
444         // Make a new wal for new region open.
445         HLog wal3 = createWAL(newConf);
446         final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
447         HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
448           @Override
449           protected boolean restoreEdit(Store s, KeyValue kv) {
450             boolean b = super.restoreEdit(s, kv);
451             countOfRestoredEdits.incrementAndGet();
452             return b;
453           }
454         };
455         long seqid3 = region3.initialize();
456         // HRegionServer usually does this. It knows the largest seqid across all regions.
457         wal3.setSequenceNumber(seqid3);
458         Result result3 = region3.get(g);
459         // Assert that count of cells is same as before crash.
460         assertEquals(result2.size(), result3.size());
461         assertEquals(htd.getFamilies().size() * countPerFamily,
462           countOfRestoredEdits.get());
463 
464         // I can't close wal1.  Its been appropriated when we split.
465         region3.close();
466         wal3.closeAndDelete();
467         return null;
468       }
469     });
470   }
471 
472   /**
473    * Test that we recover correctly when there is a failure in between the
474    * flushes. i.e. Some stores got flushed but others did not.
475    *
476    * Unfortunately, there is no easy hook to flush at a store level. The way
477    * we get around this is by flushing at the region level, and then deleting
478    * the recently flushed store file for one of the Stores. This would put us
479    * back in the situation where all but that store got flushed and the region
480    * died.
481    *
482    * We restart Region again, and verify that the edits were replayed.
483    *
484    * @throws IOException
485    * @throws IllegalAccessException
486    * @throws NoSuchFieldException
487    * @throws IllegalArgumentException
488    * @throws SecurityException
489    */
490   @Test
491   public void testReplayEditsAfterPartialFlush()
492   throws IOException, SecurityException, IllegalArgumentException,
493       NoSuchFieldException, IllegalAccessException, InterruptedException {
494     final TableName tableName =
495         TableName.valueOf("testReplayEditsWrittenViaHRegion");
496     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
497     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
498     deleteDir(basedir);
499     final byte[] rowName = tableName.getName();
500     final int countPerFamily = 10;
501     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
502     HRegion region3 = HRegion.createHRegion(hri,
503             hbaseRootDir, this.conf, htd);
504     HRegion.closeHRegion(region3);
505     // Write countPerFamily edits into the three families.  Do a flush on one
506     // of the families during the load of edits so its seqid is not same as
507     // others to test we do right thing when different seqids.
508     HLog wal = createWAL(this.conf);
509     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
510     long seqid = region.getOpenSeqNum();
511     // HRegionServer usually does this. It knows the largest seqid across all regions.
512     wal.setSequenceNumber(seqid);
513     for (HColumnDescriptor hcd: htd.getFamilies()) {
514       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
515     }
516 
517     // Now assert edits made it in.
518     final Get g = new Get(rowName);
519     Result result = region.get(g);
520     assertEquals(countPerFamily * htd.getFamilies().size(),
521       result.size());
522 
523     // Let us flush the region
524     region.flushcache();
525     region.close(true);
526     wal.close();
527 
528     // delete the store files in the second column family to simulate a failure
529     // in between the flushcache();
530     // we have 3 families. killing the middle one ensures that taking the maximum
531     // will make us fail.
532     int cf_count = 0;
533     for (HColumnDescriptor hcd: htd.getFamilies()) {
534       cf_count++;
535       if (cf_count == 2) {
536         region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
537       }
538     }
539 
540 
541     // Let us try to split and recover
542     runWALSplit(this.conf);
543     HLog wal2 = createWAL(this.conf);
544     HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
545     long seqid2 = region2.getOpenSeqNum();
546     // HRegionServer usually does this. It knows the largest seqid across all regions.
547     wal2.setSequenceNumber(seqid2);
548     assertTrue(seqid + result.size() < seqid2);
549 
550     final Result result1b = region2.get(g);
551     assertEquals(result.size(), result1b.size());
552   }
553 
554 
555   // StoreFlusher implementation used in testReplayEditsAfterAbortingFlush.
556   // Only throws exception if throwExceptionWhenFlushing is set true.
557   public static class CustomStoreFlusher extends DefaultStoreFlusher {
558     // Switch between throw and not throw exception in flush
559     static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
560 
561     public CustomStoreFlusher(Configuration conf, Store store) {
562       super(conf, store);
563     }
564     @Override
565     public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
566         TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status)
567             throws IOException {
568       if (throwExceptionWhenFlushing.get()) {
569         throw new IOException("Simulated exception by tests");
570       }
571       return super.flushSnapshot(snapshot, cacheFlushId, snapshotTimeRangeTracker,
572           flushedSize, status);
573     }
574 
575   };
576 
577   /**
578    * Test that we could recover the data correctly after aborting flush. In the
579    * test, first we abort flush after writing some data, then writing more data
580    * and flush again, at last verify the data.
581    * @throws IOException
582    */
583   @Test
584   public void testReplayEditsAfterAbortingFlush() throws IOException {
585     final TableName tableName =
586         TableName.valueOf("testReplayEditsAfterAbortingFlush");
587     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
588     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
589     deleteDir(basedir);
590     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
591     HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
592     region3.close();
593     region3.getLog().closeAndDelete();
594     // Write countPerFamily edits into the three families. Do a flush on one
595     // of the families during the load of edits so its seqid is not same as
596     // others to test we do right thing when different seqids.
597     HLog wal = createWAL(this.conf);
598     RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
599     Mockito.doReturn(false).when(rsServices).isAborted();
600     Configuration customConf = new Configuration(this.conf);
601     customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
602         CustomStoreFlusher.class.getName());
603     HRegion region = new HRegion(basedir, wal, this.fs, customConf, hri, htd, rsServices);
604     long seqid = region.initialize();
605     // HRegionServer usually does this. It knows the largest seqid across all
606     // regions.
607     wal.setSequenceNumber(seqid);
608 
609     int writtenRowCount = 10;
610     List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
611         htd.getFamilies());
612     for (int i = 0; i < writtenRowCount; i++) {
613       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
614       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
615           Bytes.toBytes("val"));
616       region.put(put);
617     }
618 
619     // Now assert edits made it in.
620     RegionScanner scanner = region.getScanner(new Scan());
621     assertEquals(writtenRowCount, getScannedCount(scanner));
622 
623     // Let us flush the region
624     CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
625     try {
626       region.flushcache();
627       fail("Injected exception hasn't been thrown");
628     } catch (Throwable t) {
629       LOG.info("Expected simulated exception when flushing region,"
630           + t.getMessage());
631       // simulated to abort server
632       Mockito.doReturn(true).when(rsServices).isAborted();
633     }
634     // writing more data
635     int moreRow = 10;
636     for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
637       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
638       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
639           Bytes.toBytes("val"));
640       region.put(put);
641     }
642     writtenRowCount += moreRow;
643     // call flush again
644     CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
645     try {
646       region.flushcache();
647     } catch (IOException t) {
648       LOG.info("Expected exception when flushing region because server is stopped,"
649           + t.getMessage());
650     }
651 
652     region.close(true);
653     wal.close();
654 
655     // Let us try to split and recover
656     runWALSplit(this.conf);
657     HLog wal2 = createWAL(this.conf);
658     Mockito.doReturn(false).when(rsServices).isAborted();
659     HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd,
660         rsServices);
661     long seqid2 = region2.initialize();
662     // HRegionServer usually does this. It knows the largest seqid across all
663     // regions.
664     wal2.setSequenceNumber(seqid2);
665 
666     scanner = region2.getScanner(new Scan());
667     assertEquals(writtenRowCount, getScannedCount(scanner));
668   }
669 
670   private int getScannedCount(RegionScanner scanner) throws IOException {
671     int scannedCount = 0;
672     List<KeyValue> results = new ArrayList<KeyValue>();
673     while (true) {
674       boolean existMore = scanner.next(results);
675       if (!results.isEmpty())
676         scannedCount++;
677       if (!existMore)
678         break;
679       results.clear();
680     }
681     return scannedCount;
682   }
683 
684   /**
685    * Create an HRegion with the result of a HLog split and test we only see the
686    * good edits
687    * @throws Exception
688    */
689   @Test
690   public void testReplayEditsWrittenIntoWAL() throws Exception {
691     final TableName tableName =
692         TableName.valueOf("testReplayEditsWrittenIntoWAL");
693     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
694     final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
695     deleteDir(basedir);
696 
697     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
698     HRegion region2 = HRegion.createHRegion(hri,
699             hbaseRootDir, this.conf, htd);
700     HRegion.closeHRegion(region2);
701     final HLog wal = createWAL(this.conf);
702     final byte[] rowName = tableName.getName();
703     final byte[] regionName = hri.getEncodedNameAsBytes();
704 
705     // Add 1k to each family.
706     final int countPerFamily = 1000;
707     for (HColumnDescriptor hcd: htd.getFamilies()) {
708       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
709           ee, wal, htd);
710     }
711 
712     // Add a cache flush, shouldn't have any effect
713     wal.startCacheFlush(regionName);
714     wal.completeCacheFlush(regionName);
715 
716     // Add an edit to another family, should be skipped.
717     WALEdit edit = new WALEdit();
718     long now = ee.currentTimeMillis();
719     edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
720       now, rowName));
721     wal.append(hri, tableName, edit, now, htd);
722 
723     // Delete the c family to verify deletes make it over.
724     edit = new WALEdit();
725     now = ee.currentTimeMillis();
726     edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
727       KeyValue.Type.DeleteFamily));
728     wal.append(hri, tableName, edit, now, htd);
729 
730     // Sync.
731     wal.sync();
732     // Set down maximum recovery so we dfsclient doesn't linger retrying something
733     // long gone.
734     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
735     // Make a new conf and a new fs for the splitter to run on so we can take
736     // over old wal.
737     final Configuration newConf = HBaseConfiguration.create(this.conf);
738     User user = HBaseTestingUtility.getDifferentUser(newConf,
739       ".replay.wal.secondtime");
740     user.runAs(new PrivilegedExceptionAction() {
741       public Object run() throws Exception {
742         runWALSplit(newConf);
743         FileSystem newFS = FileSystem.get(newConf);
744         // 100k seems to make for about 4 flushes during HRegion#initialize.
745         newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
746         // Make a new wal for new region.
747         HLog newWal = createWAL(newConf);
748         final AtomicInteger flushcount = new AtomicInteger(0);
749         try {
750           final HRegion region =
751               new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
752             protected boolean internalFlushcache(
753                 final HLog wal, final long myseqid, MonitoredTask status)
754             throws IOException {
755               LOG.info("InternalFlushCache Invoked");
756               boolean b = super.internalFlushcache(wal, myseqid,
757                   Mockito.mock(MonitoredTask.class));
758               flushcount.incrementAndGet();
759               return b;
760             };
761           };
762           long seqid = region.initialize();
763           // We flushed during init.
764           assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
765           assertTrue(seqid > wal.getSequenceNumber());
766 
767           Get get = new Get(rowName);
768           Result result = region.get(get);
769           // Make sure we only see the good edits
770           assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
771             result.size());
772           region.close();
773         } finally {
774           newWal.closeAndDelete();
775         }
776         return null;
777       }
778     });
779   }
780 
781   @Test
782   // the following test is for HBASE-6065
783   public void testSequentialEditLogSeqNum() throws IOException {
784     final TableName tableName =
785         TableName.valueOf("testSequentialEditLogSeqNum");
786     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
787     final Path basedir =
788         FSUtils.getTableDir(this.hbaseRootDir, tableName);
789     deleteDir(basedir);
790     final byte[] rowName = tableName.getName();
791     final int countPerFamily = 10;
792     final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
793 
794     // Mock the HLog
795     MockHLog wal = createMockWAL(this.conf);
796 
797     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
798     long seqid = region.getOpenSeqNum();
799     // HRegionServer usually does this. It knows the largest seqid across all
800     // regions.
801     wal.setSequenceNumber(seqid);
802     for (HColumnDescriptor hcd : htd.getFamilies()) {
803       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
804     }
805     // get the seq no after first set of entries.
806     long sequenceNumber = wal.getSequenceNumber();
807 
808     // Let us flush the region
809     // But this time completeflushcache is not yet done
810     region.flushcache();
811     for (HColumnDescriptor hcd : htd.getFamilies()) {
812       addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
813     }
814     long lastestSeqNumber = wal.getSequenceNumber();
815     // get the current seq no
816     wal.doCompleteCacheFlush = true;
817     // allow complete cache flush with the previous seq number got after first
818     // set of edits.
819     wal.completeCacheFlush(hri.getEncodedNameAsBytes());
820     wal.close();
821     FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
822     HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0],
823       this.fs, this.conf, null, null, null);
824     FileStatus[] listStatus1 = this.fs.listStatus(
825         new Path(FSUtils.getTableDir(hbaseRootDir, tableName),
826             new Path(hri.getEncodedName(), "recovered.edits")));
827     int editCount = 0;
828     for (FileStatus fileStatus : listStatus1) {
829       editCount = Integer.parseInt(fileStatus.getPath().getName());
830     }
831     // The sequence number should be same 
832     assertEquals(
833         "The sequence number of the recoverd.edits and the current edit seq should be same",
834         lastestSeqNumber, editCount);
835   }
836 
837   static class MockHLog extends FSHLog {
838     boolean doCompleteCacheFlush = false;
839 
840     public MockHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) throws IOException {
841       super(fs, rootDir, logName, conf);
842     }
843 
844     @Override
845     public void completeCacheFlush(byte[] encodedRegionName) {
846       if (!doCompleteCacheFlush) {
847         return;
848       }
849       super.completeCacheFlush(encodedRegionName);
850     }
851   }
852 
853   private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
854     HTableDescriptor htd = new HTableDescriptor(tableName);
855     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
856     htd.addFamily(a);
857     return htd;
858   }
859   
860   private MockHLog createMockWAL(Configuration conf) throws IOException {
861     MockHLog wal = new MockHLog(FileSystem.get(conf), hbaseRootDir, logName, conf);
862     // Set down maximum recovery so we dfsclient doesn't linger retrying something
863     // long gone.
864     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
865     return wal;
866   }
867 
868   // Flusher used in this test.  Keep count of how often we are called and
869   // actually run the flush inside here.
870   class TestFlusher implements FlushRequester {
871     private HRegion r;
872 
873     @Override
874     public void requestFlush(HRegion region) {
875       try {
876         r.flushcache();
877       } catch (IOException e) {
878         throw new RuntimeException("Exception flushing", e);
879       }
880     }
881 
882     @Override
883     public void requestDelayedFlush(HRegion region, long when) {
884       // TODO Auto-generated method stub
885       
886     }
887   }
888 
889   private void addWALEdits (final TableName tableName, final HRegionInfo hri,
890       final byte [] rowName, final byte [] family,
891       final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd)
892   throws IOException {
893     String familyStr = Bytes.toString(family);
894     for (int j = 0; j < count; j++) {
895       byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
896       byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
897       WALEdit edit = new WALEdit();
898       edit.add(new KeyValue(rowName, family, qualifierBytes,
899         ee.currentTimeMillis(), columnBytes));
900       wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd);
901     }
902   }
903 
904   private void addRegionEdits (final byte [] rowName, final byte [] family,
905       final int count, EnvironmentEdge ee, final HRegion r,
906       final String qualifierPrefix)
907   throws IOException {
908     for (int j = 0; j < count; j++) {
909       byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
910       Put p = new Put(rowName);
911       p.add(family, qualifier, ee.currentTimeMillis(), rowName);
912       r.put(p);
913     }
914   }
915 
916   /*
917    * Creates an HRI around an HTD that has <code>tableName</code> and three
918    * column families named 'a','b', and 'c'.
919    * @param tableName Name of table to use when we create HTableDescriptor.
920    */
921    private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
922     return new HRegionInfo(tableName, null, null, false);
923    }
924 
925   /*
926    * Run the split.  Verify only single split file made.
927    * @param c
928    * @return The single split file made
929    * @throws IOException
930    */
931   private Path runWALSplit(final Configuration c) throws IOException {
932     List<Path> splits = HLogSplitter.split(
933       hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c);
934     // Split should generate only 1 file since there's only 1 region
935     assertEquals("splits=" + splits, 1, splits.size());
936     // Make sure the file exists
937     assertTrue(fs.exists(splits.get(0)));
938     LOG.info("Split file=" + splits.get(0));
939     return splits.get(0);
940   }
941 
942   /*
943    * @param c
944    * @return WAL with retries set down from 5 to 1 only.
945    * @throws IOException
946    */
947   private HLog createWAL(final Configuration c) throws IOException {
948     HLog wal = HLogFactory.createHLog(FileSystem.get(c), 
949         hbaseRootDir, logName, c);
950     // Set down maximum recovery so we dfsclient doesn't linger retrying something
951     // long gone.
952     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
953     return wal;
954   }
955 
956   private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
957     HTableDescriptor htd = new HTableDescriptor(tableName);
958     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
959     htd.addFamily(a);
960     HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
961     htd.addFamily(b);
962     HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
963     htd.addFamily(c);
964     return htd;
965   }
966 
967 }
968