View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  import static org.junit.Assert.fail;
24  
25  import java.io.IOException;
26  import java.security.PrivilegedExceptionAction;
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.SortedSet;
30  import java.util.concurrent.atomic.AtomicBoolean;
31  import java.util.concurrent.atomic.AtomicInteger;
32  import java.util.concurrent.atomic.AtomicLong;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileStatus;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.Cell;
41  import org.apache.hadoop.hbase.HBaseConfiguration;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.KeyValue;
48  import org.apache.hadoop.hbase.MasterNotRunningException;
49  import org.apache.hadoop.hbase.MediumTests;
50  import org.apache.hadoop.hbase.MiniHBaseCluster;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.TableName;
53  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
54  import org.apache.hadoop.hbase.client.Delete;
55  import org.apache.hadoop.hbase.client.Get;
56  import org.apache.hadoop.hbase.client.HTable;
57  import org.apache.hadoop.hbase.client.Put;
58  import org.apache.hadoop.hbase.client.Result;
59  import org.apache.hadoop.hbase.client.ResultScanner;
60  import org.apache.hadoop.hbase.client.Scan;
61  import org.apache.hadoop.hbase.io.hfile.HFile;
62  import org.apache.hadoop.hbase.io.hfile.HFileContext;
63  import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
64  import org.apache.hadoop.hbase.master.HMaster;
65  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
66  import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
67  import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
68  import org.apache.hadoop.hbase.regionserver.FlushRequester;
69  import org.apache.hadoop.hbase.regionserver.HRegion;
70  import org.apache.hadoop.hbase.regionserver.HRegionServer;
71  import org.apache.hadoop.hbase.regionserver.RegionScanner;
72  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
73  import org.apache.hadoop.hbase.regionserver.Store;
74  import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
75  import org.apache.hadoop.hbase.security.User;
76  import org.apache.hadoop.hbase.util.Bytes;
77  import org.apache.hadoop.hbase.util.EnvironmentEdge;
78  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
79  import org.apache.hadoop.hbase.util.FSUtils;
80  import org.apache.hadoop.hbase.util.Pair;
81  import org.junit.After;
82  import org.junit.AfterClass;
83  import org.junit.Before;
84  import org.junit.BeforeClass;
85  import org.junit.Test;
86  import org.junit.experimental.categories.Category;
87  import org.mockito.Mockito;
88  
89  /**
90   * Test replay of edits out of a WAL split.
91   */
92  @Category(MediumTests.class)
93  public class TestWALReplay {
94    public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
95    static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
96    private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
97    private Path hbaseRootDir = null;
98    private String logName;
99    private Path oldLogDir;
100   private Path logDir;
101   private FileSystem fs;
102   private Configuration conf;
103 
104   @BeforeClass
105   public static void setUpBeforeClass() throws Exception {
106     Configuration conf = TEST_UTIL.getConfiguration();
107     conf.setBoolean("dfs.support.append", true);
108     // The below config supported by 0.20-append and CDH3b2
109     conf.setInt("dfs.client.block.recovery.retries", 2);
110     TEST_UTIL.startMiniCluster(3);
111     Path hbaseRootDir =
112       TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
113     LOG.info("hbase.rootdir=" + hbaseRootDir);
114     FSUtils.setRootDir(conf, hbaseRootDir);
115   }
116 
117   @AfterClass
118   public static void tearDownAfterClass() throws Exception {
119     TEST_UTIL.shutdownMiniCluster();
120   }
121 
122   @Before
123   public void setUp() throws Exception {
124     this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
125     this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
126     this.hbaseRootDir = FSUtils.getRootDir(this.conf);
127     this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
128     this.logName = HConstants.HREGION_LOGDIR_NAME;
129     this.logDir = new Path(this.hbaseRootDir, logName);
130     if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
131       TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
132     }
133   }
134 
135   @After
136   public void tearDown() throws Exception {
137     TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
138   }
139 
140   /*
141    * @param p Directory to cleanup
142    */
143   private void deleteDir(final Path p) throws IOException {
144     if (this.fs.exists(p)) {
145       if (!this.fs.delete(p, true)) {
146         throw new IOException("Failed remove of " + p);
147       }
148     }
149   }
150 
151   /**
152    * 
153    * @throws Exception
154    */
155   @Test
156   public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
157     final TableName tableName =
158         TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
159     byte[] family1 = Bytes.toBytes("cf1");
160     byte[] family2 = Bytes.toBytes("cf2");
161     byte[] qualifier = Bytes.toBytes("q");
162     byte[] value = Bytes.toBytes("testV");
163     byte[][] familys = { family1, family2 };
164     TEST_UTIL.createTable(tableName, familys);
165     HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
166     Put put = new Put(Bytes.toBytes("r1"));
167     put.add(family1, qualifier, value);
168     htable.put(put);
169     ResultScanner resultScanner = htable.getScanner(new Scan());
170     int count = 0;
171     while (resultScanner.next() != null) {
172       count++;
173     }
174     resultScanner.close();
175     assertEquals(1, count);
176 
177     MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
178     List<HRegion> regions = hbaseCluster.getRegions(tableName);
179     assertEquals(1, regions.size());
180 
181     // move region to another regionserver
182     HRegion destRegion = regions.get(0);
183     int originServerNum = hbaseCluster
184         .getServerWith(destRegion.getRegionName());
185     assertTrue("Please start more than 1 regionserver", hbaseCluster
186         .getRegionServerThreads().size() > 1);
187     int destServerNum = 0;
188     while (destServerNum == originServerNum) {
189       destServerNum++;
190     }
191     HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
192     HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
193     // move region to destination regionserver
194     moveRegionAndWait(destRegion, destServer);
195 
196     // delete the row
197     Delete del = new Delete(Bytes.toBytes("r1"));
198     htable.delete(del);
199     resultScanner = htable.getScanner(new Scan());
200     count = 0;
201     while (resultScanner.next() != null) {
202       count++;
203     }
204     resultScanner.close();
205     assertEquals(0, count);
206 
207     // flush region and make major compaction
208     destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
209     // wait to complete major compaction
210     for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
211         .getStores().values()) {
212       store.triggerMajorCompaction();
213     }
214     destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
215 
216     // move region to origin regionserver
217     moveRegionAndWait(destRegion, originServer);
218     // abort the origin regionserver
219     originServer.abort("testing");
220 
221     // see what we get
222     Result result = htable.get(new Get(Bytes.toBytes("r1")));
223     if (result != null) {
224       assertTrue("Row is deleted, but we get" + result.toString(),
225           (result == null) || result.isEmpty());
226     }
227     resultScanner.close();
228   }
229 
230   private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
231       throws InterruptedException, MasterNotRunningException,
232       ZooKeeperConnectionException, IOException {
233     HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
234     TEST_UTIL.getHBaseAdmin().move(
235         destRegion.getRegionInfo().getEncodedNameAsBytes(),
236         Bytes.toBytes(destServer.getServerName().getServerName()));
237     while (true) {
238       ServerName serverName = master.getAssignmentManager()
239         .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
240       if (serverName != null && serverName.equals(destServer.getServerName())) {
241         TEST_UTIL.assertRegionOnServer(
242           destRegion.getRegionInfo(), serverName, 200);
243         break;
244       }
245       Thread.sleep(10);
246     }
247   }
248 
249   /**
250    * Tests for hbase-2727.
251    * @throws Exception
252    * @see https://issues.apache.org/jira/browse/HBASE-2727
253    */
254   @Test
255   public void test2727() throws Exception {
256     // Test being able to have > 1 set of edits in the recovered.edits directory.
257     // Ensure edits are replayed properly.
258     final TableName tableName =
259         TableName.valueOf("test2727");
260     HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
261     Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
262     deleteDir(basedir);
263 
264     HTableDescriptor htd = createBasic3FamilyHTD(tableName);
265     HRegion region2 = HRegion.createHRegion(hri,
266         hbaseRootDir, this.conf, htd);
267     HRegion.closeHRegion(region2);
268     final byte [] rowName = tableName.getName();
269 
270     HLog wal1 = createWAL(this.conf);
271     // Add 1k to each family.
272     final int countPerFamily = 1000;
273     final AtomicLong sequenceId = new AtomicLong(1);
274     for (HColumnDescriptor hcd: htd.getFamilies()) {
275       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
276           wal1, htd, sequenceId);
277     }
278     wal1.close();
279     runWALSplit(this.conf);
280 
281     HLog wal2 = createWAL(this.conf);
282     // Add 1k to each family.
283     for (HColumnDescriptor hcd: htd.getFamilies()) {
284       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
285           ee, wal2, htd, sequenceId);
286     }
287     wal2.close();
288     runWALSplit(this.conf);
289 
290     HLog wal3 = createWAL(this.conf);
291     try {
292       HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
293       long seqid = region.getOpenSeqNum();
294       // The regions opens with sequenceId as 1. With 6k edits, its sequence number reaches 6k + 1.
295       // When opened, this region would apply 6k edits, and increment the sequenceId by 1
296       assertTrue(seqid > sequenceId.get());
297       assertEquals(seqid - 1, sequenceId.get());
298       LOG.debug("region.getOpenSeqNum(): " + region.getOpenSeqNum() + ", wal3.id: "
299           + sequenceId.get());
300 
301       // TODO: Scan all.
302       region.close();
303     } finally {
304       wal3.closeAndDelete();
305     }
306   }
307 
308   /**
309    * Test case of HRegion that is only made out of bulk loaded files.  Assert
310    * that we don't 'crash'.
311    * @throws IOException
312    * @throws IllegalAccessException
313    * @throws NoSuchFieldException
314    * @throws IllegalArgumentException
315    * @throws SecurityException
316    */
317   @Test
318   public void testRegionMadeOfBulkLoadedFilesOnly()
319   throws IOException, SecurityException, IllegalArgumentException,
320       NoSuchFieldException, IllegalAccessException, InterruptedException {
321     final TableName tableName =
322         TableName.valueOf("testReplayEditsWrittenViaHRegion");
323     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
324     final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
325     deleteDir(basedir);
326     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
327     HRegion region2 = HRegion.createHRegion(hri,
328         hbaseRootDir, this.conf, htd);
329     HRegion.closeHRegion(region2);
330     HLog wal = createWAL(this.conf);
331     HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
332     Path f =  new Path(basedir, "hfile");
333     HFileContext context = new HFileContextBuilder().build();
334     HFile.Writer writer =
335     HFile.getWriterFactoryNoCache(conf).withPath(fs, f)
336         .withFileContext(context).create();
337     byte [] family = htd.getFamilies().iterator().next().getName();
338     byte [] row = tableName.getName();
339     writer.append(new KeyValue(row, family, family, row));
340     writer.close();
341     List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
342     hfs.add(Pair.newPair(family, f.toString()));
343     region.bulkLoadHFiles(hfs, true);
344     // Add an edit so something in the WAL
345     region.put((new Put(row)).add(family, family, family));
346     wal.sync();
347 
348     // Now 'crash' the region by stealing its wal
349     final Configuration newConf = HBaseConfiguration.create(this.conf);
350     User user = HBaseTestingUtility.getDifferentUser(newConf,
351         tableName.getNameAsString());
352     user.runAs(new PrivilegedExceptionAction() {
353       public Object run() throws Exception {
354         runWALSplit(newConf);
355         HLog wal2 = createWAL(newConf);
356 
357         HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
358           hbaseRootDir, hri, htd, wal2);
359         long seqid2 = region2.getOpenSeqNum();
360         assertTrue(seqid2 > -1);
361 
362         // I can't close wal1.  Its been appropriated when we split.
363         region2.close();
364         wal2.closeAndDelete();
365         return null;
366       }
367     });
368   }
369 
370   /**
371    * Test writing edits into an HRegion, closing it, splitting logs, opening
372    * Region again.  Verify seqids.
373    * @throws IOException
374    * @throws IllegalAccessException
375    * @throws NoSuchFieldException
376    * @throws IllegalArgumentException
377    * @throws SecurityException
378    */
379   @Test
380   public void testReplayEditsWrittenViaHRegion()
381   throws IOException, SecurityException, IllegalArgumentException,
382       NoSuchFieldException, IllegalAccessException, InterruptedException {
383     final TableName tableName =
384         TableName.valueOf("testReplayEditsWrittenViaHRegion");
385     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
386     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
387     deleteDir(basedir);
388     final byte[] rowName = tableName.getName();
389     final int countPerFamily = 10;
390     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
391     HRegion region3 = HRegion.createHRegion(hri,
392             hbaseRootDir, this.conf, htd);
393     HRegion.closeHRegion(region3);
394     // Write countPerFamily edits into the three families.  Do a flush on one
395     // of the families during the load of edits so its seqid is not same as
396     // others to test we do right thing when different seqids.
397     HLog wal = createWAL(this.conf);
398     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
399     long seqid = region.getOpenSeqNum();
400     boolean first = true;
401     for (HColumnDescriptor hcd: htd.getFamilies()) {
402       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
403       if (first ) {
404         // If first, so we have at least one family w/ different seqid to rest.
405         region.flushcache();
406         first = false;
407       }
408     }
409     // Now assert edits made it in.
410     final Get g = new Get(rowName);
411     Result result = region.get(g);
412     assertEquals(countPerFamily * htd.getFamilies().size(),
413       result.size());
414     // Now close the region (without flush), split the log, reopen the region and assert that
415     // replay of log has the correct effect, that our seqids are calculated correctly so
416     // all edits in logs are seen as 'stale'/old.
417     region.close(true);
418     wal.close();
419     runWALSplit(this.conf);
420     HLog wal2 = createWAL(this.conf);
421     HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
422     long seqid2 = region2.getOpenSeqNum();
423     assertTrue(seqid + result.size() < seqid2);
424     final Result result1b = region2.get(g);
425     assertEquals(result.size(), result1b.size());
426 
427     // Next test.  Add more edits, then 'crash' this region by stealing its wal
428     // out from under it and assert that replay of the log adds the edits back
429     // correctly when region is opened again.
430     for (HColumnDescriptor hcd: htd.getFamilies()) {
431       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
432     }
433     // Get count of edits.
434     final Result result2 = region2.get(g);
435     assertEquals(2 * result.size(), result2.size());
436     wal2.sync();
437     // Set down maximum recovery so we dfsclient doesn't linger retrying something
438     // long gone.
439     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal2).getOutputStream(), 1);
440     final Configuration newConf = HBaseConfiguration.create(this.conf);
441     User user = HBaseTestingUtility.getDifferentUser(newConf,
442       tableName.getNameAsString());
443     user.runAs(new PrivilegedExceptionAction() {
444       public Object run() throws Exception {
445         runWALSplit(newConf);
446         FileSystem newFS = FileSystem.get(newConf);
447         // Make a new wal for new region open.
448         HLog wal3 = createWAL(newConf);
449         final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
450         HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
451           @Override
452           protected boolean restoreEdit(Store s, KeyValue kv) {
453             boolean b = super.restoreEdit(s, kv);
454             countOfRestoredEdits.incrementAndGet();
455             return b;
456           }
457         };
458         long seqid3 = region3.initialize();
459         Result result3 = region3.get(g);
460         // Assert that count of cells is same as before crash.
461         assertEquals(result2.size(), result3.size());
462         assertEquals(htd.getFamilies().size() * countPerFamily,
463           countOfRestoredEdits.get());
464 
465         // I can't close wal1.  Its been appropriated when we split.
466         region3.close();
467         wal3.closeAndDelete();
468         return null;
469       }
470     });
471   }
472 
473   /**
474    * Test that we recover correctly when there is a failure in between the
475    * flushes. i.e. Some stores got flushed but others did not.
476    *
477    * Unfortunately, there is no easy hook to flush at a store level. The way
478    * we get around this is by flushing at the region level, and then deleting
479    * the recently flushed store file for one of the Stores. This would put us
480    * back in the situation where all but that store got flushed and the region
481    * died.
482    *
483    * We restart Region again, and verify that the edits were replayed.
484    *
485    * @throws IOException
486    * @throws IllegalAccessException
487    * @throws NoSuchFieldException
488    * @throws IllegalArgumentException
489    * @throws SecurityException
490    */
491   @Test
492   public void testReplayEditsAfterPartialFlush()
493   throws IOException, SecurityException, IllegalArgumentException,
494       NoSuchFieldException, IllegalAccessException, InterruptedException {
495     final TableName tableName =
496         TableName.valueOf("testReplayEditsWrittenViaHRegion");
497     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
498     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
499     deleteDir(basedir);
500     final byte[] rowName = tableName.getName();
501     final int countPerFamily = 10;
502     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
503     HRegion region3 = HRegion.createHRegion(hri,
504             hbaseRootDir, this.conf, htd);
505     HRegion.closeHRegion(region3);
506     // Write countPerFamily edits into the three families.  Do a flush on one
507     // of the families during the load of edits so its seqid is not same as
508     // others to test we do right thing when different seqids.
509     HLog wal = createWAL(this.conf);
510     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
511     long seqid = region.getOpenSeqNum();
512     for (HColumnDescriptor hcd: htd.getFamilies()) {
513       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
514     }
515 
516     // Now assert edits made it in.
517     final Get g = new Get(rowName);
518     Result result = region.get(g);
519     assertEquals(countPerFamily * htd.getFamilies().size(),
520       result.size());
521 
522     // Let us flush the region
523     region.flushcache();
524     region.close(true);
525     wal.close();
526 
527     // delete the store files in the second column family to simulate a failure
528     // in between the flushcache();
529     // we have 3 families. killing the middle one ensures that taking the maximum
530     // will make us fail.
531     int cf_count = 0;
532     for (HColumnDescriptor hcd: htd.getFamilies()) {
533       cf_count++;
534       if (cf_count == 2) {
535         region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
536       }
537     }
538 
539 
540     // Let us try to split and recover
541     runWALSplit(this.conf);
542     HLog wal2 = createWAL(this.conf);
543     HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
544     long seqid2 = region2.getOpenSeqNum();
545     assertTrue(seqid + result.size() < seqid2);
546 
547     final Result result1b = region2.get(g);
548     assertEquals(result.size(), result1b.size());
549   }
550 
551 
552   // StoreFlusher implementation used in testReplayEditsAfterAbortingFlush.
553   // Only throws exception if throwExceptionWhenFlushing is set true.
554   public static class CustomStoreFlusher extends DefaultStoreFlusher {
555     // Switch between throw and not throw exception in flush
556     static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
557 
558     public CustomStoreFlusher(Configuration conf, Store store) {
559       super(conf, store);
560     }
561     @Override
562     public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
563         TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status)
564             throws IOException {
565       if (throwExceptionWhenFlushing.get()) {
566         throw new IOException("Simulated exception by tests");
567       }
568       return super.flushSnapshot(snapshot, cacheFlushId, snapshotTimeRangeTracker,
569           flushedSize, status);
570     }
571 
572   };
573 
574   /**
575    * Test that we could recover the data correctly after aborting flush. In the
576    * test, first we abort flush after writing some data, then writing more data
577    * and flush again, at last verify the data.
578    * @throws IOException
579    */
580   @Test
581   public void testReplayEditsAfterAbortingFlush() throws IOException {
582     final TableName tableName =
583         TableName.valueOf("testReplayEditsAfterAbortingFlush");
584     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
585     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
586     deleteDir(basedir);
587     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
588     HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
589     region3.close();
590     region3.getLog().closeAndDelete();
591     // Write countPerFamily edits into the three families. Do a flush on one
592     // of the families during the load of edits so its seqid is not same as
593     // others to test we do right thing when different seqids.
594     HLog wal = createWAL(this.conf);
595     RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
596     Mockito.doReturn(false).when(rsServices).isAborted();
597     Configuration customConf = new Configuration(this.conf);
598     customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
599         CustomStoreFlusher.class.getName());
600     HRegion region =
601       HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
602     int writtenRowCount = 10;
603     List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
604         htd.getFamilies());
605     for (int i = 0; i < writtenRowCount; i++) {
606       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
607       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
608           Bytes.toBytes("val"));
609       region.put(put);
610     }
611 
612     // Now assert edits made it in.
613     RegionScanner scanner = region.getScanner(new Scan());
614     assertEquals(writtenRowCount, getScannedCount(scanner));
615 
616     // Let us flush the region
617     CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
618     try {
619       region.flushcache();
620       fail("Injected exception hasn't been thrown");
621     } catch (Throwable t) {
622       LOG.info("Expected simulated exception when flushing region,"
623           + t.getMessage());
624       // simulated to abort server
625       Mockito.doReturn(true).when(rsServices).isAborted();
626     }
627     // writing more data
628     int moreRow = 10;
629     for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
630       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
631       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
632           Bytes.toBytes("val"));
633       region.put(put);
634     }
635     writtenRowCount += moreRow;
636     // call flush again
637     CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
638     try {
639       region.flushcache();
640     } catch (IOException t) {
641       LOG.info("Expected exception when flushing region because server is stopped,"
642           + t.getMessage());
643     }
644 
645     region.close(true);
646     wal.close();
647 
648     // Let us try to split and recover
649     runWALSplit(this.conf);
650     HLog wal2 = createWAL(this.conf);
651     Mockito.doReturn(false).when(rsServices).isAborted();
652     HRegion region2 =
653       HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
654     scanner = region2.getScanner(new Scan());
655     assertEquals(writtenRowCount, getScannedCount(scanner));
656   }
657 
658   private int getScannedCount(RegionScanner scanner) throws IOException {
659     int scannedCount = 0;
660     List<Cell> results = new ArrayList<Cell>();
661     while (true) {
662       boolean existMore = scanner.next(results);
663       if (!results.isEmpty())
664         scannedCount++;
665       if (!existMore)
666         break;
667       results.clear();
668     }
669     return scannedCount;
670   }
671 
672   /**
673    * Create an HRegion with the result of a HLog split and test we only see the
674    * good edits
675    * @throws Exception
676    */
677   @Test
678   public void testReplayEditsWrittenIntoWAL() throws Exception {
679     final TableName tableName =
680         TableName.valueOf("testReplayEditsWrittenIntoWAL");
681     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
682     final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
683     deleteDir(basedir);
684 
685     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
686     HRegion region2 = HRegion.createHRegion(hri,
687             hbaseRootDir, this.conf, htd);
688     HRegion.closeHRegion(region2);
689     final HLog wal = createWAL(this.conf);
690     final byte[] rowName = tableName.getName();
691     final byte[] regionName = hri.getEncodedNameAsBytes();
692     final AtomicLong sequenceId = new AtomicLong(1);
693 
694     // Add 1k to each family.
695     final int countPerFamily = 1000;
696     for (HColumnDescriptor hcd: htd.getFamilies()) {
697       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
698           ee, wal, htd, sequenceId);
699     }
700 
701     // Add a cache flush, shouldn't have any effect
702     wal.startCacheFlush(regionName);
703     wal.completeCacheFlush(regionName);
704 
705     // Add an edit to another family, should be skipped.
706     WALEdit edit = new WALEdit();
707     long now = ee.currentTimeMillis();
708     edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
709       now, rowName));
710     wal.append(hri, tableName, edit, now, htd, sequenceId);
711 
712     // Delete the c family to verify deletes make it over.
713     edit = new WALEdit();
714     now = ee.currentTimeMillis();
715     edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
716       KeyValue.Type.DeleteFamily));
717     wal.append(hri, tableName, edit, now, htd, sequenceId);
718 
719     // Sync.
720     wal.sync();
721     // Set down maximum recovery so we dfsclient doesn't linger retrying something
722     // long gone.
723     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
724     // Make a new conf and a new fs for the splitter to run on so we can take
725     // over old wal.
726     final Configuration newConf = HBaseConfiguration.create(this.conf);
727     User user = HBaseTestingUtility.getDifferentUser(newConf,
728       ".replay.wal.secondtime");
729     user.runAs(new PrivilegedExceptionAction() {
730       public Object run() throws Exception {
731         runWALSplit(newConf);
732         FileSystem newFS = FileSystem.get(newConf);
733         // 100k seems to make for about 4 flushes during HRegion#initialize.
734         newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
735         // Make a new wal for new region.
736         HLog newWal = createWAL(newConf);
737         final AtomicInteger flushcount = new AtomicInteger(0);
738         try {
739           final HRegion region =
740               new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
741             protected boolean internalFlushcache(
742                 final HLog wal, final long myseqid, MonitoredTask status)
743             throws IOException {
744               LOG.info("InternalFlushCache Invoked");
745               boolean b = super.internalFlushcache(wal, myseqid,
746                   Mockito.mock(MonitoredTask.class));
747               flushcount.incrementAndGet();
748               return b;
749             };
750           };
751           long seqid = region.initialize();
752           // We flushed during init.
753           assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
754           assertTrue(seqid - 1 == sequenceId.get());
755 
756           Get get = new Get(rowName);
757           Result result = region.get(get);
758           // Make sure we only see the good edits
759           assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
760             result.size());
761           region.close();
762         } finally {
763           newWal.closeAndDelete();
764         }
765         return null;
766       }
767     });
768   }
769 
770   @Test
771   // the following test is for HBASE-6065
772   public void testSequentialEditLogSeqNum() throws IOException {
773     final TableName tableName =
774         TableName.valueOf("testSequentialEditLogSeqNum");
775     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
776     final Path basedir =
777         FSUtils.getTableDir(this.hbaseRootDir, tableName);
778     deleteDir(basedir);
779     final byte[] rowName = tableName.getName();
780     final int countPerFamily = 10;
781     final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
782 
783     // Mock the HLog
784     MockHLog wal = createMockWAL(this.conf);
785 
786     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
787     for (HColumnDescriptor hcd : htd.getFamilies()) {
788       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
789     }
790 
791     // Let us flush the region
792     // But this time completeflushcache is not yet done
793     region.flushcache();
794     for (HColumnDescriptor hcd : htd.getFamilies()) {
795       addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
796     }
797     long lastestSeqNumber = region.getSequenceId().get();
798     // get the current seq no
799     wal.doCompleteCacheFlush = true;
800     // allow complete cache flush with the previous seq number got after first
801     // set of edits.
802     wal.completeCacheFlush(hri.getEncodedNameAsBytes());
803     wal.close();
804     FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
805     HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0],
806       this.fs, this.conf, null, null, null);
807     FileStatus[] listStatus1 = this.fs.listStatus(
808         new Path(FSUtils.getTableDir(hbaseRootDir, tableName),
809             new Path(hri.getEncodedName(), "recovered.edits")));
810     int editCount = 0;
811     for (FileStatus fileStatus : listStatus1) {
812       editCount = Integer.parseInt(fileStatus.getPath().getName());
813     }
814     // The sequence number should be same 
815     assertEquals(
816         "The sequence number of the recoverd.edits and the current edit seq should be same",
817         lastestSeqNumber, editCount);
818   }
819 
820   static class MockHLog extends FSHLog {
821     boolean doCompleteCacheFlush = false;
822 
823     public MockHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) throws IOException {
824       super(fs, rootDir, logName, conf);
825     }
826 
827     @Override
828     public void completeCacheFlush(byte[] encodedRegionName) {
829       if (!doCompleteCacheFlush) {
830         return;
831       }
832       super.completeCacheFlush(encodedRegionName);
833     }
834   }
835 
836   private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
837     HTableDescriptor htd = new HTableDescriptor(tableName);
838     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
839     htd.addFamily(a);
840     return htd;
841   }
842   
843   private MockHLog createMockWAL(Configuration conf) throws IOException {
844     MockHLog wal = new MockHLog(FileSystem.get(conf), hbaseRootDir, logName, conf);
845     // Set down maximum recovery so we dfsclient doesn't linger retrying something
846     // long gone.
847     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
848     return wal;
849   }
850 
851   // Flusher used in this test.  Keep count of how often we are called and
852   // actually run the flush inside here.
853   class TestFlusher implements FlushRequester {
854     private HRegion r;
855 
856     @Override
857     public void requestFlush(HRegion region) {
858       try {
859         r.flushcache();
860       } catch (IOException e) {
861         throw new RuntimeException("Exception flushing", e);
862       }
863     }
864 
865     @Override
866     public void requestDelayedFlush(HRegion region, long when) {
867       // TODO Auto-generated method stub
868       
869     }
870   }
871 
872   private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
873       final byte[] family, final int count, EnvironmentEdge ee, final HLog wal,
874       final HTableDescriptor htd, final AtomicLong sequenceId)
875   throws IOException {
876     String familyStr = Bytes.toString(family);
877     for (int j = 0; j < count; j++) {
878       byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
879       byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
880       WALEdit edit = new WALEdit();
881       edit.add(new KeyValue(rowName, family, qualifierBytes,
882         ee.currentTimeMillis(), columnBytes));
883       wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd, sequenceId);
884     }
885   }
886 
887   private void addRegionEdits (final byte [] rowName, final byte [] family,
888       final int count, EnvironmentEdge ee, final HRegion r,
889       final String qualifierPrefix)
890   throws IOException {
891     for (int j = 0; j < count; j++) {
892       byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
893       Put p = new Put(rowName);
894       p.add(family, qualifier, ee.currentTimeMillis(), rowName);
895       r.put(p);
896     }
897   }
898 
899   /*
900    * Creates an HRI around an HTD that has <code>tableName</code> and three
901    * column families named 'a','b', and 'c'.
902    * @param tableName Name of table to use when we create HTableDescriptor.
903    */
904    private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
905     return new HRegionInfo(tableName, null, null, false);
906    }
907 
908   /*
909    * Run the split.  Verify only single split file made.
910    * @param c
911    * @return The single split file made
912    * @throws IOException
913    */
914   private Path runWALSplit(final Configuration c) throws IOException {
915     List<Path> splits = HLogSplitter.split(
916       hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c);
917     // Split should generate only 1 file since there's only 1 region
918     assertEquals("splits=" + splits, 1, splits.size());
919     // Make sure the file exists
920     assertTrue(fs.exists(splits.get(0)));
921     LOG.info("Split file=" + splits.get(0));
922     return splits.get(0);
923   }
924 
925   /*
926    * @param c
927    * @return WAL with retries set down from 5 to 1 only.
928    * @throws IOException
929    */
930   private HLog createWAL(final Configuration c) throws IOException {
931     HLog wal = HLogFactory.createHLog(FileSystem.get(c), 
932         hbaseRootDir, logName, c);
933     // Set down maximum recovery so we dfsclient doesn't linger retrying something
934     // long gone.
935     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
936     return wal;
937   }
938 
939   private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
940     HTableDescriptor htd = new HTableDescriptor(tableName);
941     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
942     htd.addFamily(a);
943     HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
944     htd.addFamily(b);
945     HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
946     htd.addFamily(c);
947     return htd;
948   }
949 
950 }
951