View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  import static org.junit.Assert.fail;
24  
25  import java.io.IOException;
26  import java.security.PrivilegedExceptionAction;
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.SortedSet;
30  import java.util.concurrent.atomic.AtomicBoolean;
31  import java.util.concurrent.atomic.AtomicInteger;
32  import java.util.concurrent.atomic.AtomicLong;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileStatus;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.Cell;
41  import org.apache.hadoop.hbase.HBaseConfiguration;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.KeyValue;
48  import org.apache.hadoop.hbase.MasterNotRunningException;
49  import org.apache.hadoop.hbase.MediumTests;
50  import org.apache.hadoop.hbase.MiniHBaseCluster;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.TableName;
53  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
54  import org.apache.hadoop.hbase.client.Delete;
55  import org.apache.hadoop.hbase.client.Get;
56  import org.apache.hadoop.hbase.client.HTable;
57  import org.apache.hadoop.hbase.client.Put;
58  import org.apache.hadoop.hbase.client.Result;
59  import org.apache.hadoop.hbase.client.ResultScanner;
60  import org.apache.hadoop.hbase.client.Scan;
61  import org.apache.hadoop.hbase.io.hfile.HFile;
62  import org.apache.hadoop.hbase.master.HMaster;
63  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
64  import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
65  import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
66  import org.apache.hadoop.hbase.regionserver.FlushRequester;
67  import org.apache.hadoop.hbase.regionserver.HRegion;
68  import org.apache.hadoop.hbase.regionserver.HRegionServer;
69  import org.apache.hadoop.hbase.regionserver.RegionScanner;
70  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
71  import org.apache.hadoop.hbase.regionserver.Store;
72  import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
73  import org.apache.hadoop.hbase.security.User;
74  import org.apache.hadoop.hbase.util.Bytes;
75  import org.apache.hadoop.hbase.util.EnvironmentEdge;
76  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
77  import org.apache.hadoop.hbase.util.FSUtils;
78  import org.apache.hadoop.hbase.util.Pair;
79  import org.junit.After;
80  import org.junit.AfterClass;
81  import org.junit.Before;
82  import org.junit.BeforeClass;
83  import org.junit.Test;
84  import org.junit.experimental.categories.Category;
85  import org.mockito.Mockito;
86  
87  /**
88   * Test replay of edits out of a WAL split.
89   */
90  @Category(MediumTests.class)
91  public class TestWALReplay {
92    public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
93    static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
94    private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
95    private Path hbaseRootDir = null;
96    private String logName;
97    private Path oldLogDir;
98    private Path logDir;
99    private FileSystem fs;
100   private Configuration conf;
101 
102   @BeforeClass
103   public static void setUpBeforeClass() throws Exception {
104     Configuration conf = TEST_UTIL.getConfiguration();
105     conf.setBoolean("dfs.support.append", true);
106     // The below config supported by 0.20-append and CDH3b2
107     conf.setInt("dfs.client.block.recovery.retries", 2);
108     TEST_UTIL.startMiniCluster(3);
109     Path hbaseRootDir =
110       TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
111     LOG.info("hbase.rootdir=" + hbaseRootDir);
112     FSUtils.setRootDir(conf, hbaseRootDir);
113   }
114 
115   @AfterClass
116   public static void tearDownAfterClass() throws Exception {
117     TEST_UTIL.shutdownMiniCluster();
118   }
119 
120   @Before
121   public void setUp() throws Exception {
122     this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
123     this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
124     this.hbaseRootDir = FSUtils.getRootDir(this.conf);
125     this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
126     this.logName = HConstants.HREGION_LOGDIR_NAME;
127     this.logDir = new Path(this.hbaseRootDir, logName);
128     if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
129       TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
130     }
131   }
132 
133   @After
134   public void tearDown() throws Exception {
135     TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
136   }
137 
138   /*
139    * @param p Directory to cleanup
140    */
141   private void deleteDir(final Path p) throws IOException {
142     if (this.fs.exists(p)) {
143       if (!this.fs.delete(p, true)) {
144         throw new IOException("Failed remove of " + p);
145       }
146     }
147   }
148 
149   /**
150    * 
151    * @throws Exception
152    */
153   @Test
154   public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
155     final TableName tableName =
156         TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
157     byte[] family1 = Bytes.toBytes("cf1");
158     byte[] family2 = Bytes.toBytes("cf2");
159     byte[] qualifier = Bytes.toBytes("q");
160     byte[] value = Bytes.toBytes("testV");
161     byte[][] familys = { family1, family2 };
162     TEST_UTIL.createTable(tableName, familys);
163     HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
164     Put put = new Put(Bytes.toBytes("r1"));
165     put.add(family1, qualifier, value);
166     htable.put(put);
167     ResultScanner resultScanner = htable.getScanner(new Scan());
168     int count = 0;
169     while (resultScanner.next() != null) {
170       count++;
171     }
172     resultScanner.close();
173     assertEquals(1, count);
174 
175     MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
176     List<HRegion> regions = hbaseCluster.getRegions(tableName);
177     assertEquals(1, regions.size());
178 
179     // move region to another regionserver
180     HRegion destRegion = regions.get(0);
181     int originServerNum = hbaseCluster
182         .getServerWith(destRegion.getRegionName());
183     assertTrue("Please start more than 1 regionserver", hbaseCluster
184         .getRegionServerThreads().size() > 1);
185     int destServerNum = 0;
186     while (destServerNum == originServerNum) {
187       destServerNum++;
188     }
189     HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
190     HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
191     // move region to destination regionserver
192     moveRegionAndWait(destRegion, destServer);
193 
194     // delete the row
195     Delete del = new Delete(Bytes.toBytes("r1"));
196     htable.delete(del);
197     resultScanner = htable.getScanner(new Scan());
198     count = 0;
199     while (resultScanner.next() != null) {
200       count++;
201     }
202     resultScanner.close();
203     assertEquals(0, count);
204 
205     // flush region and make major compaction
206     destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
207     // wait to complete major compaction
208     for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
209         .getStores().values()) {
210       store.triggerMajorCompaction();
211     }
212     destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
213 
214     // move region to origin regionserver
215     moveRegionAndWait(destRegion, originServer);
216     // abort the origin regionserver
217     originServer.abort("testing");
218 
219     // see what we get
220     Result result = htable.get(new Get(Bytes.toBytes("r1")));
221     if (result != null) {
222       assertTrue("Row is deleted, but we get" + result.toString(),
223           (result == null) || result.isEmpty());
224     }
225     resultScanner.close();
226   }
227 
228   private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
229       throws InterruptedException, MasterNotRunningException,
230       ZooKeeperConnectionException, IOException {
231     HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
232     TEST_UTIL.getHBaseAdmin().move(
233         destRegion.getRegionInfo().getEncodedNameAsBytes(),
234         Bytes.toBytes(destServer.getServerName().getServerName()));
235     while (true) {
236       ServerName serverName = master.getAssignmentManager()
237         .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
238       if (serverName != null && serverName.equals(destServer.getServerName())) {
239         TEST_UTIL.assertRegionOnServer(
240           destRegion.getRegionInfo(), serverName, 200);
241         break;
242       }
243       Thread.sleep(10);
244     }
245   }
246 
247   /**
248    * Tests for hbase-2727.
249    * @throws Exception
250    * @see https://issues.apache.org/jira/browse/HBASE-2727
251    */
252   @Test
253   public void test2727() throws Exception {
254     // Test being able to have > 1 set of edits in the recovered.edits directory.
255     // Ensure edits are replayed properly.
256     final TableName tableName =
257         TableName.valueOf("test2727");
258     HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
259     Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
260     deleteDir(basedir);
261 
262     HTableDescriptor htd = createBasic3FamilyHTD(tableName);
263     HRegion region2 = HRegion.createHRegion(hri,
264         hbaseRootDir, this.conf, htd);
265     HRegion.closeHRegion(region2);
266     final byte [] rowName = tableName.getName();
267 
268     HLog wal1 = createWAL(this.conf);
269     // Add 1k to each family.
270     final int countPerFamily = 1000;
271     for (HColumnDescriptor hcd: htd.getFamilies()) {
272       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
273           wal1, htd);
274     }
275     wal1.close();
276     runWALSplit(this.conf);
277 
278     HLog wal2 = createWAL(this.conf);
279     // Up the sequenceid so that these edits are after the ones added above.
280     wal2.setSequenceNumber(wal1.getSequenceNumber());
281     // Add 1k to each family.
282     for (HColumnDescriptor hcd: htd.getFamilies()) {
283       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
284           ee, wal2, htd);
285     }
286     wal2.close();
287     runWALSplit(this.conf);
288 
289     HLog wal3 = createWAL(this.conf);
290     wal3.setSequenceNumber(wal2.getSequenceNumber());
291     try {
292       long wal3SeqId = wal3.getSequenceNumber();
293       HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
294       long seqid = region.getOpenSeqNum();
295       assertTrue(seqid > wal3SeqId);
296 
297       // TODO: Scan all.
298       region.close();
299     } finally {
300       wal3.closeAndDelete();
301     }
302   }
303 
304   /**
305    * Test case of HRegion that is only made out of bulk loaded files.  Assert
306    * that we don't 'crash'.
307    * @throws IOException
308    * @throws IllegalAccessException
309    * @throws NoSuchFieldException
310    * @throws IllegalArgumentException
311    * @throws SecurityException
312    */
313   @Test
314   public void testRegionMadeOfBulkLoadedFilesOnly()
315   throws IOException, SecurityException, IllegalArgumentException,
316       NoSuchFieldException, IllegalAccessException, InterruptedException {
317     final TableName tableName =
318         TableName.valueOf("testReplayEditsWrittenViaHRegion");
319     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
320     final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
321     deleteDir(basedir);
322     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
323     HRegion region2 = HRegion.createHRegion(hri,
324         hbaseRootDir, this.conf, htd);
325     HRegion.closeHRegion(region2);
326     HLog wal = createWAL(this.conf);
327     HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
328     Path f =  new Path(basedir, "hfile");
329     HFile.Writer writer =
330       HFile.getWriterFactoryNoCache(conf).withPath(fs, f).create();
331     byte [] family = htd.getFamilies().iterator().next().getName();
332     byte [] row = tableName.getName();
333     writer.append(new KeyValue(row, family, family, row));
334     writer.close();
335     List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
336     hfs.add(Pair.newPair(family, f.toString()));
337     region.bulkLoadHFiles(hfs, true);
338     // Add an edit so something in the WAL
339     region.put((new Put(row)).add(family, family, family));
340     wal.sync();
341 
342     // Now 'crash' the region by stealing its wal
343     final Configuration newConf = HBaseConfiguration.create(this.conf);
344     User user = HBaseTestingUtility.getDifferentUser(newConf,
345         tableName.getNameAsString());
346     user.runAs(new PrivilegedExceptionAction() {
347       public Object run() throws Exception {
348         runWALSplit(newConf);
349         HLog wal2 = createWAL(newConf);
350 
351         HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
352           hbaseRootDir, hri, htd, wal2);
353         long seqid2 = region2.getOpenSeqNum();
354         assertTrue(seqid2 > -1);
355 
356         // I can't close wal1.  Its been appropriated when we split.
357         region2.close();
358         wal2.closeAndDelete();
359         return null;
360       }
361     });
362   }
363 
364   /**
365    * Test writing edits into an HRegion, closing it, splitting logs, opening
366    * Region again.  Verify seqids.
367    * @throws IOException
368    * @throws IllegalAccessException
369    * @throws NoSuchFieldException
370    * @throws IllegalArgumentException
371    * @throws SecurityException
372    */
373   @Test
374   public void testReplayEditsWrittenViaHRegion()
375   throws IOException, SecurityException, IllegalArgumentException,
376       NoSuchFieldException, IllegalAccessException, InterruptedException {
377     final TableName tableName =
378         TableName.valueOf("testReplayEditsWrittenViaHRegion");
379     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
380     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
381     deleteDir(basedir);
382     final byte[] rowName = tableName.getName();
383     final int countPerFamily = 10;
384     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
385     HRegion region3 = HRegion.createHRegion(hri,
386             hbaseRootDir, this.conf, htd);
387     HRegion.closeHRegion(region3);
388     // Write countPerFamily edits into the three families.  Do a flush on one
389     // of the families during the load of edits so its seqid is not same as
390     // others to test we do right thing when different seqids.
391     HLog wal = createWAL(this.conf);
392     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
393     long seqid = region.getOpenSeqNum();
394     // HRegionServer usually does this. It knows the largest seqid across all regions.
395     wal.setSequenceNumber(seqid);
396     boolean first = true;
397     for (HColumnDescriptor hcd: htd.getFamilies()) {
398       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
399       if (first ) {
400         // If first, so we have at least one family w/ different seqid to rest.
401         region.flushcache();
402         first = false;
403       }
404     }
405     // Now assert edits made it in.
406     final Get g = new Get(rowName);
407     Result result = region.get(g);
408     assertEquals(countPerFamily * htd.getFamilies().size(),
409       result.size());
410     // Now close the region (without flush), split the log, reopen the region and assert that
411     // replay of log has the correct effect, that our seqids are calculated correctly so
412     // all edits in logs are seen as 'stale'/old.
413     region.close(true);
414     wal.close();
415     runWALSplit(this.conf);
416     HLog wal2 = createWAL(this.conf);
417     HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
418     long seqid2 = region2.getOpenSeqNum();
419     // HRegionServer usually does this. It knows the largest seqid across all regions.
420     wal2.setSequenceNumber(seqid2);
421     assertTrue(seqid + result.size() < seqid2);
422     final Result result1b = region2.get(g);
423     assertEquals(result.size(), result1b.size());
424 
425     // Next test.  Add more edits, then 'crash' this region by stealing its wal
426     // out from under it and assert that replay of the log adds the edits back
427     // correctly when region is opened again.
428     for (HColumnDescriptor hcd: htd.getFamilies()) {
429       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
430     }
431     // Get count of edits.
432     final Result result2 = region2.get(g);
433     assertEquals(2 * result.size(), result2.size());
434     wal2.sync();
435     // Set down maximum recovery so we dfsclient doesn't linger retrying something
436     // long gone.
437     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal2).getOutputStream(), 1);
438     final Configuration newConf = HBaseConfiguration.create(this.conf);
439     User user = HBaseTestingUtility.getDifferentUser(newConf,
440       tableName.getNameAsString());
441     user.runAs(new PrivilegedExceptionAction() {
442       public Object run() throws Exception {
443         runWALSplit(newConf);
444         FileSystem newFS = FileSystem.get(newConf);
445         // Make a new wal for new region open.
446         HLog wal3 = createWAL(newConf);
447         final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
448         HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
449           @Override
450           protected boolean restoreEdit(Store s, KeyValue kv) {
451             boolean b = super.restoreEdit(s, kv);
452             countOfRestoredEdits.incrementAndGet();
453             return b;
454           }
455         };
456         long seqid3 = region3.initialize();
457         // HRegionServer usually does this. It knows the largest seqid across all regions.
458         wal3.setSequenceNumber(seqid3);
459         Result result3 = region3.get(g);
460         // Assert that count of cells is same as before crash.
461         assertEquals(result2.size(), result3.size());
462         assertEquals(htd.getFamilies().size() * countPerFamily,
463           countOfRestoredEdits.get());
464 
465         // I can't close wal1.  Its been appropriated when we split.
466         region3.close();
467         wal3.closeAndDelete();
468         return null;
469       }
470     });
471   }
472 
473   /**
474    * Test that we recover correctly when there is a failure in between the
475    * flushes. i.e. Some stores got flushed but others did not.
476    *
477    * Unfortunately, there is no easy hook to flush at a store level. The way
478    * we get around this is by flushing at the region level, and then deleting
479    * the recently flushed store file for one of the Stores. This would put us
480    * back in the situation where all but that store got flushed and the region
481    * died.
482    *
483    * We restart Region again, and verify that the edits were replayed.
484    *
485    * @throws IOException
486    * @throws IllegalAccessException
487    * @throws NoSuchFieldException
488    * @throws IllegalArgumentException
489    * @throws SecurityException
490    */
491   @Test
492   public void testReplayEditsAfterPartialFlush()
493   throws IOException, SecurityException, IllegalArgumentException,
494       NoSuchFieldException, IllegalAccessException, InterruptedException {
495     final TableName tableName =
496         TableName.valueOf("testReplayEditsWrittenViaHRegion");
497     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
498     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
499     deleteDir(basedir);
500     final byte[] rowName = tableName.getName();
501     final int countPerFamily = 10;
502     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
503     HRegion region3 = HRegion.createHRegion(hri,
504             hbaseRootDir, this.conf, htd);
505     HRegion.closeHRegion(region3);
506     // Write countPerFamily edits into the three families.  Do a flush on one
507     // of the families during the load of edits so its seqid is not same as
508     // others to test we do right thing when different seqids.
509     HLog wal = createWAL(this.conf);
510     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
511     long seqid = region.getOpenSeqNum();
512     // HRegionServer usually does this. It knows the largest seqid across all regions.
513     wal.setSequenceNumber(seqid);
514     for (HColumnDescriptor hcd: htd.getFamilies()) {
515       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
516     }
517 
518     // Now assert edits made it in.
519     final Get g = new Get(rowName);
520     Result result = region.get(g);
521     assertEquals(countPerFamily * htd.getFamilies().size(),
522       result.size());
523 
524     // Let us flush the region
525     region.flushcache();
526     region.close(true);
527     wal.close();
528 
529     // delete the store files in the second column family to simulate a failure
530     // in between the flushcache();
531     // we have 3 families. killing the middle one ensures that taking the maximum
532     // will make us fail.
533     int cf_count = 0;
534     for (HColumnDescriptor hcd: htd.getFamilies()) {
535       cf_count++;
536       if (cf_count == 2) {
537         region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
538       }
539     }
540 
541 
542     // Let us try to split and recover
543     runWALSplit(this.conf);
544     HLog wal2 = createWAL(this.conf);
545     HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
546     long seqid2 = region2.getOpenSeqNum();
547     // HRegionServer usually does this. It knows the largest seqid across all regions.
548     wal2.setSequenceNumber(seqid2);
549     assertTrue(seqid + result.size() < seqid2);
550 
551     final Result result1b = region2.get(g);
552     assertEquals(result.size(), result1b.size());
553   }
554 
555 
556   // StoreFlusher implementation used in testReplayEditsAfterAbortingFlush.
557   // Only throws exception if throwExceptionWhenFlushing is set true.
558   public static class CustomStoreFlusher extends DefaultStoreFlusher {
559     // Switch between throw and not throw exception in flush
560     static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
561 
562     public CustomStoreFlusher(Configuration conf, Store store) {
563       super(conf, store);
564     }
565     @Override
566     public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
567         TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status)
568             throws IOException {
569       if (throwExceptionWhenFlushing.get()) {
570         throw new IOException("Simulated exception by tests");
571       }
572       return super.flushSnapshot(snapshot, cacheFlushId, snapshotTimeRangeTracker,
573           flushedSize, status);
574     }
575 
576   };
577 
578   /**
579    * Test that we could recover the data correctly after aborting flush. In the
580    * test, first we abort flush after writing some data, then writing more data
581    * and flush again, at last verify the data.
582    * @throws IOException
583    */
584   @Test
585   public void testReplayEditsAfterAbortingFlush() throws IOException {
586     final TableName tableName =
587         TableName.valueOf("testReplayEditsAfterAbortingFlush");
588     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
589     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
590     deleteDir(basedir);
591     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
592     HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
593     region3.close();
594     region3.getLog().closeAndDelete();
595     // Write countPerFamily edits into the three families. Do a flush on one
596     // of the families during the load of edits so its seqid is not same as
597     // others to test we do right thing when different seqids.
598     HLog wal = createWAL(this.conf);
599     RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
600     Mockito.doReturn(false).when(rsServices).isAborted();
601     Configuration customConf = new Configuration(this.conf);
602     customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
603         CustomStoreFlusher.class.getName());
604     HRegion region = new HRegion(basedir, wal, this.fs, customConf, hri, htd, rsServices);
605     long seqid = region.initialize();
606     // HRegionServer usually does this. It knows the largest seqid across all
607     // regions.
608     wal.setSequenceNumber(seqid);
609 
610     int writtenRowCount = 10;
611     List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
612         htd.getFamilies());
613     for (int i = 0; i < writtenRowCount; i++) {
614       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
615       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
616           Bytes.toBytes("val"));
617       region.put(put);
618     }
619 
620     // Now assert edits made it in.
621     RegionScanner scanner = region.getScanner(new Scan());
622     assertEquals(writtenRowCount, getScannedCount(scanner));
623 
624     // Let us flush the region
625     CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
626     try {
627       region.flushcache();
628       fail("Injected exception hasn't been thrown");
629     } catch (Throwable t) {
630       LOG.info("Expected simulated exception when flushing region,"
631           + t.getMessage());
632       // simulated to abort server
633       Mockito.doReturn(true).when(rsServices).isAborted();
634     }
635     // writing more data
636     int moreRow = 10;
637     for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
638       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
639       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
640           Bytes.toBytes("val"));
641       region.put(put);
642     }
643     writtenRowCount += moreRow;
644     // call flush again
645     CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
646     try {
647       region.flushcache();
648     } catch (IOException t) {
649       LOG.info("Expected exception when flushing region because server is stopped,"
650           + t.getMessage());
651     }
652 
653     region.close(true);
654     wal.close();
655 
656     // Let us try to split and recover
657     runWALSplit(this.conf);
658     HLog wal2 = createWAL(this.conf);
659     Mockito.doReturn(false).when(rsServices).isAborted();
660     HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, htd,
661         rsServices);
662     long seqid2 = region2.initialize();
663     // HRegionServer usually does this. It knows the largest seqid across all
664     // regions.
665     wal2.setSequenceNumber(seqid2);
666 
667     scanner = region2.getScanner(new Scan());
668     assertEquals(writtenRowCount, getScannedCount(scanner));
669   }
670 
671   private int getScannedCount(RegionScanner scanner) throws IOException {
672     int scannedCount = 0;
673     List<Cell> results = new ArrayList<Cell>();
674     while (true) {
675       boolean existMore = scanner.next(results);
676       if (!results.isEmpty())
677         scannedCount++;
678       if (!existMore)
679         break;
680       results.clear();
681     }
682     return scannedCount;
683   }
684 
685   /**
686    * Create an HRegion with the result of a HLog split and test we only see the
687    * good edits
688    * @throws Exception
689    */
690   @Test
691   public void testReplayEditsWrittenIntoWAL() throws Exception {
692     final TableName tableName =
693         TableName.valueOf("testReplayEditsWrittenIntoWAL");
694     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
695     final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
696     deleteDir(basedir);
697 
698     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
699     HRegion region2 = HRegion.createHRegion(hri,
700             hbaseRootDir, this.conf, htd);
701     HRegion.closeHRegion(region2);
702     final HLog wal = createWAL(this.conf);
703     final byte[] rowName = tableName.getName();
704     final byte[] regionName = hri.getEncodedNameAsBytes();
705 
706     // Add 1k to each family.
707     final int countPerFamily = 1000;
708     for (HColumnDescriptor hcd: htd.getFamilies()) {
709       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
710           ee, wal, htd);
711     }
712 
713     // Add a cache flush, shouldn't have any effect
714     wal.startCacheFlush(regionName);
715     wal.completeCacheFlush(regionName);
716 
717     // Add an edit to another family, should be skipped.
718     WALEdit edit = new WALEdit();
719     long now = ee.currentTimeMillis();
720     edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
721       now, rowName));
722     wal.append(hri, tableName, edit, now, htd);
723 
724     // Delete the c family to verify deletes make it over.
725     edit = new WALEdit();
726     now = ee.currentTimeMillis();
727     edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
728       KeyValue.Type.DeleteFamily));
729     wal.append(hri, tableName, edit, now, htd);
730 
731     // Sync.
732     wal.sync();
733     // Set down maximum recovery so we dfsclient doesn't linger retrying something
734     // long gone.
735     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
736     // Make a new conf and a new fs for the splitter to run on so we can take
737     // over old wal.
738     final Configuration newConf = HBaseConfiguration.create(this.conf);
739     User user = HBaseTestingUtility.getDifferentUser(newConf,
740       ".replay.wal.secondtime");
741     user.runAs(new PrivilegedExceptionAction() {
742       public Object run() throws Exception {
743         runWALSplit(newConf);
744         FileSystem newFS = FileSystem.get(newConf);
745         // 100k seems to make for about 4 flushes during HRegion#initialize.
746         newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
747         // Make a new wal for new region.
748         HLog newWal = createWAL(newConf);
749         final AtomicInteger flushcount = new AtomicInteger(0);
750         try {
751           final HRegion region =
752               new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
753             protected boolean internalFlushcache(
754                 final HLog wal, final long myseqid, MonitoredTask status)
755             throws IOException {
756               LOG.info("InternalFlushCache Invoked");
757               boolean b = super.internalFlushcache(wal, myseqid,
758                   Mockito.mock(MonitoredTask.class));
759               flushcount.incrementAndGet();
760               return b;
761             };
762           };
763           long seqid = region.initialize();
764           // We flushed during init.
765           assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
766           assertTrue(seqid > wal.getSequenceNumber());
767 
768           Get get = new Get(rowName);
769           Result result = region.get(get);
770           // Make sure we only see the good edits
771           assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
772             result.size());
773           region.close();
774         } finally {
775           newWal.closeAndDelete();
776         }
777         return null;
778       }
779     });
780   }
781 
782   @Test
783   // the following test is for HBASE-6065
784   public void testSequentialEditLogSeqNum() throws IOException {
785     final TableName tableName =
786         TableName.valueOf("testSequentialEditLogSeqNum");
787     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
788     final Path basedir =
789         FSUtils.getTableDir(this.hbaseRootDir, tableName);
790     deleteDir(basedir);
791     final byte[] rowName = tableName.getName();
792     final int countPerFamily = 10;
793     final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
794 
795     // Mock the HLog
796     MockHLog wal = createMockWAL(this.conf);
797 
798     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
799     long seqid = region.getOpenSeqNum();
800     // HRegionServer usually does this. It knows the largest seqid across all
801     // regions.
802     wal.setSequenceNumber(seqid);
803     for (HColumnDescriptor hcd : htd.getFamilies()) {
804       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
805     }
806     // get the seq no after first set of entries.
807     long sequenceNumber = wal.getSequenceNumber();
808 
809     // Let us flush the region
810     // But this time completeflushcache is not yet done
811     region.flushcache();
812     for (HColumnDescriptor hcd : htd.getFamilies()) {
813       addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
814     }
815     long lastestSeqNumber = wal.getSequenceNumber();
816     // get the current seq no
817     wal.doCompleteCacheFlush = true;
818     // allow complete cache flush with the previous seq number got after first
819     // set of edits.
820     wal.completeCacheFlush(hri.getEncodedNameAsBytes());
821     wal.close();
822     FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
823     HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0],
824       this.fs, this.conf, null, null, null);
825     FileStatus[] listStatus1 = this.fs.listStatus(
826         new Path(FSUtils.getTableDir(hbaseRootDir, tableName),
827             new Path(hri.getEncodedName(), "recovered.edits")));
828     int editCount = 0;
829     for (FileStatus fileStatus : listStatus1) {
830       editCount = Integer.parseInt(fileStatus.getPath().getName());
831     }
832     // The sequence number should be same 
833     assertEquals(
834         "The sequence number of the recoverd.edits and the current edit seq should be same",
835         lastestSeqNumber, editCount);
836   }
837 
838   static class MockHLog extends FSHLog {
839     boolean doCompleteCacheFlush = false;
840 
841     public MockHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) throws IOException {
842       super(fs, rootDir, logName, conf);
843     }
844 
845     @Override
846     public void completeCacheFlush(byte[] encodedRegionName) {
847       if (!doCompleteCacheFlush) {
848         return;
849       }
850       super.completeCacheFlush(encodedRegionName);
851     }
852   }
853 
854   private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
855     HTableDescriptor htd = new HTableDescriptor(tableName);
856     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
857     htd.addFamily(a);
858     return htd;
859   }
860   
861   private MockHLog createMockWAL(Configuration conf) throws IOException {
862     MockHLog wal = new MockHLog(FileSystem.get(conf), hbaseRootDir, logName, conf);
863     // Set down maximum recovery so we dfsclient doesn't linger retrying something
864     // long gone.
865     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
866     return wal;
867   }
868 
869   // Flusher used in this test.  Keep count of how often we are called and
870   // actually run the flush inside here.
871   class TestFlusher implements FlushRequester {
872     private HRegion r;
873 
874     @Override
875     public void requestFlush(HRegion region) {
876       try {
877         r.flushcache();
878       } catch (IOException e) {
879         throw new RuntimeException("Exception flushing", e);
880       }
881     }
882 
883     @Override
884     public void requestDelayedFlush(HRegion region, long when) {
885       // TODO Auto-generated method stub
886       
887     }
888   }
889 
890   private void addWALEdits (final TableName tableName, final HRegionInfo hri,
891       final byte [] rowName, final byte [] family,
892       final int count, EnvironmentEdge ee, final HLog wal, final HTableDescriptor htd)
893   throws IOException {
894     String familyStr = Bytes.toString(family);
895     for (int j = 0; j < count; j++) {
896       byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
897       byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
898       WALEdit edit = new WALEdit();
899       edit.add(new KeyValue(rowName, family, qualifierBytes,
900         ee.currentTimeMillis(), columnBytes));
901       wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd);
902     }
903   }
904 
905   private void addRegionEdits (final byte [] rowName, final byte [] family,
906       final int count, EnvironmentEdge ee, final HRegion r,
907       final String qualifierPrefix)
908   throws IOException {
909     for (int j = 0; j < count; j++) {
910       byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
911       Put p = new Put(rowName);
912       p.add(family, qualifier, ee.currentTimeMillis(), rowName);
913       r.put(p);
914     }
915   }
916 
917   /*
918    * Creates an HRI around an HTD that has <code>tableName</code> and three
919    * column families named 'a','b', and 'c'.
920    * @param tableName Name of table to use when we create HTableDescriptor.
921    */
922    private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
923     return new HRegionInfo(tableName, null, null, false);
924    }
925 
926   /*
927    * Run the split.  Verify only single split file made.
928    * @param c
929    * @return The single split file made
930    * @throws IOException
931    */
932   private Path runWALSplit(final Configuration c) throws IOException {
933     List<Path> splits = HLogSplitter.split(
934       hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c);
935     // Split should generate only 1 file since there's only 1 region
936     assertEquals("splits=" + splits, 1, splits.size());
937     // Make sure the file exists
938     assertTrue(fs.exists(splits.get(0)));
939     LOG.info("Split file=" + splits.get(0));
940     return splits.get(0);
941   }
942 
943   /*
944    * @param c
945    * @return WAL with retries set down from 5 to 1 only.
946    * @throws IOException
947    */
948   private HLog createWAL(final Configuration c) throws IOException {
949     HLog wal = HLogFactory.createHLog(FileSystem.get(c), 
950         hbaseRootDir, logName, c);
951     // Set down maximum recovery so we dfsclient doesn't linger retrying something
952     // long gone.
953     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
954     return wal;
955   }
956 
957   private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
958     HTableDescriptor htd = new HTableDescriptor(tableName);
959     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
960     htd.addFamily(a);
961     HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
962     htd.addFamily(b);
963     HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
964     htd.addFamily(c);
965     return htd;
966   }
967 
968 }
969