View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertNotNull;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  import static org.mockito.Mockito.when;
26  
27  import java.io.IOException;
28  import java.security.PrivilegedExceptionAction;
29  import java.util.ArrayList;
30  import java.util.List;
31  import java.util.concurrent.atomic.AtomicBoolean;
32  import java.util.concurrent.atomic.AtomicInteger;
33  import java.util.concurrent.atomic.AtomicLong;
34  
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FileSystem;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.fs.PathFilter;
42  import org.apache.hadoop.hbase.Cell;
43  import org.apache.hadoop.hbase.HBaseConfiguration;
44  import org.apache.hadoop.hbase.HBaseTestingUtility;
45  import org.apache.hadoop.hbase.HColumnDescriptor;
46  import org.apache.hadoop.hbase.HConstants;
47  import org.apache.hadoop.hbase.HRegionInfo;
48  import org.apache.hadoop.hbase.HTableDescriptor;
49  import org.apache.hadoop.hbase.KeyValue;
50  import org.apache.hadoop.hbase.MasterNotRunningException;
51  import org.apache.hadoop.hbase.testclassification.MediumTests;
52  import org.apache.hadoop.hbase.MiniHBaseCluster;
53  import org.apache.hadoop.hbase.ServerName;
54  import org.apache.hadoop.hbase.TableName;
55  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
56  import org.apache.hadoop.hbase.client.Delete;
57  import org.apache.hadoop.hbase.client.Get;
58  import org.apache.hadoop.hbase.client.HTable;
59  import org.apache.hadoop.hbase.client.Put;
60  import org.apache.hadoop.hbase.client.Result;
61  import org.apache.hadoop.hbase.client.ResultScanner;
62  import org.apache.hadoop.hbase.client.Scan;
63  import org.apache.hadoop.hbase.client.Table;
64  import org.apache.hadoop.hbase.master.HMaster;
65  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
66  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
67  import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
68  import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
69  import org.apache.hadoop.hbase.regionserver.FlushRequestListener;
70  import org.apache.hadoop.hbase.regionserver.FlushRequester;
71  import org.apache.hadoop.hbase.regionserver.HRegion;
72  import org.apache.hadoop.hbase.regionserver.HRegionServer;
73  import org.apache.hadoop.hbase.regionserver.MemStoreSnapshot;
74  import org.apache.hadoop.hbase.regionserver.RegionScanner;
75  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
76  import org.apache.hadoop.hbase.regionserver.Store;
77  import org.apache.hadoop.hbase.security.User;
78  import org.apache.hadoop.hbase.util.Bytes;
79  import org.apache.hadoop.hbase.util.EnvironmentEdge;
80  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
81  import org.apache.hadoop.hbase.util.FSUtils;
82  import org.apache.hadoop.hbase.util.HFileTestUtil;
83  import org.apache.hadoop.hbase.util.Pair;
84  import org.apache.hadoop.hbase.wal.DefaultWALProvider;
85  import org.apache.hadoop.hbase.wal.WAL;
86  import org.apache.hadoop.hbase.wal.WALKey;
87  import org.apache.hadoop.hbase.wal.WALFactory;
88  import org.apache.hadoop.hbase.wal.WALSplitter;
89  import org.junit.After;
90  import org.junit.AfterClass;
91  import org.junit.Before;
92  import org.junit.BeforeClass;
93  import org.junit.Test;
94  import org.junit.Rule;
95  import org.junit.rules.TestName;
96  import org.junit.experimental.categories.Category;
97  import org.mockito.Mockito;
98  
99  /**
100  * Test replay of edits out of a WAL split.
101  */
102 @Category(MediumTests.class)
103 public class TestWALReplay {
104   public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
105   static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
106   private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
107   private Path hbaseRootDir = null;
108   private String logName;
109   private Path oldLogDir;
110   private Path logDir;
111   private FileSystem fs;
112   private Configuration conf;
113   private RecoveryMode mode;
114   private WALFactory wals;
115 
116   @Rule
117   public final TestName currentTest = new TestName();
118 
119 
120   @BeforeClass
121   public static void setUpBeforeClass() throws Exception {
122     Configuration conf = TEST_UTIL.getConfiguration();
123     conf.setBoolean("dfs.support.append", true);
124     // The below config supported by 0.20-append and CDH3b2
125     conf.setInt("dfs.client.block.recovery.retries", 2);
126     TEST_UTIL.startMiniCluster(3);
127     Path hbaseRootDir =
128       TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
129     LOG.info("hbase.rootdir=" + hbaseRootDir);
130     FSUtils.setRootDir(conf, hbaseRootDir);
131   }
132 
133   @AfterClass
134   public static void tearDownAfterClass() throws Exception {
135     TEST_UTIL.shutdownMiniCluster();
136   }
137 
138   @Before
139   public void setUp() throws Exception {
140     this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
141     this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
142     this.hbaseRootDir = FSUtils.getRootDir(this.conf);
143     this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
144     this.logName = DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName() + "-manual");
145     this.logDir = new Path(this.hbaseRootDir, logName);
146     if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
147       TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
148     }
149     this.mode = (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ?
150         RecoveryMode.LOG_REPLAY : RecoveryMode.LOG_SPLITTING);
151     this.wals = new WALFactory(conf, null, currentTest.getMethodName());
152   }
153 
154   @After
155   public void tearDown() throws Exception {
156     this.wals.close();
157     TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
158   }
159 
160   /*
161    * @param p Directory to cleanup
162    */
163   private void deleteDir(final Path p) throws IOException {
164     if (this.fs.exists(p)) {
165       if (!this.fs.delete(p, true)) {
166         throw new IOException("Failed remove of " + p);
167       }
168     }
169   }
170 
171   /**
172    *
173    * @throws Exception
174    */
175   @Test
176   public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
177     final TableName tableName =
178         TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
179     byte[] family1 = Bytes.toBytes("cf1");
180     byte[] family2 = Bytes.toBytes("cf2");
181     byte[] qualifier = Bytes.toBytes("q");
182     byte[] value = Bytes.toBytes("testV");
183     byte[][] familys = { family1, family2 };
184     TEST_UTIL.createTable(tableName, familys);
185     Table htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
186     Put put = new Put(Bytes.toBytes("r1"));
187     put.add(family1, qualifier, value);
188     htable.put(put);
189     ResultScanner resultScanner = htable.getScanner(new Scan());
190     int count = 0;
191     while (resultScanner.next() != null) {
192       count++;
193     }
194     resultScanner.close();
195     assertEquals(1, count);
196 
197     MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
198     List<HRegion> regions = hbaseCluster.getRegions(tableName);
199     assertEquals(1, regions.size());
200 
201     // move region to another regionserver
202     HRegion destRegion = regions.get(0);
203     int originServerNum = hbaseCluster
204         .getServerWith(destRegion.getRegionName());
205     assertTrue("Please start more than 1 regionserver", hbaseCluster
206         .getRegionServerThreads().size() > 1);
207     int destServerNum = 0;
208     while (destServerNum == originServerNum) {
209       destServerNum++;
210     }
211     HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
212     HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
213     // move region to destination regionserver
214     moveRegionAndWait(destRegion, destServer);
215 
216     // delete the row
217     Delete del = new Delete(Bytes.toBytes("r1"));
218     htable.delete(del);
219     resultScanner = htable.getScanner(new Scan());
220     count = 0;
221     while (resultScanner.next() != null) {
222       count++;
223     }
224     resultScanner.close();
225     assertEquals(0, count);
226 
227     // flush region and make major compaction
228     destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
229     // wait to complete major compaction
230     for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
231         .getStores().values()) {
232       store.triggerMajorCompaction();
233     }
234     destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
235 
236     // move region to origin regionserver
237     moveRegionAndWait(destRegion, originServer);
238     // abort the origin regionserver
239     originServer.abort("testing");
240 
241     // see what we get
242     Result result = htable.get(new Get(Bytes.toBytes("r1")));
243     if (result != null) {
244       assertTrue("Row is deleted, but we get" + result.toString(),
245           (result == null) || result.isEmpty());
246     }
247     resultScanner.close();
248   }
249 
250   private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
251       throws InterruptedException, MasterNotRunningException,
252       ZooKeeperConnectionException, IOException {
253     HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
254     TEST_UTIL.getHBaseAdmin().move(
255         destRegion.getRegionInfo().getEncodedNameAsBytes(),
256         Bytes.toBytes(destServer.getServerName().getServerName()));
257     while (true) {
258       ServerName serverName = master.getAssignmentManager()
259         .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
260       if (serverName != null && serverName.equals(destServer.getServerName())) {
261         TEST_UTIL.assertRegionOnServer(
262           destRegion.getRegionInfo(), serverName, 200);
263         break;
264       }
265       Thread.sleep(10);
266     }
267   }
268 
269   /**
270    * Tests for hbase-2727.
271    * @throws Exception
272    * @see https://issues.apache.org/jira/browse/HBASE-2727
273    */
274   @Test
275   public void test2727() throws Exception {
276     // Test being able to have > 1 set of edits in the recovered.edits directory.
277     // Ensure edits are replayed properly.
278     final TableName tableName =
279         TableName.valueOf("test2727");
280     HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
281     Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
282     deleteDir(basedir);
283 
284     HTableDescriptor htd = createBasic3FamilyHTD(tableName);
285     HRegion region2 = HRegion.createHRegion(hri,
286         hbaseRootDir, this.conf, htd);
287     HRegion.closeHRegion(region2);
288     final byte [] rowName = tableName.getName();
289 
290     WAL wal1 = createWAL(this.conf);
291     // Add 1k to each family.
292     final int countPerFamily = 1000;
293     final AtomicLong sequenceId = new AtomicLong(1);
294     for (HColumnDescriptor hcd: htd.getFamilies()) {
295       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
296           wal1, htd, sequenceId);
297     }
298     wal1.shutdown();
299     runWALSplit(this.conf);
300 
301     WAL wal2 = createWAL(this.conf);
302     // Add 1k to each family.
303     for (HColumnDescriptor hcd: htd.getFamilies()) {
304       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
305           ee, wal2, htd, sequenceId);
306     }
307     wal2.shutdown();
308     runWALSplit(this.conf);
309 
310     WAL wal3 = createWAL(this.conf);
311     try {
312       HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
313       long seqid = region.getOpenSeqNum();
314       // The regions opens with sequenceId as 1. With 6k edits, its sequence number reaches 6k + 1.
315       // When opened, this region would apply 6k edits, and increment the sequenceId by 1
316       assertTrue(seqid > sequenceId.get());
317       assertEquals(seqid - 1, sequenceId.get());
318       LOG.debug("region.getOpenSeqNum(): " + region.getOpenSeqNum() + ", wal3.id: "
319           + sequenceId.get());
320 
321       // TODO: Scan all.
322       region.close();
323     } finally {
324       wal3.close();
325     }
326   }
327 
328   /**
329    * Test case of HRegion that is only made out of bulk loaded files.  Assert
330    * that we don't 'crash'.
331    * @throws IOException
332    * @throws IllegalAccessException
333    * @throws NoSuchFieldException
334    * @throws IllegalArgumentException
335    * @throws SecurityException
336    */
337   @Test
338   public void testRegionMadeOfBulkLoadedFilesOnly()
339   throws IOException, SecurityException, IllegalArgumentException,
340       NoSuchFieldException, IllegalAccessException, InterruptedException {
341     final TableName tableName =
342         TableName.valueOf("testRegionMadeOfBulkLoadedFilesOnly");
343     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
344     final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
345     deleteDir(basedir);
346     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
347     HRegion region2 = HRegion.createHRegion(hri,
348         hbaseRootDir, this.conf, htd);
349     HRegion.closeHRegion(region2);
350     WAL wal = createWAL(this.conf);
351     HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
352 
353     byte [] family = htd.getFamilies().iterator().next().getName();
354     Path f =  new Path(basedir, "hfile");
355     HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""),
356         Bytes.toBytes("z"), 10);
357     List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
358     hfs.add(Pair.newPair(family, f.toString()));
359     region.bulkLoadHFiles(hfs, true);
360 
361     // Add an edit so something in the WAL
362     byte [] row = tableName.getName();
363     region.put((new Put(row)).add(family, family, family));
364     wal.sync();
365     final int rowsInsertedCount = 11;
366 
367     assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
368 
369     // Now 'crash' the region by stealing its wal
370     final Configuration newConf = HBaseConfiguration.create(this.conf);
371     User user = HBaseTestingUtility.getDifferentUser(newConf,
372         tableName.getNameAsString());
373     user.runAs(new PrivilegedExceptionAction() {
374       @Override
375       public Object run() throws Exception {
376         runWALSplit(newConf);
377         WAL wal2 = createWAL(newConf);
378 
379         HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
380           hbaseRootDir, hri, htd, wal2);
381         long seqid2 = region2.getOpenSeqNum();
382         assertTrue(seqid2 > -1);
383         assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
384 
385         // I can't close wal1.  Its been appropriated when we split.
386         region2.close();
387         wal2.close();
388         return null;
389       }
390     });
391   }
392 
393   /**
394    * HRegion test case that is made of a major compacted HFile (created with three bulk loaded
395    * files) and an edit in the memstore.
396    * This is for HBASE-10958 "[dataloss] Bulk loading with seqids can prevent some log entries
397    * from being replayed"
398    * @throws IOException
399    * @throws IllegalAccessException
400    * @throws NoSuchFieldException
401    * @throws IllegalArgumentException
402    * @throws SecurityException
403    */
404   @Test
405   public void testCompactedBulkLoadedFiles()
406       throws IOException, SecurityException, IllegalArgumentException,
407       NoSuchFieldException, IllegalAccessException, InterruptedException {
408     final TableName tableName =
409         TableName.valueOf("testCompactedBulkLoadedFiles");
410     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
411     final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
412     deleteDir(basedir);
413     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
414     HRegion region2 = HRegion.createHRegion(hri,
415         hbaseRootDir, this.conf, htd);
416     HRegion.closeHRegion(region2);
417     WAL wal = createWAL(this.conf);
418     HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
419 
420     // Add an edit so something in the WAL
421     byte [] row = tableName.getName();
422     byte [] family = htd.getFamilies().iterator().next().getName();
423     region.put((new Put(row)).add(family, family, family));
424     wal.sync();
425 
426     List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
427     for (int i = 0; i < 3; i++) {
428       Path f = new Path(basedir, "hfile"+i);
429       HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(i + "00"),
430           Bytes.toBytes(i + "50"), 10);
431       hfs.add(Pair.newPair(family, f.toString()));
432     }
433     region.bulkLoadHFiles(hfs, true);
434     final int rowsInsertedCount = 31;
435     assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
436 
437     // major compact to turn all the bulk loaded files into one normal file
438     region.compactStores(true);
439     assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
440 
441     // Now 'crash' the region by stealing its wal
442     final Configuration newConf = HBaseConfiguration.create(this.conf);
443     User user = HBaseTestingUtility.getDifferentUser(newConf,
444         tableName.getNameAsString());
445     user.runAs(new PrivilegedExceptionAction() {
446       @Override
447       public Object run() throws Exception {
448         runWALSplit(newConf);
449         WAL wal2 = createWAL(newConf);
450 
451         HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
452             hbaseRootDir, hri, htd, wal2);
453         long seqid2 = region2.getOpenSeqNum();
454         assertTrue(seqid2 > -1);
455         assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
456 
457         // I can't close wal1.  Its been appropriated when we split.
458         region2.close();
459         wal2.close();
460         return null;
461       }
462     });
463   }
464 
465 
466   /**
467    * Test writing edits into an HRegion, closing it, splitting logs, opening
468    * Region again.  Verify seqids.
469    * @throws IOException
470    * @throws IllegalAccessException
471    * @throws NoSuchFieldException
472    * @throws IllegalArgumentException
473    * @throws SecurityException
474    */
475   @Test
476   public void testReplayEditsWrittenViaHRegion()
477   throws IOException, SecurityException, IllegalArgumentException,
478       NoSuchFieldException, IllegalAccessException, InterruptedException {
479     final TableName tableName =
480         TableName.valueOf("testReplayEditsWrittenViaHRegion");
481     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
482     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
483     deleteDir(basedir);
484     final byte[] rowName = tableName.getName();
485     final int countPerFamily = 10;
486     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
487     HRegion region3 = HRegion.createHRegion(hri,
488             hbaseRootDir, this.conf, htd);
489     HRegion.closeHRegion(region3);
490     // Write countPerFamily edits into the three families.  Do a flush on one
491     // of the families during the load of edits so its seqid is not same as
492     // others to test we do right thing when different seqids.
493     WAL wal = createWAL(this.conf);
494     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
495     long seqid = region.getOpenSeqNum();
496     boolean first = true;
497     for (HColumnDescriptor hcd: htd.getFamilies()) {
498       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
499       if (first ) {
500         // If first, so we have at least one family w/ different seqid to rest.
501         region.flushcache();
502         first = false;
503       }
504     }
505     // Now assert edits made it in.
506     final Get g = new Get(rowName);
507     Result result = region.get(g);
508     assertEquals(countPerFamily * htd.getFamilies().size(),
509       result.size());
510     // Now close the region (without flush), split the log, reopen the region and assert that
511     // replay of log has the correct effect, that our seqids are calculated correctly so
512     // all edits in logs are seen as 'stale'/old.
513     region.close(true);
514     wal.shutdown();
515     runWALSplit(this.conf);
516     WAL wal2 = createWAL(this.conf);
517     HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
518     long seqid2 = region2.getOpenSeqNum();
519     assertTrue(seqid + result.size() < seqid2);
520     final Result result1b = region2.get(g);
521     assertEquals(result.size(), result1b.size());
522 
523     // Next test.  Add more edits, then 'crash' this region by stealing its wal
524     // out from under it and assert that replay of the log adds the edits back
525     // correctly when region is opened again.
526     for (HColumnDescriptor hcd: htd.getFamilies()) {
527       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
528     }
529     // Get count of edits.
530     final Result result2 = region2.get(g);
531     assertEquals(2 * result.size(), result2.size());
532     wal2.sync();
533     final Configuration newConf = HBaseConfiguration.create(this.conf);
534     User user = HBaseTestingUtility.getDifferentUser(newConf,
535       tableName.getNameAsString());
536     user.runAs(new PrivilegedExceptionAction() {
537       @Override
538       public Object run() throws Exception {
539         runWALSplit(newConf);
540         FileSystem newFS = FileSystem.get(newConf);
541         // Make a new wal for new region open.
542         WAL wal3 = createWAL(newConf);
543         final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
544         HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
545           @Override
546           protected boolean restoreEdit(Store s, Cell cell) {
547             boolean b = super.restoreEdit(s, cell);
548             countOfRestoredEdits.incrementAndGet();
549             return b;
550           }
551         };
552         long seqid3 = region3.initialize();
553         Result result3 = region3.get(g);
554         // Assert that count of cells is same as before crash.
555         assertEquals(result2.size(), result3.size());
556         assertEquals(htd.getFamilies().size() * countPerFamily,
557           countOfRestoredEdits.get());
558 
559         // I can't close wal1.  Its been appropriated when we split.
560         region3.close();
561         wal3.close();
562         return null;
563       }
564     });
565   }
566 
567   /**
568    * Test that we recover correctly when there is a failure in between the
569    * flushes. i.e. Some stores got flushed but others did not.
570    *
571    * Unfortunately, there is no easy hook to flush at a store level. The way
572    * we get around this is by flushing at the region level, and then deleting
573    * the recently flushed store file for one of the Stores. This would put us
574    * back in the situation where all but that store got flushed and the region
575    * died.
576    *
577    * We restart Region again, and verify that the edits were replayed.
578    *
579    * @throws IOException
580    * @throws IllegalAccessException
581    * @throws NoSuchFieldException
582    * @throws IllegalArgumentException
583    * @throws SecurityException
584    */
585   @Test
586   public void testReplayEditsAfterPartialFlush()
587   throws IOException, SecurityException, IllegalArgumentException,
588       NoSuchFieldException, IllegalAccessException, InterruptedException {
589     final TableName tableName =
590         TableName.valueOf("testReplayEditsWrittenViaHRegion");
591     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
592     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
593     deleteDir(basedir);
594     final byte[] rowName = tableName.getName();
595     final int countPerFamily = 10;
596     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
597     HRegion region3 = HRegion.createHRegion(hri,
598             hbaseRootDir, this.conf, htd);
599     HRegion.closeHRegion(region3);
600     // Write countPerFamily edits into the three families.  Do a flush on one
601     // of the families during the load of edits so its seqid is not same as
602     // others to test we do right thing when different seqids.
603     WAL wal = createWAL(this.conf);
604     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
605     long seqid = region.getOpenSeqNum();
606     for (HColumnDescriptor hcd: htd.getFamilies()) {
607       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
608     }
609 
610     // Now assert edits made it in.
611     final Get g = new Get(rowName);
612     Result result = region.get(g);
613     assertEquals(countPerFamily * htd.getFamilies().size(),
614       result.size());
615 
616     // Let us flush the region
617     region.flushcache();
618     region.close(true);
619     wal.shutdown();
620 
621     // delete the store files in the second column family to simulate a failure
622     // in between the flushcache();
623     // we have 3 families. killing the middle one ensures that taking the maximum
624     // will make us fail.
625     int cf_count = 0;
626     for (HColumnDescriptor hcd: htd.getFamilies()) {
627       cf_count++;
628       if (cf_count == 2) {
629         region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
630       }
631     }
632 
633 
634     // Let us try to split and recover
635     runWALSplit(this.conf);
636     WAL wal2 = createWAL(this.conf);
637     HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
638     long seqid2 = region2.getOpenSeqNum();
639     assertTrue(seqid + result.size() < seqid2);
640 
641     final Result result1b = region2.get(g);
642     assertEquals(result.size(), result1b.size());
643   }
644 
645 
646   // StoreFlusher implementation used in testReplayEditsAfterAbortingFlush.
647   // Only throws exception if throwExceptionWhenFlushing is set true.
648   public static class CustomStoreFlusher extends DefaultStoreFlusher {
649     // Switch between throw and not throw exception in flush
650     static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
651 
652     public CustomStoreFlusher(Configuration conf, Store store) {
653       super(conf, store);
654     }
655     @Override
656     public List<Path> flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushId,
657         MonitoredTask status) throws IOException {
658       if (throwExceptionWhenFlushing.get()) {
659         throw new IOException("Simulated exception by tests");
660       }
661       return super.flushSnapshot(snapshot, cacheFlushId, status);
662     }
663 
664   };
665 
666   /**
667    * Test that we could recover the data correctly after aborting flush. In the
668    * test, first we abort flush after writing some data, then writing more data
669    * and flush again, at last verify the data.
670    * @throws IOException
671    */
672   @Test
673   public void testReplayEditsAfterAbortingFlush() throws IOException {
674     final TableName tableName =
675         TableName.valueOf("testReplayEditsAfterAbortingFlush");
676     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
677     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
678     deleteDir(basedir);
679     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
680     HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
681     region3.close();
682     region3.getWAL().close();
683     // Write countPerFamily edits into the three families. Do a flush on one
684     // of the families during the load of edits so its seqid is not same as
685     // others to test we do right thing when different seqids.
686     WAL wal = createWAL(this.conf);
687     RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
688     Mockito.doReturn(false).when(rsServices).isAborted();
689     when(rsServices.getServerName()).thenReturn(ServerName.valueOf("foo", 10, 10));
690     Configuration customConf = new Configuration(this.conf);
691     customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
692         CustomStoreFlusher.class.getName());
693     HRegion region =
694       HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
695     int writtenRowCount = 10;
696     List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
697         htd.getFamilies());
698     for (int i = 0; i < writtenRowCount; i++) {
699       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
700       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
701           Bytes.toBytes("val"));
702       region.put(put);
703     }
704 
705     // Now assert edits made it in.
706     RegionScanner scanner = region.getScanner(new Scan());
707     assertEquals(writtenRowCount, getScannedCount(scanner));
708 
709     // Let us flush the region
710     CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
711     try {
712       region.flushcache();
713       fail("Injected exception hasn't been thrown");
714     } catch (Throwable t) {
715       LOG.info("Expected simulated exception when flushing region,"
716           + t.getMessage());
717       // simulated to abort server
718       Mockito.doReturn(true).when(rsServices).isAborted();
719     }
720     // writing more data
721     int moreRow = 10;
722     for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
723       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
724       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
725           Bytes.toBytes("val"));
726       region.put(put);
727     }
728     writtenRowCount += moreRow;
729     // call flush again
730     CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
731     try {
732       region.flushcache();
733     } catch (IOException t) {
734       LOG.info("Expected exception when flushing region because server is stopped,"
735           + t.getMessage());
736     }
737 
738     region.close(true);
739     wal.shutdown();
740 
741     // Let us try to split and recover
742     runWALSplit(this.conf);
743     WAL wal2 = createWAL(this.conf);
744     Mockito.doReturn(false).when(rsServices).isAborted();
745     HRegion region2 =
746       HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
747     scanner = region2.getScanner(new Scan());
748     assertEquals(writtenRowCount, getScannedCount(scanner));
749   }
750 
751   private int getScannedCount(RegionScanner scanner) throws IOException {
752     int scannedCount = 0;
753     List<Cell> results = new ArrayList<Cell>();
754     while (true) {
755       boolean existMore = scanner.next(results);
756       if (!results.isEmpty())
757         scannedCount++;
758       if (!existMore)
759         break;
760       results.clear();
761     }
762     return scannedCount;
763   }
764 
765   /**
766    * Create an HRegion with the result of a WAL split and test we only see the
767    * good edits
768    * @throws Exception
769    */
770   @Test
771   public void testReplayEditsWrittenIntoWAL() throws Exception {
772     final TableName tableName =
773         TableName.valueOf("testReplayEditsWrittenIntoWAL");
774     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
775     final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
776     deleteDir(basedir);
777 
778     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
779     HRegion region2 = HRegion.createHRegion(hri,
780             hbaseRootDir, this.conf, htd);
781     HRegion.closeHRegion(region2);
782     final WAL wal = createWAL(this.conf);
783     final byte[] rowName = tableName.getName();
784     final byte[] regionName = hri.getEncodedNameAsBytes();
785     final AtomicLong sequenceId = new AtomicLong(1);
786 
787     // Add 1k to each family.
788     final int countPerFamily = 1000;
789     for (HColumnDescriptor hcd: htd.getFamilies()) {
790       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
791           ee, wal, htd, sequenceId);
792     }
793 
794     // Add a cache flush, shouldn't have any effect
795     wal.startCacheFlush(regionName);
796     wal.completeCacheFlush(regionName);
797 
798     // Add an edit to another family, should be skipped.
799     WALEdit edit = new WALEdit();
800     long now = ee.currentTime();
801     edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
802       now, rowName));
803     wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now), edit, sequenceId,
804         true, null);
805 
806     // Delete the c family to verify deletes make it over.
807     edit = new WALEdit();
808     now = ee.currentTime();
809     edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
810       KeyValue.Type.DeleteFamily));
811     wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now), edit, sequenceId,
812         true, null);
813 
814     // Sync.
815     wal.sync();
816     // Make a new conf and a new fs for the splitter to run on so we can take
817     // over old wal.
818     final Configuration newConf = HBaseConfiguration.create(this.conf);
819     User user = HBaseTestingUtility.getDifferentUser(newConf,
820       ".replay.wal.secondtime");
821     user.runAs(new PrivilegedExceptionAction() {
822       @Override
823       public Object run() throws Exception {
824         runWALSplit(newConf);
825         FileSystem newFS = FileSystem.get(newConf);
826         // 100k seems to make for about 4 flushes during HRegion#initialize.
827         newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
828         // Make a new wal for new region.
829         WAL newWal = createWAL(newConf);
830         final AtomicInteger flushcount = new AtomicInteger(0);
831         try {
832           final HRegion region =
833               new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
834             @Override
835             protected FlushResult internalFlushcache(
836                 final WAL wal, final long myseqid, MonitoredTask status)
837             throws IOException {
838               LOG.info("InternalFlushCache Invoked");
839               FlushResult fs = super.internalFlushcache(wal, myseqid,
840                   Mockito.mock(MonitoredTask.class));
841               flushcount.incrementAndGet();
842               return fs;
843             };
844           };
845           long seqid = region.initialize();
846           // We flushed during init.
847           assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
848           assertTrue(seqid - 1 == sequenceId.get());
849 
850           Get get = new Get(rowName);
851           Result result = region.get(get);
852           // Make sure we only see the good edits
853           assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
854             result.size());
855           region.close();
856         } finally {
857           newWal.close();
858         }
859         return null;
860       }
861     });
862   }
863 
864   @Test
865   // the following test is for HBASE-6065
866   public void testSequentialEditLogSeqNum() throws IOException {
867     final TableName tableName = TableName.valueOf(currentTest.getMethodName());
868     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
869     final Path basedir =
870         FSUtils.getTableDir(this.hbaseRootDir, tableName);
871     deleteDir(basedir);
872     final byte[] rowName = tableName.getName();
873     final int countPerFamily = 10;
874     final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
875 
876     // Mock the WAL
877     MockWAL wal = createMockWAL();
878 
879     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
880     for (HColumnDescriptor hcd : htd.getFamilies()) {
881       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
882     }
883 
884     // Let us flush the region
885     // But this time completeflushcache is not yet done
886     region.flushcache();
887     for (HColumnDescriptor hcd : htd.getFamilies()) {
888       addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
889     }
890     long lastestSeqNumber = region.getSequenceId().get();
891     // get the current seq no
892     wal.doCompleteCacheFlush = true;
893     // allow complete cache flush with the previous seq number got after first
894     // set of edits.
895     wal.completeCacheFlush(hri.getEncodedNameAsBytes());
896     wal.shutdown();
897     FileStatus[] listStatus = wal.getFiles();
898     assertNotNull(listStatus);
899     assertTrue(listStatus.length > 0);
900     WALSplitter.splitLogFile(hbaseRootDir, listStatus[0],
901         this.fs, this.conf, null, null, null, mode, wals);
902     FileStatus[] listStatus1 = this.fs.listStatus(
903       new Path(FSUtils.getTableDir(hbaseRootDir, tableName), new Path(hri.getEncodedName(),
904           "recovered.edits")), new PathFilter() {
905         @Override
906         public boolean accept(Path p) {
907           if (WALSplitter.isSequenceIdFile(p)) {
908             return false;
909           }
910           return true;
911         }
912       });
913     int editCount = 0;
914     for (FileStatus fileStatus : listStatus1) {
915       editCount = Integer.parseInt(fileStatus.getPath().getName());
916     }
917     // The sequence number should be same
918     assertEquals(
919         "The sequence number of the recoverd.edits and the current edit seq should be same",
920         lastestSeqNumber, editCount);
921   }
922 
923   static class MockWAL extends FSHLog {
924     boolean doCompleteCacheFlush = false;
925 
926     public MockWAL(FileSystem fs, Path rootDir, String logName, Configuration conf)
927         throws IOException {
928       super(fs, rootDir, logName, HConstants.HREGION_OLDLOGDIR_NAME, conf, null, true, null, null);
929     }
930 
931     @Override
932     public void completeCacheFlush(byte[] encodedRegionName) {
933       if (!doCompleteCacheFlush) {
934         return;
935       }
936       super.completeCacheFlush(encodedRegionName);
937     }
938   }
939 
940   private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
941     HTableDescriptor htd = new HTableDescriptor(tableName);
942     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
943     htd.addFamily(a);
944     return htd;
945   }
946 
947   private MockWAL createMockWAL() throws IOException {
948     MockWAL wal = new MockWAL(fs, hbaseRootDir, logName, conf);
949     // Set down maximum recovery so we dfsclient doesn't linger retrying something
950     // long gone.
951     HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
952     return wal;
953   }
954 
955   // Flusher used in this test.  Keep count of how often we are called and
956   // actually run the flush inside here.
957   class TestFlusher implements FlushRequester {
958     private HRegion r;
959 
960     @Override
961     public void requestFlush(HRegion region) {
962       try {
963         r.flushcache();
964       } catch (IOException e) {
965         throw new RuntimeException("Exception flushing", e);
966       }
967     }
968 
969     @Override
970     public void requestDelayedFlush(HRegion region, long when) {
971       // TODO Auto-generated method stub
972 
973     }
974 
975     @Override
976     public void registerFlushRequestListener(FlushRequestListener listener) {
977 
978     }
979 
980     @Override
981     public boolean unregisterFlushRequestListener(FlushRequestListener listener) {
982       return false;
983     }
984 
985     @Override
986     public void setGlobalMemstoreLimit(long globalMemStoreSize) {
987 
988     }
989   }
990 
991   private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
992       final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
993       final HTableDescriptor htd, final AtomicLong sequenceId)
994   throws IOException {
995     String familyStr = Bytes.toString(family);
996     for (int j = 0; j < count; j++) {
997       byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
998       byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
999       WALEdit edit = new WALEdit();
1000       edit.add(new KeyValue(rowName, family, qualifierBytes,
1001         ee.currentTime(), columnBytes));
1002       wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, ee.currentTime()),
1003           edit, sequenceId, true, null);
1004     }
1005     wal.sync();
1006   }
1007 
1008   static List<Put> addRegionEdits (final byte [] rowName, final byte [] family,
1009       final int count, EnvironmentEdge ee, final HRegion r,
1010       final String qualifierPrefix)
1011   throws IOException {
1012     List<Put> puts = new ArrayList<Put>();
1013     for (int j = 0; j < count; j++) {
1014       byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
1015       Put p = new Put(rowName);
1016       p.add(family, qualifier, ee.currentTime(), rowName);
1017       r.put(p);
1018       puts.add(p);
1019     }
1020     return puts;
1021   }
1022 
1023   /*
1024    * Creates an HRI around an HTD that has <code>tableName</code> and three
1025    * column families named 'a','b', and 'c'.
1026    * @param tableName Name of table to use when we create HTableDescriptor.
1027    */
1028    private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
1029     return new HRegionInfo(tableName, null, null, false);
1030    }
1031 
1032   /*
1033    * Run the split.  Verify only single split file made.
1034    * @param c
1035    * @return The single split file made
1036    * @throws IOException
1037    */
1038   private Path runWALSplit(final Configuration c) throws IOException {
1039     List<Path> splits = WALSplitter.split(
1040       hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c, wals);
1041     // Split should generate only 1 file since there's only 1 region
1042     assertEquals("splits=" + splits, 1, splits.size());
1043     // Make sure the file exists
1044     assertTrue(fs.exists(splits.get(0)));
1045     LOG.info("Split file=" + splits.get(0));
1046     return splits.get(0);
1047   }
1048 
1049   /*
1050    * @param c
1051    * @return WAL with retries set down from 5 to 1 only.
1052    * @throws IOException
1053    */
1054   private WAL createWAL(final Configuration c) throws IOException {
1055     FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c);
1056     // Set down maximum recovery so we dfsclient doesn't linger retrying something
1057     // long gone.
1058     HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
1059     return wal;
1060   }
1061 
1062   private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
1063     HTableDescriptor htd = new HTableDescriptor(tableName);
1064     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
1065     htd.addFamily(a);
1066     HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
1067     htd.addFamily(b);
1068     HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
1069     htd.addFamily(c);
1070     return htd;
1071   }
1072 }