View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver.wal;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  import static org.junit.Assert.fail;
24  
25  import java.io.IOException;
26  import java.security.PrivilegedExceptionAction;
27  import java.util.ArrayList;
28  import java.util.List;
29  import java.util.SortedSet;
30  import java.util.concurrent.atomic.AtomicBoolean;
31  import java.util.concurrent.atomic.AtomicInteger;
32  import java.util.concurrent.atomic.AtomicLong;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.fs.FileStatus;
38  import org.apache.hadoop.fs.FileSystem;
39  import org.apache.hadoop.fs.Path;
40  import org.apache.hadoop.hbase.Cell;
41  import org.apache.hadoop.hbase.HBaseConfiguration;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.KeyValue;
48  import org.apache.hadoop.hbase.MasterNotRunningException;
49  import org.apache.hadoop.hbase.MediumTests;
50  import org.apache.hadoop.hbase.MiniHBaseCluster;
51  import org.apache.hadoop.hbase.ServerName;
52  import org.apache.hadoop.hbase.TableName;
53  import org.apache.hadoop.hbase.ZooKeeperConnectionException;
54  import org.apache.hadoop.hbase.client.Delete;
55  import org.apache.hadoop.hbase.client.Get;
56  import org.apache.hadoop.hbase.client.HTable;
57  import org.apache.hadoop.hbase.client.Put;
58  import org.apache.hadoop.hbase.client.Result;
59  import org.apache.hadoop.hbase.client.ResultScanner;
60  import org.apache.hadoop.hbase.client.Scan;
61  import org.apache.hadoop.hbase.master.HMaster;
62  import org.apache.hadoop.hbase.monitoring.MonitoredTask;
63  import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
64  import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
65  import org.apache.hadoop.hbase.regionserver.FlushRequester;
66  import org.apache.hadoop.hbase.regionserver.HRegion;
67  import org.apache.hadoop.hbase.regionserver.HRegionServer;
68  import org.apache.hadoop.hbase.regionserver.RegionScanner;
69  import org.apache.hadoop.hbase.regionserver.RegionServerServices;
70  import org.apache.hadoop.hbase.regionserver.Store;
71  import org.apache.hadoop.hbase.regionserver.TimeRangeTracker;
72  import org.apache.hadoop.hbase.security.User;
73  import org.apache.hadoop.hbase.util.Bytes;
74  import org.apache.hadoop.hbase.util.EnvironmentEdge;
75  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
76  import org.apache.hadoop.hbase.util.FSUtils;
77  import org.apache.hadoop.hbase.util.HFileTestUtil;
78  import org.apache.hadoop.hbase.util.Pair;
79  import org.junit.After;
80  import org.junit.AfterClass;
81  import org.junit.Before;
82  import org.junit.BeforeClass;
83  import org.junit.Test;
84  import org.junit.experimental.categories.Category;
85  import org.mockito.Mockito;
86  
87  /**
88   * Test replay of edits out of a WAL split.
89   */
90  @Category(MediumTests.class)
91  public class TestWALReplay {
92    public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
93    static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
94    private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
95    private Path hbaseRootDir = null;
96    private String logName;
97    private Path oldLogDir;
98    private Path logDir;
99    private FileSystem fs;
100   private Configuration conf;
101 
102   @BeforeClass
103   public static void setUpBeforeClass() throws Exception {
104     Configuration conf = TEST_UTIL.getConfiguration();
105     conf.setBoolean("dfs.support.append", true);
106     // The below config supported by 0.20-append and CDH3b2
107     conf.setInt("dfs.client.block.recovery.retries", 2);
108     TEST_UTIL.startMiniCluster(3);
109     Path hbaseRootDir =
110       TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
111     LOG.info("hbase.rootdir=" + hbaseRootDir);
112     FSUtils.setRootDir(conf, hbaseRootDir);
113   }
114 
115   @AfterClass
116   public static void tearDownAfterClass() throws Exception {
117     TEST_UTIL.shutdownMiniCluster();
118   }
119 
120   @Before
121   public void setUp() throws Exception {
122     this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
123     this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
124     this.hbaseRootDir = FSUtils.getRootDir(this.conf);
125     this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
126     this.logName = HConstants.HREGION_LOGDIR_NAME;
127     this.logDir = new Path(this.hbaseRootDir, logName);
128     if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
129       TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
130     }
131   }
132 
133   @After
134   public void tearDown() throws Exception {
135     TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
136   }
137 
138   /*
139    * @param p Directory to cleanup
140    */
141   private void deleteDir(final Path p) throws IOException {
142     if (this.fs.exists(p)) {
143       if (!this.fs.delete(p, true)) {
144         throw new IOException("Failed remove of " + p);
145       }
146     }
147   }
148 
149   /**
150    * 
151    * @throws Exception
152    */
153   @Test
154   public void testReplayEditsAfterRegionMovedWithMultiCF() throws Exception {
155     final TableName tableName =
156         TableName.valueOf("testReplayEditsAfterRegionMovedWithMultiCF");
157     byte[] family1 = Bytes.toBytes("cf1");
158     byte[] family2 = Bytes.toBytes("cf2");
159     byte[] qualifier = Bytes.toBytes("q");
160     byte[] value = Bytes.toBytes("testV");
161     byte[][] familys = { family1, family2 };
162     TEST_UTIL.createTable(tableName, familys);
163     HTable htable = new HTable(TEST_UTIL.getConfiguration(), tableName);
164     Put put = new Put(Bytes.toBytes("r1"));
165     put.add(family1, qualifier, value);
166     htable.put(put);
167     ResultScanner resultScanner = htable.getScanner(new Scan());
168     int count = 0;
169     while (resultScanner.next() != null) {
170       count++;
171     }
172     resultScanner.close();
173     assertEquals(1, count);
174 
175     MiniHBaseCluster hbaseCluster = TEST_UTIL.getMiniHBaseCluster();
176     List<HRegion> regions = hbaseCluster.getRegions(tableName);
177     assertEquals(1, regions.size());
178 
179     // move region to another regionserver
180     HRegion destRegion = regions.get(0);
181     int originServerNum = hbaseCluster
182         .getServerWith(destRegion.getRegionName());
183     assertTrue("Please start more than 1 regionserver", hbaseCluster
184         .getRegionServerThreads().size() > 1);
185     int destServerNum = 0;
186     while (destServerNum == originServerNum) {
187       destServerNum++;
188     }
189     HRegionServer originServer = hbaseCluster.getRegionServer(originServerNum);
190     HRegionServer destServer = hbaseCluster.getRegionServer(destServerNum);
191     // move region to destination regionserver
192     moveRegionAndWait(destRegion, destServer);
193 
194     // delete the row
195     Delete del = new Delete(Bytes.toBytes("r1"));
196     htable.delete(del);
197     resultScanner = htable.getScanner(new Scan());
198     count = 0;
199     while (resultScanner.next() != null) {
200       count++;
201     }
202     resultScanner.close();
203     assertEquals(0, count);
204 
205     // flush region and make major compaction
206     destServer.getOnlineRegion(destRegion.getRegionName()).flushcache();
207     // wait to complete major compaction
208     for (Store store : destServer.getOnlineRegion(destRegion.getRegionName())
209         .getStores().values()) {
210       store.triggerMajorCompaction();
211     }
212     destServer.getOnlineRegion(destRegion.getRegionName()).compactStores();
213 
214     // move region to origin regionserver
215     moveRegionAndWait(destRegion, originServer);
216     // abort the origin regionserver
217     originServer.abort("testing");
218 
219     // see what we get
220     Result result = htable.get(new Get(Bytes.toBytes("r1")));
221     if (result != null) {
222       assertTrue("Row is deleted, but we get" + result.toString(),
223           (result == null) || result.isEmpty());
224     }
225     resultScanner.close();
226   }
227 
228   private void moveRegionAndWait(HRegion destRegion, HRegionServer destServer)
229       throws InterruptedException, MasterNotRunningException,
230       ZooKeeperConnectionException, IOException {
231     HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
232     TEST_UTIL.getHBaseAdmin().move(
233         destRegion.getRegionInfo().getEncodedNameAsBytes(),
234         Bytes.toBytes(destServer.getServerName().getServerName()));
235     while (true) {
236       ServerName serverName = master.getAssignmentManager()
237         .getRegionStates().getRegionServerOfRegion(destRegion.getRegionInfo());
238       if (serverName != null && serverName.equals(destServer.getServerName())) {
239         TEST_UTIL.assertRegionOnServer(
240           destRegion.getRegionInfo(), serverName, 200);
241         break;
242       }
243       Thread.sleep(10);
244     }
245   }
246 
247   /**
248    * Tests for hbase-2727.
249    * @throws Exception
250    * @see https://issues.apache.org/jira/browse/HBASE-2727
251    */
252   @Test
253   public void test2727() throws Exception {
254     // Test being able to have > 1 set of edits in the recovered.edits directory.
255     // Ensure edits are replayed properly.
256     final TableName tableName =
257         TableName.valueOf("test2727");
258     HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
259     Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
260     deleteDir(basedir);
261 
262     HTableDescriptor htd = createBasic3FamilyHTD(tableName);
263     HRegion region2 = HRegion.createHRegion(hri,
264         hbaseRootDir, this.conf, htd);
265     HRegion.closeHRegion(region2);
266     final byte [] rowName = tableName.getName();
267 
268     HLog wal1 = createWAL(this.conf);
269     // Add 1k to each family.
270     final int countPerFamily = 1000;
271     final AtomicLong sequenceId = new AtomicLong(1);
272     for (HColumnDescriptor hcd: htd.getFamilies()) {
273       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee,
274           wal1, htd, sequenceId);
275     }
276     wal1.close();
277     runWALSplit(this.conf);
278 
279     HLog wal2 = createWAL(this.conf);
280     // Add 1k to each family.
281     for (HColumnDescriptor hcd: htd.getFamilies()) {
282       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
283           ee, wal2, htd, sequenceId);
284     }
285     wal2.close();
286     runWALSplit(this.conf);
287 
288     HLog wal3 = createWAL(this.conf);
289     try {
290       HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal3);
291       long seqid = region.getOpenSeqNum();
292       // The regions opens with sequenceId as 1. With 6k edits, its sequence number reaches 6k + 1.
293       // When opened, this region would apply 6k edits, and increment the sequenceId by 1
294       assertTrue(seqid > sequenceId.get());
295       assertEquals(seqid - 1, sequenceId.get());
296       LOG.debug("region.getOpenSeqNum(): " + region.getOpenSeqNum() + ", wal3.id: "
297           + sequenceId.get());
298 
299       // TODO: Scan all.
300       region.close();
301     } finally {
302       wal3.closeAndDelete();
303     }
304   }
305 
306   /**
307    * Test case of HRegion that is only made out of bulk loaded files.  Assert
308    * that we don't 'crash'.
309    * @throws IOException
310    * @throws IllegalAccessException
311    * @throws NoSuchFieldException
312    * @throws IllegalArgumentException
313    * @throws SecurityException
314    */
315   @Test
316   public void testRegionMadeOfBulkLoadedFilesOnly()
317   throws IOException, SecurityException, IllegalArgumentException,
318       NoSuchFieldException, IllegalAccessException, InterruptedException {
319     final TableName tableName =
320         TableName.valueOf("testRegionMadeOfBulkLoadedFilesOnly");
321     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
322     final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
323     deleteDir(basedir);
324     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
325     HRegion region2 = HRegion.createHRegion(hri,
326         hbaseRootDir, this.conf, htd);
327     HRegion.closeHRegion(region2);
328     HLog wal = createWAL(this.conf);
329     HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
330 
331     byte [] family = htd.getFamilies().iterator().next().getName();
332     Path f =  new Path(basedir, "hfile");
333     HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(""),
334         Bytes.toBytes("z"), 10);
335     List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
336     hfs.add(Pair.newPair(family, f.toString()));
337     region.bulkLoadHFiles(hfs, true);
338 
339     // Add an edit so something in the WAL
340     byte [] row = tableName.getName();
341     region.put((new Put(row)).add(family, family, family));
342     wal.sync();
343     final int rowsInsertedCount = 11;
344 
345     assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
346 
347     // Now 'crash' the region by stealing its wal
348     final Configuration newConf = HBaseConfiguration.create(this.conf);
349     User user = HBaseTestingUtility.getDifferentUser(newConf,
350         tableName.getNameAsString());
351     user.runAs(new PrivilegedExceptionAction() {
352       public Object run() throws Exception {
353         runWALSplit(newConf);
354         HLog wal2 = createWAL(newConf);
355 
356         HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
357           hbaseRootDir, hri, htd, wal2);
358         long seqid2 = region2.getOpenSeqNum();
359         assertTrue(seqid2 > -1);
360         assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
361 
362         // I can't close wal1.  Its been appropriated when we split.
363         region2.close();
364         wal2.closeAndDelete();
365         return null;
366       }
367     });
368   }
369 
370   /**
371    * HRegion test case that is made of a major compacted HFile (created with three bulk loaded
372    * files) and an edit in the memstore.
373    * This is for HBASE-10958 "[dataloss] Bulk loading with seqids can prevent some log entries
374    * from being replayed"
375    * @throws IOException
376    * @throws IllegalAccessException
377    * @throws NoSuchFieldException
378    * @throws IllegalArgumentException
379    * @throws SecurityException
380    */
381   @Test
382   public void testCompactedBulkLoadedFiles()
383       throws IOException, SecurityException, IllegalArgumentException,
384       NoSuchFieldException, IllegalAccessException, InterruptedException {
385     final TableName tableName =
386         TableName.valueOf("testCompactedBulkLoadedFiles");
387     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
388     final Path basedir = new Path(this.hbaseRootDir, tableName.getNameAsString());
389     deleteDir(basedir);
390     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
391     HRegion region2 = HRegion.createHRegion(hri,
392         hbaseRootDir, this.conf, htd);
393     HRegion.closeHRegion(region2);
394     HLog wal = createWAL(this.conf);
395     HRegion region = HRegion.openHRegion(hri, htd, wal, this.conf);
396 
397     // Add an edit so something in the WAL
398     byte [] row = tableName.getName();
399     byte [] family = htd.getFamilies().iterator().next().getName();
400     region.put((new Put(row)).add(family, family, family));
401     wal.sync();
402 
403     List <Pair<byte[],String>>  hfs= new ArrayList<Pair<byte[],String>>(1);
404     for (int i = 0; i < 3; i++) {
405       Path f = new Path(basedir, "hfile"+i);
406       HFileTestUtil.createHFile(this.conf, fs, f, family, family, Bytes.toBytes(i + "00"),
407           Bytes.toBytes(i + "50"), 10);
408       hfs.add(Pair.newPair(family, f.toString()));
409     }
410     region.bulkLoadHFiles(hfs, true);
411     final int rowsInsertedCount = 31;
412     assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
413 
414     // major compact to turn all the bulk loaded files into one normal file
415     region.compactStores(true);
416     assertEquals(rowsInsertedCount, getScannedCount(region.getScanner(new Scan())));
417 
418     // Now 'crash' the region by stealing its wal
419     final Configuration newConf = HBaseConfiguration.create(this.conf);
420     User user = HBaseTestingUtility.getDifferentUser(newConf,
421         tableName.getNameAsString());
422     user.runAs(new PrivilegedExceptionAction() {
423       public Object run() throws Exception {
424         runWALSplit(newConf);
425         HLog wal2 = createWAL(newConf);
426 
427         HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf),
428             hbaseRootDir, hri, htd, wal2);
429         long seqid2 = region2.getOpenSeqNum();
430         assertTrue(seqid2 > -1);
431         assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan())));
432 
433         // I can't close wal1.  Its been appropriated when we split.
434         region2.close();
435         wal2.closeAndDelete();
436         return null;
437       }
438     });
439   }
440 
441 
442   /**
443    * Test writing edits into an HRegion, closing it, splitting logs, opening
444    * Region again.  Verify seqids.
445    * @throws IOException
446    * @throws IllegalAccessException
447    * @throws NoSuchFieldException
448    * @throws IllegalArgumentException
449    * @throws SecurityException
450    */
451   @Test
452   public void testReplayEditsWrittenViaHRegion()
453   throws IOException, SecurityException, IllegalArgumentException,
454       NoSuchFieldException, IllegalAccessException, InterruptedException {
455     final TableName tableName =
456         TableName.valueOf("testReplayEditsWrittenViaHRegion");
457     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
458     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
459     deleteDir(basedir);
460     final byte[] rowName = tableName.getName();
461     final int countPerFamily = 10;
462     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
463     HRegion region3 = HRegion.createHRegion(hri,
464             hbaseRootDir, this.conf, htd);
465     HRegion.closeHRegion(region3);
466     // Write countPerFamily edits into the three families.  Do a flush on one
467     // of the families during the load of edits so its seqid is not same as
468     // others to test we do right thing when different seqids.
469     HLog wal = createWAL(this.conf);
470     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
471     long seqid = region.getOpenSeqNum();
472     boolean first = true;
473     for (HColumnDescriptor hcd: htd.getFamilies()) {
474       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
475       if (first ) {
476         // If first, so we have at least one family w/ different seqid to rest.
477         region.flushcache();
478         first = false;
479       }
480     }
481     // Now assert edits made it in.
482     final Get g = new Get(rowName);
483     Result result = region.get(g);
484     assertEquals(countPerFamily * htd.getFamilies().size(),
485       result.size());
486     // Now close the region (without flush), split the log, reopen the region and assert that
487     // replay of log has the correct effect, that our seqids are calculated correctly so
488     // all edits in logs are seen as 'stale'/old.
489     region.close(true);
490     wal.close();
491     runWALSplit(this.conf);
492     HLog wal2 = createWAL(this.conf);
493     HRegion region2 = HRegion.openHRegion(conf, this.fs, hbaseRootDir, hri, htd, wal2);
494     long seqid2 = region2.getOpenSeqNum();
495     assertTrue(seqid + result.size() < seqid2);
496     final Result result1b = region2.get(g);
497     assertEquals(result.size(), result1b.size());
498 
499     // Next test.  Add more edits, then 'crash' this region by stealing its wal
500     // out from under it and assert that replay of the log adds the edits back
501     // correctly when region is opened again.
502     for (HColumnDescriptor hcd: htd.getFamilies()) {
503       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
504     }
505     // Get count of edits.
506     final Result result2 = region2.get(g);
507     assertEquals(2 * result.size(), result2.size());
508     wal2.sync();
509     // Set down maximum recovery so we dfsclient doesn't linger retrying something
510     // long gone.
511     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal2).getOutputStream(), 1);
512     final Configuration newConf = HBaseConfiguration.create(this.conf);
513     User user = HBaseTestingUtility.getDifferentUser(newConf,
514       tableName.getNameAsString());
515     user.runAs(new PrivilegedExceptionAction() {
516       public Object run() throws Exception {
517         runWALSplit(newConf);
518         FileSystem newFS = FileSystem.get(newConf);
519         // Make a new wal for new region open.
520         HLog wal3 = createWAL(newConf);
521         final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
522         HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, htd, null) {
523           @Override
524           protected boolean restoreEdit(Store s, KeyValue kv) {
525             boolean b = super.restoreEdit(s, kv);
526             countOfRestoredEdits.incrementAndGet();
527             return b;
528           }
529         };
530         long seqid3 = region3.initialize();
531         Result result3 = region3.get(g);
532         // Assert that count of cells is same as before crash.
533         assertEquals(result2.size(), result3.size());
534         assertEquals(htd.getFamilies().size() * countPerFamily,
535           countOfRestoredEdits.get());
536 
537         // I can't close wal1.  Its been appropriated when we split.
538         region3.close();
539         wal3.closeAndDelete();
540         return null;
541       }
542     });
543   }
544 
545   /**
546    * Test that we recover correctly when there is a failure in between the
547    * flushes. i.e. Some stores got flushed but others did not.
548    *
549    * Unfortunately, there is no easy hook to flush at a store level. The way
550    * we get around this is by flushing at the region level, and then deleting
551    * the recently flushed store file for one of the Stores. This would put us
552    * back in the situation where all but that store got flushed and the region
553    * died.
554    *
555    * We restart Region again, and verify that the edits were replayed.
556    *
557    * @throws IOException
558    * @throws IllegalAccessException
559    * @throws NoSuchFieldException
560    * @throws IllegalArgumentException
561    * @throws SecurityException
562    */
563   @Test
564   public void testReplayEditsAfterPartialFlush()
565   throws IOException, SecurityException, IllegalArgumentException,
566       NoSuchFieldException, IllegalAccessException, InterruptedException {
567     final TableName tableName =
568         TableName.valueOf("testReplayEditsWrittenViaHRegion");
569     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
570     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
571     deleteDir(basedir);
572     final byte[] rowName = tableName.getName();
573     final int countPerFamily = 10;
574     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
575     HRegion region3 = HRegion.createHRegion(hri,
576             hbaseRootDir, this.conf, htd);
577     HRegion.closeHRegion(region3);
578     // Write countPerFamily edits into the three families.  Do a flush on one
579     // of the families during the load of edits so its seqid is not same as
580     // others to test we do right thing when different seqids.
581     HLog wal = createWAL(this.conf);
582     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
583     long seqid = region.getOpenSeqNum();
584     for (HColumnDescriptor hcd: htd.getFamilies()) {
585       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
586     }
587 
588     // Now assert edits made it in.
589     final Get g = new Get(rowName);
590     Result result = region.get(g);
591     assertEquals(countPerFamily * htd.getFamilies().size(),
592       result.size());
593 
594     // Let us flush the region
595     region.flushcache();
596     region.close(true);
597     wal.close();
598 
599     // delete the store files in the second column family to simulate a failure
600     // in between the flushcache();
601     // we have 3 families. killing the middle one ensures that taking the maximum
602     // will make us fail.
603     int cf_count = 0;
604     for (HColumnDescriptor hcd: htd.getFamilies()) {
605       cf_count++;
606       if (cf_count == 2) {
607         region.getRegionFileSystem().deleteFamily(hcd.getNameAsString());
608       }
609     }
610 
611 
612     // Let us try to split and recover
613     runWALSplit(this.conf);
614     HLog wal2 = createWAL(this.conf);
615     HRegion region2 = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal2);
616     long seqid2 = region2.getOpenSeqNum();
617     assertTrue(seqid + result.size() < seqid2);
618 
619     final Result result1b = region2.get(g);
620     assertEquals(result.size(), result1b.size());
621   }
622 
623 
624   // StoreFlusher implementation used in testReplayEditsAfterAbortingFlush.
625   // Only throws exception if throwExceptionWhenFlushing is set true.
626   public static class CustomStoreFlusher extends DefaultStoreFlusher {
627     // Switch between throw and not throw exception in flush
628     static final AtomicBoolean throwExceptionWhenFlushing = new AtomicBoolean(false);
629 
630     public CustomStoreFlusher(Configuration conf, Store store) {
631       super(conf, store);
632     }
633     @Override
634     public List<Path> flushSnapshot(SortedSet<KeyValue> snapshot, long cacheFlushId,
635         TimeRangeTracker snapshotTimeRangeTracker, AtomicLong flushedSize, MonitoredTask status)
636             throws IOException {
637       if (throwExceptionWhenFlushing.get()) {
638         throw new IOException("Simulated exception by tests");
639       }
640       return super.flushSnapshot(snapshot, cacheFlushId, snapshotTimeRangeTracker,
641           flushedSize, status);
642     }
643 
644   };
645 
646   /**
647    * Test that we could recover the data correctly after aborting flush. In the
648    * test, first we abort flush after writing some data, then writing more data
649    * and flush again, at last verify the data.
650    * @throws IOException
651    */
652   @Test
653   public void testReplayEditsAfterAbortingFlush() throws IOException {
654     final TableName tableName =
655         TableName.valueOf("testReplayEditsAfterAbortingFlush");
656     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
657     final Path basedir = FSUtils.getTableDir(this.hbaseRootDir, tableName);
658     deleteDir(basedir);
659     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
660     HRegion region3 = HRegion.createHRegion(hri, hbaseRootDir, this.conf, htd);
661     region3.close();
662     region3.getLog().closeAndDelete();
663     // Write countPerFamily edits into the three families. Do a flush on one
664     // of the families during the load of edits so its seqid is not same as
665     // others to test we do right thing when different seqids.
666     HLog wal = createWAL(this.conf);
667     RegionServerServices rsServices = Mockito.mock(RegionServerServices.class);
668     Mockito.doReturn(false).when(rsServices).isAborted();
669     Configuration customConf = new Configuration(this.conf);
670     customConf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
671         CustomStoreFlusher.class.getName());
672     HRegion region =
673       HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal, customConf, rsServices, null);
674     int writtenRowCount = 10;
675     List<HColumnDescriptor> families = new ArrayList<HColumnDescriptor>(
676         htd.getFamilies());
677     for (int i = 0; i < writtenRowCount; i++) {
678       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
679       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
680           Bytes.toBytes("val"));
681       region.put(put);
682     }
683 
684     // Now assert edits made it in.
685     RegionScanner scanner = region.getScanner(new Scan());
686     assertEquals(writtenRowCount, getScannedCount(scanner));
687 
688     // Let us flush the region
689     CustomStoreFlusher.throwExceptionWhenFlushing.set(true);
690     try {
691       region.flushcache();
692       fail("Injected exception hasn't been thrown");
693     } catch (Throwable t) {
694       LOG.info("Expected simulated exception when flushing region,"
695           + t.getMessage());
696       // simulated to abort server
697       Mockito.doReturn(true).when(rsServices).isAborted();
698     }
699     // writing more data
700     int moreRow = 10;
701     for (int i = writtenRowCount; i < writtenRowCount + moreRow; i++) {
702       Put put = new Put(Bytes.toBytes(tableName + Integer.toString(i)));
703       put.add(families.get(i % families.size()).getName(), Bytes.toBytes("q"),
704           Bytes.toBytes("val"));
705       region.put(put);
706     }
707     writtenRowCount += moreRow;
708     // call flush again
709     CustomStoreFlusher.throwExceptionWhenFlushing.set(false);
710     try {
711       region.flushcache();
712     } catch (IOException t) {
713       LOG.info("Expected exception when flushing region because server is stopped,"
714           + t.getMessage());
715     }
716 
717     region.close(true);
718     wal.close();
719 
720     // Let us try to split and recover
721     runWALSplit(this.conf);
722     HLog wal2 = createWAL(this.conf);
723     Mockito.doReturn(false).when(rsServices).isAborted();
724     HRegion region2 =
725       HRegion.openHRegion(this.hbaseRootDir, hri, htd, wal2, this.conf, rsServices, null);
726     scanner = region2.getScanner(new Scan());
727     assertEquals(writtenRowCount, getScannedCount(scanner));
728   }
729 
730   private int getScannedCount(RegionScanner scanner) throws IOException {
731     int scannedCount = 0;
732     List<Cell> results = new ArrayList<Cell>();
733     while (true) {
734       boolean existMore = scanner.next(results);
735       if (!results.isEmpty())
736         scannedCount++;
737       if (!existMore)
738         break;
739       results.clear();
740     }
741     return scannedCount;
742   }
743 
744   /**
745    * Create an HRegion with the result of a HLog split and test we only see the
746    * good edits
747    * @throws Exception
748    */
749   @Test
750   public void testReplayEditsWrittenIntoWAL() throws Exception {
751     final TableName tableName =
752         TableName.valueOf("testReplayEditsWrittenIntoWAL");
753     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
754     final Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName);
755     deleteDir(basedir);
756 
757     final HTableDescriptor htd = createBasic3FamilyHTD(tableName);
758     HRegion region2 = HRegion.createHRegion(hri,
759             hbaseRootDir, this.conf, htd);
760     HRegion.closeHRegion(region2);
761     final HLog wal = createWAL(this.conf);
762     final byte[] rowName = tableName.getName();
763     final byte[] regionName = hri.getEncodedNameAsBytes();
764     final AtomicLong sequenceId = new AtomicLong(1);
765 
766     // Add 1k to each family.
767     final int countPerFamily = 1000;
768     for (HColumnDescriptor hcd: htd.getFamilies()) {
769       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily,
770           ee, wal, htd, sequenceId);
771     }
772 
773     // Add a cache flush, shouldn't have any effect
774     wal.startCacheFlush(regionName);
775     wal.completeCacheFlush(regionName);
776 
777     // Add an edit to another family, should be skipped.
778     WALEdit edit = new WALEdit();
779     long now = ee.currentTimeMillis();
780     edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
781       now, rowName));
782     wal.append(hri, tableName, edit, now, htd, sequenceId);
783 
784     // Delete the c family to verify deletes make it over.
785     edit = new WALEdit();
786     now = ee.currentTimeMillis();
787     edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
788       KeyValue.Type.DeleteFamily));
789     wal.append(hri, tableName, edit, now, htd, sequenceId);
790 
791     // Sync.
792     wal.sync();
793     // Set down maximum recovery so we dfsclient doesn't linger retrying something
794     // long gone.
795     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
796     // Make a new conf and a new fs for the splitter to run on so we can take
797     // over old wal.
798     final Configuration newConf = HBaseConfiguration.create(this.conf);
799     User user = HBaseTestingUtility.getDifferentUser(newConf,
800       ".replay.wal.secondtime");
801     user.runAs(new PrivilegedExceptionAction() {
802       public Object run() throws Exception {
803         runWALSplit(newConf);
804         FileSystem newFS = FileSystem.get(newConf);
805         // 100k seems to make for about 4 flushes during HRegion#initialize.
806         newConf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 100);
807         // Make a new wal for new region.
808         HLog newWal = createWAL(newConf);
809         final AtomicInteger flushcount = new AtomicInteger(0);
810         try {
811           final HRegion region =
812               new HRegion(basedir, newWal, newFS, newConf, hri, htd, null) {
813             protected FlushResult internalFlushcache(
814                 final HLog wal, final long myseqid, MonitoredTask status)
815             throws IOException {
816               LOG.info("InternalFlushCache Invoked");
817               FlushResult fs = super.internalFlushcache(wal, myseqid,
818                   Mockito.mock(MonitoredTask.class));
819               flushcount.incrementAndGet();
820               return fs;
821             };
822           };
823           long seqid = region.initialize();
824           // We flushed during init.
825           assertTrue("Flushcount=" + flushcount.get(), flushcount.get() > 0);
826           assertTrue(seqid - 1 == sequenceId.get());
827 
828           Get get = new Get(rowName);
829           Result result = region.get(get);
830           // Make sure we only see the good edits
831           assertEquals(countPerFamily * (htd.getFamilies().size() - 1),
832             result.size());
833           region.close();
834         } finally {
835           newWal.closeAndDelete();
836         }
837         return null;
838       }
839     });
840   }
841 
842   @Test
843   // the following test is for HBASE-6065
844   public void testSequentialEditLogSeqNum() throws IOException {
845     final TableName tableName =
846         TableName.valueOf("testSequentialEditLogSeqNum");
847     final HRegionInfo hri = createBasic3FamilyHRegionInfo(tableName);
848     final Path basedir =
849         FSUtils.getTableDir(this.hbaseRootDir, tableName);
850     deleteDir(basedir);
851     final byte[] rowName = tableName.getName();
852     final int countPerFamily = 10;
853     final HTableDescriptor htd = createBasic1FamilyHTD(tableName);
854 
855     // Mock the HLog
856     MockHLog wal = createMockWAL(this.conf);
857 
858     HRegion region = HRegion.openHRegion(this.conf, this.fs, hbaseRootDir, hri, htd, wal);
859     for (HColumnDescriptor hcd : htd.getFamilies()) {
860       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
861     }
862 
863     // Let us flush the region
864     // But this time completeflushcache is not yet done
865     region.flushcache();
866     for (HColumnDescriptor hcd : htd.getFamilies()) {
867       addRegionEdits(rowName, hcd.getName(), 5, this.ee, region, "x");
868     }
869     long lastestSeqNumber = region.getSequenceId().get();
870     // get the current seq no
871     wal.doCompleteCacheFlush = true;
872     // allow complete cache flush with the previous seq number got after first
873     // set of edits.
874     wal.completeCacheFlush(hri.getEncodedNameAsBytes());
875     wal.close();
876     FileStatus[] listStatus = this.fs.listStatus(wal.getDir());
877     HLogSplitter.splitLogFile(hbaseRootDir, listStatus[0],
878       this.fs, this.conf, null, null, null);
879     FileStatus[] listStatus1 = this.fs.listStatus(
880         new Path(FSUtils.getTableDir(hbaseRootDir, tableName),
881             new Path(hri.getEncodedName(), "recovered.edits")));
882     int editCount = 0;
883     for (FileStatus fileStatus : listStatus1) {
884       editCount = Integer.parseInt(fileStatus.getPath().getName());
885     }
886     // The sequence number should be same 
887     assertEquals(
888         "The sequence number of the recoverd.edits and the current edit seq should be same",
889         lastestSeqNumber, editCount);
890   }
891 
892   static class MockHLog extends FSHLog {
893     boolean doCompleteCacheFlush = false;
894 
895     public MockHLog(FileSystem fs, Path rootDir, String logName, Configuration conf) throws IOException {
896       super(fs, rootDir, logName, conf);
897     }
898 
899     @Override
900     public void completeCacheFlush(byte[] encodedRegionName) {
901       if (!doCompleteCacheFlush) {
902         return;
903       }
904       super.completeCacheFlush(encodedRegionName);
905     }
906   }
907 
908   private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
909     HTableDescriptor htd = new HTableDescriptor(tableName);
910     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
911     htd.addFamily(a);
912     return htd;
913   }
914   
915   private MockHLog createMockWAL(Configuration conf) throws IOException {
916     MockHLog wal = new MockHLog(FileSystem.get(conf), hbaseRootDir, logName, conf);
917     // Set down maximum recovery so we dfsclient doesn't linger retrying something
918     // long gone.
919     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
920     return wal;
921   }
922 
923   // Flusher used in this test.  Keep count of how often we are called and
924   // actually run the flush inside here.
925   class TestFlusher implements FlushRequester {
926     private HRegion r;
927 
928     @Override
929     public void requestFlush(HRegion region) {
930       try {
931         r.flushcache();
932       } catch (IOException e) {
933         throw new RuntimeException("Exception flushing", e);
934       }
935     }
936 
937     @Override
938     public void requestDelayedFlush(HRegion region, long when) {
939       // TODO Auto-generated method stub
940       
941     }
942   }
943 
944   private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
945       final byte[] family, final int count, EnvironmentEdge ee, final HLog wal,
946       final HTableDescriptor htd, final AtomicLong sequenceId)
947   throws IOException {
948     String familyStr = Bytes.toString(family);
949     for (int j = 0; j < count; j++) {
950       byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
951       byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
952       WALEdit edit = new WALEdit();
953       edit.add(new KeyValue(rowName, family, qualifierBytes,
954         ee.currentTimeMillis(), columnBytes));
955       wal.append(hri, tableName, edit, ee.currentTimeMillis(), htd, sequenceId);
956     }
957   }
958 
959   private void addRegionEdits (final byte [] rowName, final byte [] family,
960       final int count, EnvironmentEdge ee, final HRegion r,
961       final String qualifierPrefix)
962   throws IOException {
963     for (int j = 0; j < count; j++) {
964       byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
965       Put p = new Put(rowName);
966       p.add(family, qualifier, ee.currentTimeMillis(), rowName);
967       r.put(p);
968     }
969   }
970 
971   /*
972    * Creates an HRI around an HTD that has <code>tableName</code> and three
973    * column families named 'a','b', and 'c'.
974    * @param tableName Name of table to use when we create HTableDescriptor.
975    */
976    private HRegionInfo createBasic3FamilyHRegionInfo(final TableName tableName) {
977     return new HRegionInfo(tableName, null, null, false);
978    }
979 
980   /*
981    * Run the split.  Verify only single split file made.
982    * @param c
983    * @return The single split file made
984    * @throws IOException
985    */
986   private Path runWALSplit(final Configuration c) throws IOException {
987     List<Path> splits = HLogSplitter.split(
988       hbaseRootDir, logDir, oldLogDir, FileSystem.get(c), c);
989     // Split should generate only 1 file since there's only 1 region
990     assertEquals("splits=" + splits, 1, splits.size());
991     // Make sure the file exists
992     assertTrue(fs.exists(splits.get(0)));
993     LOG.info("Split file=" + splits.get(0));
994     return splits.get(0);
995   }
996 
997   /*
998    * @param c
999    * @return WAL with retries set down from 5 to 1 only.
1000    * @throws IOException
1001    */
1002   private HLog createWAL(final Configuration c) throws IOException {
1003     HLog wal = HLogFactory.createHLog(FileSystem.get(c), 
1004         hbaseRootDir, logName, c);
1005     // Set down maximum recovery so we dfsclient doesn't linger retrying something
1006     // long gone.
1007     HBaseTestingUtility.setMaxRecoveryErrorCount(((FSHLog) wal).getOutputStream(), 1);
1008     return wal;
1009   }
1010 
1011   private HTableDescriptor createBasic3FamilyHTD(final TableName tableName) {
1012     HTableDescriptor htd = new HTableDescriptor(tableName);
1013     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
1014     htd.addFamily(a);
1015     HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
1016     htd.addFamily(b);
1017     HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
1018     htd.addFamily(c);
1019     return htd;
1020   }
1021 
1022 }
1023