1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.List;
27  import java.util.concurrent.atomic.AtomicInteger;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FileSystem;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.hbase.HBaseConfiguration;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HColumnDescriptor;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.HRegionInfo;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.KeyValue;
41  import org.apache.hadoop.hbase.client.Get;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.client.Result;
44  import org.apache.hadoop.hbase.io.hfile.HFile;
45  import org.apache.hadoop.hbase.regionserver.HRegion;
46  import org.apache.hadoop.hbase.regionserver.Store;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.apache.hadoop.hbase.util.EnvironmentEdge;
49  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
50  import org.junit.After;
51  import org.junit.AfterClass;
52  import org.junit.Before;
53  import org.junit.BeforeClass;
54  import org.junit.Test;
55  
56  /**
57   * Test replay of edits out of a WAL split.
58   */
59  public class TestWALReplay {
60    public static final Log LOG = LogFactory.getLog(TestWALReplay.class);
61    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
62    private final EnvironmentEdge ee = EnvironmentEdgeManager.getDelegate();
63    private Path hbaseRootDir = null;
64    private Path oldLogDir;
65    private Path logDir;
66    private FileSystem fs;
67    private Configuration conf;
68  
69    @BeforeClass
70    public static void setUpBeforeClass() throws Exception {
71      Configuration conf = TEST_UTIL.getConfiguration();
72      conf.setBoolean("dfs.support.append", true);
73      // The below config supported by 0.20-append and CDH3b2
74      conf.setInt("dfs.client.block.recovery.retries", 2);
75      conf.setInt("hbase.regionserver.flushlogentries", 1);
76      TEST_UTIL.startMiniDFSCluster(3);
77      TEST_UTIL.setNameNodeNameSystemLeasePeriod(100, 10000);
78      Path hbaseRootDir =
79        TEST_UTIL.getDFSCluster().getFileSystem().makeQualified(new Path("/hbase"));
80      LOG.info("hbase.rootdir=" + hbaseRootDir);
81      conf.set(HConstants.HBASE_DIR, hbaseRootDir.toString());
82    }
83  
84    @AfterClass
85    public static void tearDownAfterClass() throws Exception {
86      TEST_UTIL.shutdownMiniDFSCluster();
87    }
88  
89    @Before
90    public void setUp() throws Exception {
91      this.conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
92      this.fs = TEST_UTIL.getDFSCluster().getFileSystem();
93      this.hbaseRootDir = new Path(this.conf.get(HConstants.HBASE_DIR));
94      this.oldLogDir = new Path(this.hbaseRootDir, HConstants.HREGION_OLDLOGDIR_NAME);
95      this.logDir = new Path(this.hbaseRootDir, HConstants.HREGION_LOGDIR_NAME);
96      if (TEST_UTIL.getDFSCluster().getFileSystem().exists(this.hbaseRootDir)) {
97        TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
98      }
99    }
100 
101   @After
102   public void tearDown() throws Exception {
103     TEST_UTIL.getDFSCluster().getFileSystem().delete(this.hbaseRootDir, true);
104   }
105 
106   /*
107    * @param p Directory to cleanup
108    */
109   private void deleteDir(final Path p) throws IOException {
110     if (this.fs.exists(p)) {
111       if (!this.fs.delete(p, true)) {
112         throw new IOException("Failed remove of " + p);
113       }
114     }
115   }
116 
117   /**
118    * Tests for hbase-2727.
119    * @throws Exception
120    * @see https://issues.apache.org/jira/browse/HBASE-2727
121    */
122   @Test
123   public void test2727() throws Exception {
124     // Test being able to have > 1 set of edits in the recovered.edits directory.
125     // Ensure edits are replayed properly.
126     final String tableNameStr = "test2727";
127     HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
128     Path basedir = new Path(hbaseRootDir, tableNameStr);
129     deleteDir(basedir);
130 
131     final byte [] tableName = Bytes.toBytes(tableNameStr);
132     final byte [] rowName = tableName;
133 
134     HLog wal1 = createWAL(this.conf);
135     // Add 1k to each family.
136     final int countPerFamily = 1000;
137     for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
138       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee, wal1);
139     }
140     wal1.close();
141     runWALSplit(this.conf);
142 
143     HLog wal2 = createWAL(this.conf);
144     // Up the sequenceid so that these edits are after the ones added above.
145     wal2.setSequenceNumber(wal1.getSequenceNumber());
146     // Add 1k to each family.
147     for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
148       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee, wal2);
149     }
150     wal2.close();
151     runWALSplit(this.conf);
152 
153     HLog wal3 = createWAL(this.conf);
154     wal3.setSequenceNumber(wal2.getSequenceNumber());
155     try {
156       final HRegion region = new HRegion(basedir, wal3, this.fs, this.conf, hri,
157           null);
158       long seqid = region.initialize();
159       assertTrue(seqid > wal3.getSequenceNumber());
160 
161       // TODO: Scan all.
162       region.close();
163     } finally {
164       wal3.closeAndDelete();
165     }
166   }
167 
168   /**
169    * Test case of HRegion that is only made out of bulk loaded files.  Assert
170    * that we don't 'crash'.
171    * @throws IOException
172    * @throws IllegalAccessException 
173    * @throws NoSuchFieldException 
174    * @throws IllegalArgumentException 
175    * @throws SecurityException 
176    */
177   @Test
178   public void testRegionMadeOfBulkLoadedFilesOnly()
179   throws IOException, SecurityException, IllegalArgumentException,
180       NoSuchFieldException, IllegalAccessException {
181     final String tableNameStr = "testReplayEditsWrittenViaHRegion";
182     HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
183     Path basedir = new Path(this.hbaseRootDir, tableNameStr);
184     deleteDir(basedir);
185     HLog wal = createWAL(this.conf);
186     HRegion region = HRegion.openHRegion(hri, basedir, wal, this.conf);
187     Path f =  new Path(basedir, "hfile");
188     HFile.Writer writer = new HFile.Writer(this.fs, f);
189     byte [] family = hri.getTableDesc().getFamilies().iterator().next().getName();
190     byte [] row = Bytes.toBytes(tableNameStr);
191     writer.append(new KeyValue(row, family, family, row));
192     writer.close();
193     region.bulkLoadHFile(f.toString(), family);
194     // Add an edit so something in the WAL
195     region.put((new Put(row)).add(family, family, family));
196     wal.sync();
197 
198     // Now 'crash' the region by stealing its wal
199     Configuration newConf = HBaseTestingUtility.setDifferentUser(this.conf,
200         tableNameStr);
201     runWALSplit(newConf);
202     HLog wal2 = createWAL(newConf);
203     HRegion region2 = new HRegion(basedir, wal2, FileSystem.get(newConf),
204       newConf, hri, null);
205     long seqid2 = region2.initialize();
206     assertTrue(seqid2 > -1);
207 
208     // I can't close wal1.  Its been appropriated when we split.
209     region2.close();
210     wal2.closeAndDelete();
211   }
212 
213   /**
214    * Test writing edits into an HRegion, closing it, splitting logs, opening
215    * Region again.  Verify seqids.
216    * @throws IOException
217    * @throws IllegalAccessException 
218    * @throws NoSuchFieldException 
219    * @throws IllegalArgumentException 
220    * @throws SecurityException 
221    */
222   @Test
223   public void testReplayEditsWrittenViaHRegion()
224   throws IOException, SecurityException, IllegalArgumentException,
225       NoSuchFieldException, IllegalAccessException {
226     final String tableNameStr = "testReplayEditsWrittenViaHRegion";
227     HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
228     Path basedir = new Path(this.hbaseRootDir, tableNameStr);
229     deleteDir(basedir);
230     final byte[] rowName = Bytes.toBytes(tableNameStr);
231     final int countPerFamily = 10;
232 
233     // Write countPerFamily edits into the three families.  Do a flush on one
234     // of the families during the load of edits so its seqid is not same as
235     // others to test we do right thing when different seqids.
236     HLog wal = createWAL(this.conf);
237     HRegion region = new HRegion(basedir, wal, this.fs, this.conf, hri, null);
238     long seqid = region.initialize();
239     // HRegionServer usually does this. It knows the largest seqid across all regions.
240     wal.setSequenceNumber(seqid);
241     boolean first = true;
242     for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
243       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region, "x");
244       if (first ) {
245         // If first, so we have at least one family w/ different seqid to rest.
246         region.flushcache();
247         first = false;
248       }
249     }
250     // Now assert edits made it in.
251     Get g = new Get(rowName);
252     Result result = region.get(g, null);
253     assertEquals(countPerFamily * hri.getTableDesc().getFamilies().size(),
254       result.size());
255     // Now close the region, split the log, reopen the region and assert that
256     // replay of log has no effect, that our seqids are calculated correctly so
257     // all edits in logs are seen as 'stale'/old.
258     region.close();
259     wal.close();
260     runWALSplit(this.conf);
261     HLog wal2 = createWAL(this.conf);
262     HRegion region2 = new HRegion(basedir, wal2, this.fs, this.conf, hri, null) {
263       @Override
264       protected boolean restoreEdit(Store s, KeyValue kv) {
265         super.restoreEdit(s, kv);
266         throw new RuntimeException("Called when it should not have been!");
267       }
268     };
269     long seqid2 = region2.initialize();
270     // HRegionServer usually does this. It knows the largest seqid across all regions.
271     wal2.setSequenceNumber(seqid2);
272     assertTrue(seqid + result.size() < seqid2);
273 
274     // Next test.  Add more edits, then 'crash' this region by stealing its wal
275     // out from under it and assert that replay of the log adds the edits back
276     // correctly when region is opened again.
277     for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
278       addRegionEdits(rowName, hcd.getName(), countPerFamily, this.ee, region2, "y");
279     }
280     // Get count of edits.
281     Result result2 = region2.get(g, null);
282     assertEquals(2 * result.size(), result2.size());
283     wal2.sync();
284     // Set down maximum recovery so we dfsclient doesn't linger retrying something
285     // long gone.
286     HBaseTestingUtility.setMaxRecoveryErrorCount(wal2.getOutputStream(), 1);
287     Configuration newConf = HBaseTestingUtility.setDifferentUser(this.conf,
288       tableNameStr);
289     runWALSplit(newConf);
290     FileSystem newFS = FileSystem.get(newConf);
291     // Make a new wal for new region open.
292     HLog wal3 = createWAL(newConf);
293     final AtomicInteger countOfRestoredEdits = new AtomicInteger(0);
294     HRegion region3 = new HRegion(basedir, wal3, newFS, newConf, hri, null) {
295       @Override
296       protected boolean restoreEdit(Store s, KeyValue kv) {
297         boolean b = super.restoreEdit(s, kv);
298         countOfRestoredEdits.incrementAndGet();
299         return b;
300       }
301     };
302     long seqid3 = region3.initialize();
303     // HRegionServer usually does this. It knows the largest seqid across all regions.
304     wal3.setSequenceNumber(seqid3);
305     Result result3 = region3.get(g, null);
306     // Assert that count of cells is same as before crash.
307     assertEquals(result2.size(), result3.size());
308     assertEquals(hri.getTableDesc().getFamilies().size() * countPerFamily,
309       countOfRestoredEdits.get());
310 
311     // I can't close wal1.  Its been appropriated when we split.
312     region3.close();
313     wal3.closeAndDelete();
314   }
315 
316   /**
317    * Create an HRegion with the result of a HLog split and test we only see the
318    * good edits
319    * @throws Exception
320    */
321   @Test
322   public void testReplayEditsWrittenIntoWAL() throws Exception {
323     final String tableNameStr = "testReplayEditsWrittenIntoWAL";
324     HRegionInfo hri = createBasic3FamilyHRegionInfo(tableNameStr);
325     Path basedir = new Path(hbaseRootDir, tableNameStr);
326     deleteDir(basedir);
327     HLog wal = createWAL(this.conf);
328     final byte[] tableName = Bytes.toBytes(tableNameStr);
329     final byte[] rowName = tableName;
330     final byte[] regionName = hri.getRegionName();
331 
332     // Add 1k to each family.
333     final int countPerFamily = 1000;
334     for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
335       addWALEdits(tableName, hri, rowName, hcd.getName(), countPerFamily, ee, wal);
336     }
337 
338     // Add a cache flush, shouldn't have any effect
339     long logSeqId = wal.startCacheFlush();
340     wal.completeCacheFlush(regionName, tableName, logSeqId, hri.isMetaRegion());
341 
342     // Add an edit to another family, should be skipped.
343     WALEdit edit = new WALEdit();
344     long now = ee.currentTimeMillis();
345     edit.add(new KeyValue(rowName, Bytes.toBytes("another family"), rowName,
346       now, rowName));
347     wal.append(hri, tableName, edit, now);
348 
349     // Delete the c family to verify deletes make it over.
350     edit = new WALEdit();
351     now = ee.currentTimeMillis();
352     edit.add(new KeyValue(rowName, Bytes.toBytes("c"), null, now,
353       KeyValue.Type.DeleteFamily));
354     wal.append(hri, tableName, edit, now);
355 
356     // Sync.
357     wal.sync();
358     // Set down maximum recovery so we dfsclient doesn't linger retrying something
359     // long gone.
360     HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
361 
362     // Make a new conf and a new fs for the splitter to run on so we can take
363     // over old wal.
364     Configuration newConf = HBaseTestingUtility.setDifferentUser(this.conf,
365       ".replay.wal.secondtime");
366     runWALSplit(newConf);
367     FileSystem newFS = FileSystem.get(newConf);
368     // 100k seems to make for about 4 flushes during HRegion#initialize.
369     newConf.setInt("hbase.hregion.memstore.flush.size", 1024 * 100);
370     // Make a new wal for new region.
371     HLog newWal = createWAL(newConf);
372     final AtomicInteger flushcount = new AtomicInteger(0);
373     try {
374       final HRegion region = new HRegion(basedir, newWal, newFS, newConf, hri,
375           null) {
376         protected boolean internalFlushcache(HLog wal, long myseqid)
377         throws IOException {
378           boolean b = super.internalFlushcache(wal, myseqid);
379           flushcount.incrementAndGet();
380           return b;
381         };
382       };
383       long seqid = region.initialize();
384       // We flushed during init.
385       assertTrue(flushcount.get() > 0);
386       assertTrue(seqid > wal.getSequenceNumber());
387 
388       Get get = new Get(rowName);
389       Result result = region.get(get, -1);
390       // Make sure we only see the good edits
391       assertEquals(countPerFamily * (hri.getTableDesc().getFamilies().size() - 1),
392         result.size());
393       region.close();
394     } finally {
395       newWal.closeAndDelete();
396     }
397   }
398 
399   private void addWALEdits (final byte [] tableName, final HRegionInfo hri,
400       final byte [] rowName, final byte [] family, 
401       final int count, EnvironmentEdge ee, final HLog wal)
402   throws IOException {
403     String familyStr = Bytes.toString(family);
404     for (int j = 0; j < count; j++) {
405       byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
406       byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
407       WALEdit edit = new WALEdit();
408       edit.add(new KeyValue(rowName, family, qualifierBytes,
409         ee.currentTimeMillis(), columnBytes));
410       wal.append(hri, tableName, edit, ee.currentTimeMillis());
411     }
412   }
413 
414   private void addRegionEdits (final byte [] rowName, final byte [] family, 
415       final int count, EnvironmentEdge ee, final HRegion r,
416       final String qualifierPrefix)
417   throws IOException {
418     for (int j = 0; j < count; j++) {
419       byte[] qualifier = Bytes.toBytes(qualifierPrefix + Integer.toString(j));
420       Put p = new Put(rowName);
421       p.add(family, qualifier, ee.currentTimeMillis(), rowName);
422       r.put(p);
423     }
424   }
425 
426   /*
427    * Creates an HRI around an HTD that has <code>tableName</code> and three
428    * column families named 'a','b', and 'c'.
429    * @param tableName Name of table to use when we create HTableDescriptor.
430    */
431   private HRegionInfo createBasic3FamilyHRegionInfo(final String tableName) {
432     HTableDescriptor htd = new HTableDescriptor(tableName);
433     HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
434     htd.addFamily(a);
435     HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
436     htd.addFamily(b);
437     HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
438     htd.addFamily(c);
439     return new HRegionInfo(htd, null, null, false);
440   }
441 
442 
443   /*
444    * Run the split.  Verify only single split file made.
445    * @param c
446    * @return The single split file made
447    * @throws IOException
448    */
449   private Path runWALSplit(final Configuration c) throws IOException {
450     FileSystem fs = FileSystem.get(c);
451     List<Path> splits = HLog.splitLog(this.hbaseRootDir, this.logDir,
452       this.oldLogDir, fs, c);
453     // Split should generate only 1 file since there's only 1 region
454     assertEquals(1, splits.size());
455     // Make sure the file exists
456     assertTrue(fs.exists(splits.get(0)));
457     LOG.info("Split file=" + splits.get(0));
458     return splits.get(0);
459   }
460 
461   /*
462    * @param c
463    * @return WAL with retries set down from 5 to 1 only.
464    * @throws IOException
465    */
466   private HLog createWAL(final Configuration c) throws IOException {
467     HLog wal = new HLog(FileSystem.get(c), logDir, oldLogDir, c, null);
468     // Set down maximum recovery so we dfsclient doesn't linger retrying something
469     // long gone.
470     HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
471     return wal;
472   }
473 }