View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.fs.FileSystem;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.HBaseTestingUtility;
30  import org.apache.hadoop.hbase.TableName;
31  import org.apache.hadoop.hbase.client.Scan;
32  import org.apache.hadoop.hbase.client.Result;
33  import org.apache.hadoop.hbase.client.ResultScanner;
34  import org.apache.hadoop.hbase.client.Durability;
35  import org.apache.hadoop.hbase.client.Put;
36  import org.apache.hadoop.hbase.client.HTable;
37  import org.apache.hadoop.hbase.io.HFileLink;
38  import org.apache.hadoop.hbase.testclassification.LargeTests;
39  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.apache.hadoop.hbase.util.FSUtils;
42  import org.apache.hadoop.hbase.util.FSVisitor;
43  import org.apache.hadoop.hbase.util.TestTableName;
44  
45  import org.junit.After;
46  import org.junit.Before;
47  import org.junit.Ignore;
48  import org.junit.Rule;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  import static org.junit.Assert.assertEquals;
53  import static org.junit.Assert.assertFalse;
54  import static org.junit.Assert.assertTrue;
55  import static org.junit.Assert.fail;
56  
57  @Category(LargeTests.class)
58  @Ignore("See HBASE-13744")
59  public class TestCorruptedRegionStoreFile {
60    private static final Log LOG = LogFactory.getLog(TestCorruptedRegionStoreFile.class);
61  
62    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
63  
64    private static final String FAMILY_NAME_STR = "f";
65    private static final byte[] FAMILY_NAME = Bytes.toBytes(FAMILY_NAME_STR);
66  
67    private static final int NUM_FILES = 25;
68    private static final int ROW_PER_FILE = 2000;
69    private static final int NUM_ROWS = NUM_FILES * ROW_PER_FILE;
70  
71    @Rule public TestTableName TEST_TABLE = new TestTableName();
72  
73    private final ArrayList<Path> storeFiles = new ArrayList<Path>();
74    private Path tableDir;
75    private int rowCount;
76  
77    private static void setupConf(Configuration conf) {
78      conf.setLong("hbase.hstore.compaction.min", 20);
79      conf.setLong("hbase.hstore.compaction.max", 39);
80      conf.setLong("hbase.hstore.blockingStoreFiles", 40);
81    }
82  
83    private void setupTable(final TableName tableName) throws Exception {
84      // load the table
85      String tableNameStr = tableName.getNameAsString();
86      HTable table = UTIL.createTable(tableName, FAMILY_NAME);
87      try {
88        rowCount = 0;
89        byte[] value = new byte[1024];
90        byte[] q = Bytes.toBytes("q");
91        while (rowCount < NUM_ROWS) {
92          Put put = new Put(Bytes.toBytes(String.format("%010d", rowCount)));
93          put.setDurability(Durability.SKIP_WAL);
94          put.add(FAMILY_NAME, q, value);
95          table.put(put);
96  
97          if ((rowCount++ % ROW_PER_FILE) == 0) {
98            // flush it
99            UTIL.getHBaseAdmin().flush(tableNameStr);
100         }
101       }
102     } finally {
103       UTIL.getHBaseAdmin().flush(tableNameStr);
104       table.close();
105     }
106 
107     assertEquals(NUM_ROWS, rowCount);
108 
109     // get the store file paths
110     storeFiles.clear();
111     tableDir = FSUtils.getTableDir(getRootDir(), tableName);
112     FSVisitor.visitTableStoreFiles(getFileSystem(), tableDir, new FSVisitor.StoreFileVisitor() {
113       @Override
114       public void storeFile(final String region, final String family, final String hfile)
115           throws IOException {
116         HFileLink link = HFileLink.create(UTIL.getConfiguration(), tableName, region, family, hfile);
117         storeFiles.add(link.getOriginPath());
118       }
119     });
120     assertTrue("expected at least 1 store file", storeFiles.size() > 0);
121     LOG.info("store-files: " + storeFiles);
122   }
123 
124   @Before
125   public void setup() throws Exception {
126     setupConf(UTIL.getConfiguration());
127     UTIL.startMiniCluster(2, 3);
128 
129     setupTable(TEST_TABLE.getTableName());
130   }
131 
132   @After
133   public void tearDown() throws Exception {
134     try {
135       UTIL.shutdownMiniCluster();
136     } catch (Exception e) {
137       LOG.warn("failure shutting down cluster", e);
138     }
139   }
140 
141   @Test(timeout=180000)
142   public void testLosingFileDuringScan() throws Exception {
143     assertEquals(rowCount, fullScanAndCount(TEST_TABLE.getTableName()));
144 
145     final FileSystem fs = getFileSystem();
146     final Path tmpStoreFilePath = new Path(UTIL.getDataTestDir(), "corruptedHFile");
147 
148     // try to query with the missing file
149     int count = fullScanAndCount(TEST_TABLE.getTableName(), new ScanInjector() {
150       private boolean hasFile = true;
151 
152       @Override
153       public void beforeScanNext(HTable table) throws Exception {
154         // move the path away (now the region is corrupted)
155         if (hasFile) {
156           fs.copyToLocalFile(true, storeFiles.get(0), tmpStoreFilePath);
157           LOG.info("Move file to local");
158           evictHFileCache(storeFiles.get(0));
159           hasFile = false;
160         }
161       }
162     });
163     assertTrue("expected one file lost: rowCount=" + count + " lostRows=" + (NUM_ROWS - count),
164                count >= (NUM_ROWS - ROW_PER_FILE));
165   }
166 
167   @Test(timeout=180000)
168   public void testLosingFileAfterScannerInit() throws Exception {
169     assertEquals(rowCount, fullScanAndCount(TEST_TABLE.getTableName()));
170 
171     final FileSystem fs = getFileSystem();
172     final Path tmpStoreFilePath = new Path(UTIL.getDataTestDir(), "corruptedHFile");
173 
174     // try to query with the missing file
175     int count = fullScanAndCount(TEST_TABLE.getTableName(), new ScanInjector() {
176       private boolean hasFile = true;
177 
178       @Override
179       public void beforeScan(HTable table, Scan scan) throws Exception {
180         // move the path away (now the region is corrupted)
181         if (hasFile) {
182           fs.copyToLocalFile(true, storeFiles.get(0), tmpStoreFilePath);
183           LOG.info("Move file to local");
184           evictHFileCache(storeFiles.get(0));
185           hasFile = false;
186         }
187       }
188     });
189     assertTrue("expected one file lost: rowCount=" + count + " lostRows=" + (NUM_ROWS - count),
190                count >= (NUM_ROWS - ROW_PER_FILE));
191   }
192 
193   // ==========================================================================
194   //  Helpers
195   // ==========================================================================
196   private FileSystem getFileSystem() {
197     return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
198   }
199 
200   private Path getRootDir() {
201     return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
202   }
203 
204   private void evictHFileCache(final Path hfile) throws Exception {
205     for (RegionServerThread rst: UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
206       HRegionServer rs = rst.getRegionServer();
207       rs.getCacheConfig().getBlockCache().evictBlocksByHfileName(hfile.getName());
208     }
209     Thread.sleep(6000);
210   }
211 
212   private int fullScanAndCount(final TableName tableName) throws Exception {
213     return fullScanAndCount(tableName, new ScanInjector());
214   }
215 
216   private int fullScanAndCount(final TableName tableName, final ScanInjector injector)
217       throws Exception {
218     HTable table = new HTable(UTIL.getConfiguration(), tableName);
219     int count = 0;
220     try {
221       Scan scan = new Scan();
222       scan.setCaching(1);
223       scan.setCacheBlocks(false);
224       injector.beforeScan(table, scan);
225       ResultScanner scanner = table.getScanner(scan);
226       try {
227         while (true) {
228           injector.beforeScanNext(table);
229           Result result = scanner.next();
230           injector.afterScanNext(table, result);
231           if (result == null) break;
232           if ((count++ % (ROW_PER_FILE / 2)) == 0) {
233             LOG.debug("scan next " + count);
234           }
235         }
236       } finally {
237         scanner.close();
238         injector.afterScan(table);
239       }
240     } finally {
241       table.close();
242     }
243     return count;
244   }
245 
246   private class ScanInjector {
247     protected void beforeScan(HTable table, Scan scan) throws Exception {}
248     protected void beforeScanNext(HTable table) throws Exception {}
249     protected void afterScanNext(HTable table, Result result) throws Exception {}
250     protected void afterScan(HTable table) throws Exception {}
251   }
252 }