1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23
24 import org.apache.commons.logging.Log;
25 import org.apache.commons.logging.LogFactory;
26 import org.apache.hadoop.conf.Configuration;
27 import org.apache.hadoop.fs.FileSystem;
28 import org.apache.hadoop.fs.Path;
29 import org.apache.hadoop.hbase.HBaseTestingUtility;
30 import org.apache.hadoop.hbase.TableName;
31 import org.apache.hadoop.hbase.client.Scan;
32 import org.apache.hadoop.hbase.client.Result;
33 import org.apache.hadoop.hbase.client.ResultScanner;
34 import org.apache.hadoop.hbase.client.Durability;
35 import org.apache.hadoop.hbase.client.Put;
36 import org.apache.hadoop.hbase.client.HTable;
37 import org.apache.hadoop.hbase.io.HFileLink;
38 import org.apache.hadoop.hbase.testclassification.LargeTests;
39 import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
40 import org.apache.hadoop.hbase.util.Bytes;
41 import org.apache.hadoop.hbase.util.FSUtils;
42 import org.apache.hadoop.hbase.util.FSVisitor;
43 import org.apache.hadoop.hbase.util.TestTableName;
44
45 import org.junit.After;
46 import org.junit.Before;
47 import org.junit.Ignore;
48 import org.junit.Rule;
49 import org.junit.Test;
50 import org.junit.experimental.categories.Category;
51
52 import static org.junit.Assert.assertEquals;
53 import static org.junit.Assert.assertFalse;
54 import static org.junit.Assert.assertTrue;
55 import static org.junit.Assert.fail;
56
57 @Category(LargeTests.class)
58 @Ignore("See HBASE-13744")
59 public class TestCorruptedRegionStoreFile {
60 private static final Log LOG = LogFactory.getLog(TestCorruptedRegionStoreFile.class);
61
62 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
63
64 private static final String FAMILY_NAME_STR = "f";
65 private static final byte[] FAMILY_NAME = Bytes.toBytes(FAMILY_NAME_STR);
66
67 private static final int NUM_FILES = 25;
68 private static final int ROW_PER_FILE = 2000;
69 private static final int NUM_ROWS = NUM_FILES * ROW_PER_FILE;
70
71 @Rule public TestTableName TEST_TABLE = new TestTableName();
72
73 private final ArrayList<Path> storeFiles = new ArrayList<Path>();
74 private Path tableDir;
75 private int rowCount;
76
77 private static void setupConf(Configuration conf) {
78 conf.setLong("hbase.hstore.compaction.min", 20);
79 conf.setLong("hbase.hstore.compaction.max", 39);
80 conf.setLong("hbase.hstore.blockingStoreFiles", 40);
81 }
82
83 private void setupTable(final TableName tableName) throws Exception {
84
85 String tableNameStr = tableName.getNameAsString();
86 HTable table = UTIL.createTable(tableName, FAMILY_NAME);
87 try {
88 rowCount = 0;
89 byte[] value = new byte[1024];
90 byte[] q = Bytes.toBytes("q");
91 while (rowCount < NUM_ROWS) {
92 Put put = new Put(Bytes.toBytes(String.format("%010d", rowCount)));
93 put.setDurability(Durability.SKIP_WAL);
94 put.add(FAMILY_NAME, q, value);
95 table.put(put);
96
97 if ((rowCount++ % ROW_PER_FILE) == 0) {
98
99 UTIL.getHBaseAdmin().flush(tableNameStr);
100 }
101 }
102 } finally {
103 UTIL.getHBaseAdmin().flush(tableNameStr);
104 table.close();
105 }
106
107 assertEquals(NUM_ROWS, rowCount);
108
109
110 storeFiles.clear();
111 tableDir = FSUtils.getTableDir(getRootDir(), tableName);
112 FSVisitor.visitTableStoreFiles(getFileSystem(), tableDir, new FSVisitor.StoreFileVisitor() {
113 @Override
114 public void storeFile(final String region, final String family, final String hfile)
115 throws IOException {
116 HFileLink link = HFileLink.create(UTIL.getConfiguration(), tableName, region, family, hfile);
117 storeFiles.add(link.getOriginPath());
118 }
119 });
120 assertTrue("expected at least 1 store file", storeFiles.size() > 0);
121 LOG.info("store-files: " + storeFiles);
122 }
123
124 @Before
125 public void setup() throws Exception {
126 setupConf(UTIL.getConfiguration());
127 UTIL.startMiniCluster(2, 3);
128
129 setupTable(TEST_TABLE.getTableName());
130 }
131
132 @After
133 public void tearDown() throws Exception {
134 try {
135 UTIL.shutdownMiniCluster();
136 } catch (Exception e) {
137 LOG.warn("failure shutting down cluster", e);
138 }
139 }
140
141 @Test(timeout=180000)
142 public void testLosingFileDuringScan() throws Exception {
143 assertEquals(rowCount, fullScanAndCount(TEST_TABLE.getTableName()));
144
145 final FileSystem fs = getFileSystem();
146 final Path tmpStoreFilePath = new Path(UTIL.getDataTestDir(), "corruptedHFile");
147
148
149 int count = fullScanAndCount(TEST_TABLE.getTableName(), new ScanInjector() {
150 private boolean hasFile = true;
151
152 @Override
153 public void beforeScanNext(HTable table) throws Exception {
154
155 if (hasFile) {
156 fs.copyToLocalFile(true, storeFiles.get(0), tmpStoreFilePath);
157 LOG.info("Move file to local");
158 evictHFileCache(storeFiles.get(0));
159 hasFile = false;
160 }
161 }
162 });
163 assertTrue("expected one file lost: rowCount=" + count + " lostRows=" + (NUM_ROWS - count),
164 count >= (NUM_ROWS - ROW_PER_FILE));
165 }
166
167 @Test(timeout=180000)
168 public void testLosingFileAfterScannerInit() throws Exception {
169 assertEquals(rowCount, fullScanAndCount(TEST_TABLE.getTableName()));
170
171 final FileSystem fs = getFileSystem();
172 final Path tmpStoreFilePath = new Path(UTIL.getDataTestDir(), "corruptedHFile");
173
174
175 int count = fullScanAndCount(TEST_TABLE.getTableName(), new ScanInjector() {
176 private boolean hasFile = true;
177
178 @Override
179 public void beforeScan(HTable table, Scan scan) throws Exception {
180
181 if (hasFile) {
182 fs.copyToLocalFile(true, storeFiles.get(0), tmpStoreFilePath);
183 LOG.info("Move file to local");
184 evictHFileCache(storeFiles.get(0));
185 hasFile = false;
186 }
187 }
188 });
189 assertTrue("expected one file lost: rowCount=" + count + " lostRows=" + (NUM_ROWS - count),
190 count >= (NUM_ROWS - ROW_PER_FILE));
191 }
192
193
194
195
196 private FileSystem getFileSystem() {
197 return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
198 }
199
200 private Path getRootDir() {
201 return UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
202 }
203
204 private void evictHFileCache(final Path hfile) throws Exception {
205 for (RegionServerThread rst: UTIL.getMiniHBaseCluster().getRegionServerThreads()) {
206 HRegionServer rs = rst.getRegionServer();
207 rs.getCacheConfig().getBlockCache().evictBlocksByHfileName(hfile.getName());
208 }
209 Thread.sleep(6000);
210 }
211
212 private int fullScanAndCount(final TableName tableName) throws Exception {
213 return fullScanAndCount(tableName, new ScanInjector());
214 }
215
216 private int fullScanAndCount(final TableName tableName, final ScanInjector injector)
217 throws Exception {
218 HTable table = new HTable(UTIL.getConfiguration(), tableName);
219 int count = 0;
220 try {
221 Scan scan = new Scan();
222 scan.setCaching(1);
223 scan.setCacheBlocks(false);
224 injector.beforeScan(table, scan);
225 ResultScanner scanner = table.getScanner(scan);
226 try {
227 while (true) {
228 injector.beforeScanNext(table);
229 Result result = scanner.next();
230 injector.afterScanNext(table, result);
231 if (result == null) break;
232 if ((count++ % (ROW_PER_FILE / 2)) == 0) {
233 LOG.debug("scan next " + count);
234 }
235 }
236 } finally {
237 scanner.close();
238 injector.afterScan(table);
239 }
240 } finally {
241 table.close();
242 }
243 return count;
244 }
245
246 private class ScanInjector {
247 protected void beforeScan(HTable table, Scan scan) throws Exception {}
248 protected void beforeScanNext(HTable table) throws Exception {}
249 protected void afterScanNext(HTable table, Result result) throws Exception {}
250 protected void afterScan(HTable table) throws Exception {}
251 }
252 }