1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.regionserver;
20
21 import static org.junit.Assert.assertNotNull;
22 import static org.junit.Assert.assertTrue;
23 import static org.junit.Assert.fail;
24
25 import java.io.IOException;
26 import java.io.InputStream;
27 import java.lang.ref.SoftReference;
28 import java.util.ArrayList;
29 import java.util.Collections;
30 import java.util.List;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FSDataInputStream;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.FilterFileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.fs.PositionedReadable;
40 import org.apache.hadoop.hbase.*;
41 import org.apache.hadoop.hbase.client.HBaseAdmin;
42 import org.apache.hadoop.hbase.client.HTable;
43 import org.apache.hadoop.hbase.fs.HFileSystem;
44 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
45 import org.apache.hadoop.hbase.io.hfile.HFileScanner;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.junit.Assume;
48 import org.junit.Test;
49 import org.junit.experimental.categories.Category;
50
51
52
53
54
55 @Category(MediumTests.class)
56 public class TestFSErrorsExposed {
57 private static final Log LOG = LogFactory.getLog(TestFSErrorsExposed.class);
58
59 HBaseTestingUtility util = new HBaseTestingUtility();
60
61
62
63
64
65 @Test
66 public void testHFileScannerThrowsErrors() throws IOException {
67 Path hfilePath = new Path(new Path(
68 util.getDataTestDir("internalScannerExposesErrors"),
69 "regionname"), "familyname");
70 HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
71 FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
72 FileSystem fs = new HFileSystem(faultyfs);
73 CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
74 StoreFile.Writer writer = new StoreFile.WriterBuilder(
75 util.getConfiguration(), cacheConf, hfs, 2*1024)
76 .withOutputDir(hfilePath)
77 .build();
78 TestStoreFile.writeStoreFile(
79 writer, Bytes.toBytes("cf"), Bytes.toBytes("qual"));
80
81 StoreFile sf = new StoreFile(fs, writer.getPath(),
82 util.getConfiguration(), cacheConf, BloomType.NONE);
83
84 StoreFile.Reader reader = sf.createReader();
85 HFileScanner scanner = reader.getScanner(false, true);
86
87 FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
88 assertNotNull(inStream);
89
90 scanner.seekTo();
91
92 assertTrue(scanner.next());
93
94 faultyfs.startFaults();
95
96 try {
97 int scanned=0;
98 while (scanner.next()) {
99 scanned++;
100 }
101 fail("Scanner didn't throw after faults injected");
102 } catch (IOException ioe) {
103 LOG.info("Got expected exception", ioe);
104 assertTrue(ioe.getMessage().contains("Fault"));
105 }
106 reader.close(true);
107 }
108
109
110
111
112
113 @Test
114 public void testStoreFileScannerThrowsErrors() throws IOException {
115 Path hfilePath = new Path(new Path(
116 util.getDataTestDir("internalScannerExposesErrors"),
117 "regionname"), "familyname");
118 HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
119 FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
120 HFileSystem fs = new HFileSystem(faultyfs);
121 CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
122 StoreFile.Writer writer = new StoreFile.WriterBuilder(
123 util.getConfiguration(), cacheConf, hfs, 2 * 1024)
124 .withOutputDir(hfilePath)
125 .build();
126 TestStoreFile.writeStoreFile(
127 writer, Bytes.toBytes("cf"), Bytes.toBytes("qual"));
128
129 StoreFile sf = new StoreFile(fs, writer.getPath(), util.getConfiguration(),
130 cacheConf, BloomType.NONE);
131
132 List<StoreFileScanner> scanners = StoreFileScanner.getScannersForStoreFiles(
133 Collections.singletonList(sf), false, true, false);
134 KeyValueScanner scanner = scanners.get(0);
135
136 FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
137 assertNotNull(inStream);
138
139 scanner.seek(KeyValue.LOWESTKEY);
140
141 assertNotNull(scanner.next());
142 faultyfs.startFaults();
143
144 try {
145 int scanned=0;
146 while (scanner.next() != null) {
147 scanned++;
148 }
149 fail("Scanner didn't throw after faults injected");
150 } catch (IOException ioe) {
151 LOG.info("Got expected exception", ioe);
152 assertTrue(ioe.getMessage().contains("Could not iterate"));
153 }
154 scanner.close();
155 }
156
157
158
159
160
161
162 @Test(timeout=5 * 60 * 1000)
163 public void testFullSystemBubblesFSErrors() throws Exception {
164
165
166 Assume.assumeTrue(!util.isReadShortCircuitOn());
167
168 try {
169
170
171 util.getConfiguration().setInt(
172 "hbase.regionserver.optionallogflushinterval", Integer.MAX_VALUE);
173
174 util.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
175
176 util.startMiniCluster(1);
177 byte[] tableName = Bytes.toBytes("table");
178 byte[] fam = Bytes.toBytes("fam");
179
180 HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
181 HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
182 desc.addFamily(new HColumnDescriptor(fam)
183 .setMaxVersions(1)
184 .setBlockCacheEnabled(false)
185 );
186 admin.createTable(desc);
187
188 util.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
189
190
191 HTable table = new HTable(new Configuration(util.getConfiguration()), tableName);
192
193
194 util.loadTable(table, fam);
195 table.flushCommits();
196 util.flush();
197 util.countRows(table);
198
199
200 util.getDFSCluster().shutdownDataNodes();
201
202 try {
203 util.countRows(table);
204 fail("Did not fail to count after removing data");
205 } catch (Exception e) {
206 LOG.info("Got expected error", e);
207 assertTrue(e.getMessage().contains("Could not seek"));
208 }
209
210
211 util.getDFSCluster().restartDataNodes();
212
213 } finally {
214 util.getMiniHBaseCluster().killAll();
215 util.shutdownMiniCluster();
216 }
217 }
218
219 static class FaultyFileSystem extends FilterFileSystem {
220 List<SoftReference<FaultyInputStream>> inStreams =
221 new ArrayList<SoftReference<FaultyInputStream>>();
222
223 public FaultyFileSystem(FileSystem testFileSystem) {
224 super(testFileSystem);
225 }
226
227 @Override
228 public FSDataInputStream open(Path p, int bufferSize) throws IOException {
229 FSDataInputStream orig = fs.open(p, bufferSize);
230 FaultyInputStream faulty = new FaultyInputStream(orig);
231 inStreams.add(new SoftReference<FaultyInputStream>(faulty));
232 return faulty;
233 }
234
235
236
237
238 public void startFaults() {
239 for (SoftReference<FaultyInputStream> is: inStreams) {
240 is.get().startFaults();
241 }
242 }
243 }
244
245 static class FaultyInputStream extends FSDataInputStream {
246 boolean faultsStarted = false;
247
248 public FaultyInputStream(InputStream in) throws IOException {
249 super(in);
250 }
251
252 public void startFaults() {
253 faultsStarted = true;
254 }
255
256 public int read(long position, byte[] buffer, int offset, int length)
257 throws IOException {
258 injectFault();
259 return ((PositionedReadable)in).read(position, buffer, offset, length);
260 }
261
262 private void injectFault() throws IOException {
263 if (faultsStarted) {
264 throw new IOException("Fault injected");
265 }
266 }
267 }
268
269
270
271 }