View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.regionserver;
20  
21  import static org.junit.Assert.assertNotNull;
22  import static org.junit.Assert.assertTrue;
23  import static org.junit.Assert.fail;
24  
25  import java.io.IOException;
26  import java.io.InputStream;
27  import java.lang.ref.SoftReference;
28  import java.util.ArrayList;
29  import java.util.Collections;
30  import java.util.List;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FSDataInputStream;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.FilterFileSystem;
38  import org.apache.hadoop.fs.Path;
39  import org.apache.hadoop.fs.PositionedReadable;
40  import org.apache.hadoop.hbase.*;
41  import org.apache.hadoop.hbase.client.HBaseAdmin;
42  import org.apache.hadoop.hbase.client.HTable;
43  import org.apache.hadoop.hbase.fs.HFileSystem;
44  import org.apache.hadoop.hbase.io.hfile.CacheConfig;
45  import org.apache.hadoop.hbase.io.hfile.HFileScanner;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.junit.Assume;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  /**
52   * Test cases that ensure that file system level errors are bubbled up
53   * appropriately to clients, rather than swallowed.
54   */
55  @Category(MediumTests.class)
56  public class TestFSErrorsExposed {
57    private static final Log LOG = LogFactory.getLog(TestFSErrorsExposed.class);
58  
59    HBaseTestingUtility util = new HBaseTestingUtility();
60  
61    /**
62     * Injects errors into the pread calls of an on-disk file, and makes
63     * sure those bubble up to the HFile scanner
64     */
65    @Test
66    public void testHFileScannerThrowsErrors() throws IOException {
67      Path hfilePath = new Path(new Path(
68          util.getDataTestDir("internalScannerExposesErrors"),
69          "regionname"), "familyname");
70      HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
71      FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
72      FileSystem fs = new HFileSystem(faultyfs);
73      CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
74      StoreFile.Writer writer = new StoreFile.WriterBuilder(
75          util.getConfiguration(), cacheConf, hfs, 2*1024)
76              .withOutputDir(hfilePath)
77              .build();
78      TestStoreFile.writeStoreFile(
79          writer, Bytes.toBytes("cf"), Bytes.toBytes("qual"));
80  
81      StoreFile sf = new StoreFile(fs, writer.getPath(),
82        util.getConfiguration(), cacheConf, BloomType.NONE);
83  
84      StoreFile.Reader reader = sf.createReader();
85      HFileScanner scanner = reader.getScanner(false, true);
86  
87      FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
88      assertNotNull(inStream);
89  
90      scanner.seekTo();
91      // Do at least one successful read
92      assertTrue(scanner.next());
93  
94      faultyfs.startFaults();
95  
96      try {
97        int scanned=0;
98        while (scanner.next()) {
99          scanned++;
100       }
101       fail("Scanner didn't throw after faults injected");
102     } catch (IOException ioe) {
103       LOG.info("Got expected exception", ioe);
104       assertTrue(ioe.getMessage().contains("Fault"));
105     }
106     reader.close(true); // end of test so evictOnClose
107   }
108 
109   /**
110    * Injects errors into the pread calls of an on-disk file, and makes
111    * sure those bubble up to the StoreFileScanner
112    */
113   @Test
114   public void testStoreFileScannerThrowsErrors() throws IOException {
115     Path hfilePath = new Path(new Path(
116         util.getDataTestDir("internalScannerExposesErrors"),
117         "regionname"), "familyname");
118     HFileSystem hfs = (HFileSystem)util.getTestFileSystem();
119     FaultyFileSystem faultyfs = new FaultyFileSystem(hfs.getBackingFs());
120     HFileSystem fs = new HFileSystem(faultyfs);
121     CacheConfig cacheConf = new CacheConfig(util.getConfiguration());
122     StoreFile.Writer writer = new StoreFile.WriterBuilder(
123         util.getConfiguration(), cacheConf, hfs, 2 * 1024)
124             .withOutputDir(hfilePath)
125             .build();
126     TestStoreFile.writeStoreFile(
127         writer, Bytes.toBytes("cf"), Bytes.toBytes("qual"));
128 
129     StoreFile sf = new StoreFile(fs, writer.getPath(), util.getConfiguration(),
130       cacheConf, BloomType.NONE);
131 
132     List<StoreFileScanner> scanners = StoreFileScanner.getScannersForStoreFiles(
133         Collections.singletonList(sf), false, true, false);
134     KeyValueScanner scanner = scanners.get(0);
135 
136     FaultyInputStream inStream = faultyfs.inStreams.get(0).get();
137     assertNotNull(inStream);
138 
139     scanner.seek(KeyValue.LOWESTKEY);
140     // Do at least one successful read
141     assertNotNull(scanner.next());
142     faultyfs.startFaults();
143 
144     try {
145       int scanned=0;
146       while (scanner.next() != null) {
147         scanned++;
148       }
149       fail("Scanner didn't throw after faults injected");
150     } catch (IOException ioe) {
151       LOG.info("Got expected exception", ioe);
152       assertTrue(ioe.getMessage().contains("Could not iterate"));
153     }
154     scanner.close();
155   }
156 
157   /**
158    * Cluster test which starts a region server with a region, then
159    * removes the data from HDFS underneath it, and ensures that
160    * errors are bubbled to the client.
161    */
162   @Test(timeout=5 * 60 * 1000)
163   public void testFullSystemBubblesFSErrors() throws Exception {
164     // We won't have an error if the datanode is not there if we use short circuit
165     //  it's a known 'feature'.
166     Assume.assumeTrue(!util.isReadShortCircuitOn());
167 
168     try {
169       // We set it not to run or it will trigger server shutdown while sync'ing
170       // because all the datanodes are bad
171       util.getConfiguration().setInt(
172           "hbase.regionserver.optionallogflushinterval", Integer.MAX_VALUE);
173 
174       util.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
175 
176       util.startMiniCluster(1);
177       byte[] tableName = Bytes.toBytes("table");
178       byte[] fam = Bytes.toBytes("fam");
179 
180       HBaseAdmin admin = new HBaseAdmin(util.getConfiguration());
181       HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
182       desc.addFamily(new HColumnDescriptor(fam)
183           .setMaxVersions(1)
184           .setBlockCacheEnabled(false)
185       );
186       admin.createTable(desc);
187       // Make it fail faster.
188       util.getConfiguration().setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
189       // Make a new Configuration so it makes a new connection that has the
190       // above configuration on it; else we use the old one w/ 10 as default.
191       HTable table = new HTable(new Configuration(util.getConfiguration()), tableName);
192 
193       // Load some data
194       util.loadTable(table, fam);
195       table.flushCommits();
196       util.flush();
197       util.countRows(table);
198 
199       // Kill the DFS cluster
200       util.getDFSCluster().shutdownDataNodes();
201 
202       try {
203         util.countRows(table);
204         fail("Did not fail to count after removing data");
205       } catch (Exception e) {
206         LOG.info("Got expected error", e);
207         assertTrue(e.getMessage().contains("Could not seek"));
208       }
209 
210       // Restart data nodes so that HBase can shut down cleanly.
211       util.getDFSCluster().restartDataNodes();
212 
213     } finally {
214       util.getMiniHBaseCluster().killAll();
215       util.shutdownMiniCluster();
216     }
217   }
218 
219   static class FaultyFileSystem extends FilterFileSystem {
220     List<SoftReference<FaultyInputStream>> inStreams =
221       new ArrayList<SoftReference<FaultyInputStream>>();
222 
223     public FaultyFileSystem(FileSystem testFileSystem) {
224       super(testFileSystem);
225     }
226 
227     @Override
228     public FSDataInputStream open(Path p, int bufferSize) throws IOException  {
229       FSDataInputStream orig = fs.open(p, bufferSize);
230       FaultyInputStream faulty = new FaultyInputStream(orig);
231       inStreams.add(new SoftReference<FaultyInputStream>(faulty));
232       return faulty;
233     }
234 
235     /**
236      * Starts to simulate faults on all streams opened so far
237      */
238     public void startFaults() {
239       for (SoftReference<FaultyInputStream> is: inStreams) {
240         is.get().startFaults();
241       }
242     }
243   }
244 
245   static class FaultyInputStream extends FSDataInputStream {
246     boolean faultsStarted = false;
247 
248     public FaultyInputStream(InputStream in) throws IOException {
249       super(in);
250     }
251 
252     public void startFaults() {
253       faultsStarted = true;
254     }
255 
256     public int read(long position, byte[] buffer, int offset, int length)
257       throws IOException {
258       injectFault();
259       return ((PositionedReadable)in).read(position, buffer, offset, length);
260     }
261 
262     private void injectFault() throws IOException {
263       if (faultsStarted) {
264         throw new IOException("Fault injected");
265       }
266     }
267   }
268 
269 
270 
271 }