View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.io.hfile;
20  
21  import java.io.ByteArrayInputStream;
22  import java.io.ByteArrayOutputStream;
23  import java.io.DataInputStream;
24  import java.io.DataOutputStream;
25  import java.io.IOException;
26  import java.util.ArrayList;
27  import java.util.Collection;
28  import java.util.List;
29  
30  import org.apache.hadoop.hbase.*;
31  import org.junit.Before;
32  import org.junit.Test;
33  import org.junit.experimental.categories.Category;
34  import org.junit.runner.RunWith;
35  import org.junit.runners.Parameterized;
36  import org.junit.runners.Parameterized.Parameters;
37  
38  import static org.junit.Assert.*;
39  
40  import org.apache.commons.logging.Log;
41  import org.apache.commons.logging.LogFactory;
42  import org.apache.hadoop.fs.FSDataInputStream;
43  import org.apache.hadoop.fs.FSDataOutputStream;
44  import org.apache.hadoop.fs.FileSystem;
45  import org.apache.hadoop.fs.Path;
46  
47  @RunWith(Parameterized.class)
48  @Category(SmallTests.class)
49  public class TestFixedFileTrailer {
50  
51    private static final Log LOG = LogFactory.getLog(TestFixedFileTrailer.class);
52  
53    /**
54     * The number of used fields by version. Indexed by version minus two. 
55     * Min version that we support is V2
56     */
57    private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 14 };
58  
59    private HBaseTestingUtility util = new HBaseTestingUtility();
60    private FileSystem fs;
61    private ByteArrayOutputStream baos = new ByteArrayOutputStream();
62    private int version;
63  
64    static {
65      assert NUM_FIELDS_BY_VERSION.length == HFile.MAX_FORMAT_VERSION
66          - HFile.MIN_FORMAT_VERSION + 1;
67    }
68  
69    public TestFixedFileTrailer(int version) {
70      this.version = version;
71    }
72  
73    @Parameters
74    public static Collection<Object[]> getParameters() {
75      List<Object[]> versionsToTest = new ArrayList<Object[]>();
76      for (int v = HFile.MIN_FORMAT_VERSION; v <= HFile.MAX_FORMAT_VERSION; ++v)
77        versionsToTest.add(new Integer[] { v } );
78      return versionsToTest;
79    }
80  
81    @Before
82    public void setUp() throws IOException {
83      fs = FileSystem.get(util.getConfiguration());
84    }
85  
86    @Test
87    public void testTrailer() throws IOException {
88      FixedFileTrailer t = new FixedFileTrailer(version, 
89                             HFileBlock.MINOR_VERSION_NO_CHECKSUM);
90      t.setDataIndexCount(3);
91      t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
92  
93      t.setLastDataBlockOffset(291);
94      t.setNumDataIndexLevels(3);
95      t.setComparatorClass(KeyValue.KEY_COMPARATOR.getClass());
96      t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic.
97      t.setUncompressedDataIndexSize(827398717L); // Something random.
98  
99      t.setLoadOnOpenOffset(128);
100     t.setMetaIndexCount(7);
101 
102     t.setTotalUncompressedBytes(129731987);
103 
104     {
105       DataOutputStream dos = new DataOutputStream(baos); // Limited scope.
106       t.serialize(dos);
107       dos.flush();
108       assertEquals(dos.size(), FixedFileTrailer.getTrailerSize(version));
109     }
110 
111     byte[] bytes = baos.toByteArray();
112     baos.reset();
113 
114     assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
115 
116     ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
117 
118     // Finished writing, trying to read.
119     {
120       DataInputStream dis = new DataInputStream(bais);
121       FixedFileTrailer t2 = new FixedFileTrailer(version, 
122                               HFileBlock.MINOR_VERSION_NO_CHECKSUM);
123       t2.deserialize(dis);
124       assertEquals(-1, bais.read()); // Ensure we have read everything.
125       checkLoadedTrailer(version, t, t2);
126     }
127 
128     // Now check what happens if the trailer is corrupted.
129     Path trailerPath = new Path(util.getDataTestDir(), "trailer_" + version);
130 
131     {
132       for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1,
133           HFile.MAX_FORMAT_VERSION + 1}) {
134         bytes[bytes.length - 1] = invalidVersion;
135         writeTrailer(trailerPath, null, bytes);
136         try {
137           readTrailer(trailerPath);
138           fail("Exception expected");
139         } catch (IllegalArgumentException ex) {
140           // Make it easy to debug this.
141           String msg = ex.getMessage();
142           String cleanMsg = msg.replaceAll(
143               "^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", "");
144           assertEquals("Actual exception message is \"" + msg + "\".\n" +
145               "Cleaned-up message", // will be followed by " expected: ..."
146               "Invalid HFile version: " + invalidVersion, cleanMsg);
147           LOG.info("Got an expected exception: " + msg);
148         }
149       }
150 
151     }
152 
153     // Now write the trailer into a file and auto-detect the version.
154     writeTrailer(trailerPath, t, null);
155 
156     FixedFileTrailer t4 = readTrailer(trailerPath);
157 
158     checkLoadedTrailer(version, t, t4);
159 
160     String trailerStr = t.toString();
161     assertEquals("Invalid number of fields in the string representation "
162         + "of the trailer: " + trailerStr, NUM_FIELDS_BY_VERSION[version - 2],
163         trailerStr.split(", ").length);
164     assertEquals(trailerStr, t4.toString());
165   }
166 
167   private FixedFileTrailer readTrailer(Path trailerPath) throws IOException {
168     FSDataInputStream fsdis = fs.open(trailerPath);
169     FixedFileTrailer trailerRead = FixedFileTrailer.readFromStream(fsdis,
170         fs.getFileStatus(trailerPath).getLen());
171     fsdis.close();
172     return trailerRead;
173   }
174 
175   private void writeTrailer(Path trailerPath, FixedFileTrailer t,
176       byte[] useBytesInstead) throws IOException {
177     assert (t == null) != (useBytesInstead == null); // Expect one non-null.
178 
179     FSDataOutputStream fsdos = fs.create(trailerPath);
180     fsdos.write(135); // to make deserializer's job less trivial
181     if (useBytesInstead != null) {
182       fsdos.write(useBytesInstead);
183     } else {
184       t.serialize(fsdos);
185     }
186     fsdos.close();
187   }
188 
189   private void checkLoadedTrailer(int version, FixedFileTrailer expected,
190       FixedFileTrailer loaded) throws IOException {
191     assertEquals(version, loaded.getMajorVersion());
192     assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());
193 
194     assertEquals(Math.min(expected.getEntryCount(),
195         version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE),
196         loaded.getEntryCount());
197 
198     if (version == 1) {
199       assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset());
200     }
201 
202     if (version == 2) {
203       assertEquals(expected.getLastDataBlockOffset(),
204           loaded.getLastDataBlockOffset());
205       assertEquals(expected.getNumDataIndexLevels(),
206           loaded.getNumDataIndexLevels());
207       assertEquals(expected.createComparator().getClass().getName(),
208           loaded.createComparator().getClass().getName());
209       assertEquals(expected.getFirstDataBlockOffset(),
210           loaded.getFirstDataBlockOffset());
211       assertTrue(
212           expected.createComparator() instanceof KeyValue.KeyComparator);
213       assertEquals(expected.getUncompressedDataIndexSize(),
214           loaded.getUncompressedDataIndexSize());
215     }
216 
217     assertEquals(expected.getLoadOnOpenDataOffset(),
218         loaded.getLoadOnOpenDataOffset());
219     assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount());
220 
221     assertEquals(expected.getTotalUncompressedBytes(),
222         loaded.getTotalUncompressedBytes());
223   }
224 
225 
226 }
227