1   /*
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.io.hfile;
21  
22  import java.io.ByteArrayInputStream;
23  import java.io.ByteArrayOutputStream;
24  import java.io.DataInputStream;
25  import java.io.DataOutputStream;
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.Collection;
29  import java.util.List;
30  
31  import org.apache.hadoop.hbase.*;
32  import org.junit.Before;
33  import org.junit.Test;
34  import org.junit.experimental.categories.Category;
35  import org.junit.runner.RunWith;
36  import org.junit.runners.Parameterized;
37  import org.junit.runners.Parameterized.Parameters;
38  
39  import static org.junit.Assert.*;
40  
41  import org.apache.commons.logging.Log;
42  import org.apache.commons.logging.LogFactory;
43  import org.apache.hadoop.fs.FSDataInputStream;
44  import org.apache.hadoop.fs.FSDataOutputStream;
45  import org.apache.hadoop.fs.FileSystem;
46  import org.apache.hadoop.fs.Path;
47  
48  @RunWith(Parameterized.class)
49  @Category(MediumTests.class)
50  public class TestFixedFileTrailer {
51  
52    private static final Log LOG = LogFactory.getLog(TestFixedFileTrailer.class);
53  
54    /** The number of used fields by version. Indexed by version minus one. */
55    private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 9, 14 };
56  
57    private HBaseTestingUtility util = new HBaseTestingUtility();
58    private FileSystem fs;
59    private ByteArrayOutputStream baos = new ByteArrayOutputStream();
60    private int version;
61  
62    static {
63      assert NUM_FIELDS_BY_VERSION.length == HFile.MAX_FORMAT_VERSION
64          - HFile.MIN_FORMAT_VERSION + 1;
65    }
66  
67    public TestFixedFileTrailer(int version) {
68      this.version = version;
69    }
70  
71    @Parameters
72    public static Collection<Object[]> getParameters() {
73      List<Object[]> versionsToTest = new ArrayList<Object[]>();
74      for (int v = HFile.MIN_FORMAT_VERSION; v <= HFile.MAX_FORMAT_VERSION; ++v)
75        versionsToTest.add(new Integer[] { v } );
76      return versionsToTest;
77    }
78  
79    @Before
80    public void setUp() throws IOException {
81      fs = FileSystem.get(util.getConfiguration());
82    }
83  
84    @Test
85    public void testTrailer() throws IOException {
86      FixedFileTrailer t = new FixedFileTrailer(version, 
87                             HFileBlock.MINOR_VERSION_NO_CHECKSUM);
88      t.setDataIndexCount(3);
89      t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
90  
91      if (version == 1) {
92        t.setFileInfoOffset(876);
93      }
94  
95      if (version == 2) {
96        t.setLastDataBlockOffset(291);
97        t.setNumDataIndexLevels(3);
98        t.setComparatorClass(KeyValue.KEY_COMPARATOR.getClass());
99        t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic.
100       t.setUncompressedDataIndexSize(827398717L); // Something random.
101     }
102 
103     t.setLoadOnOpenOffset(128);
104     t.setMetaIndexCount(7);
105 
106     t.setTotalUncompressedBytes(129731987);
107 
108     {
109       DataOutputStream dos = new DataOutputStream(baos); // Limited scope.
110       t.serialize(dos);
111       dos.flush();
112       assertEquals(dos.size(), FixedFileTrailer.getTrailerSize(version));
113     }
114 
115     byte[] bytes = baos.toByteArray();
116     baos.reset();
117 
118     assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
119 
120     ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
121 
122     // Finished writing, trying to read.
123     {
124       DataInputStream dis = new DataInputStream(bais);
125       FixedFileTrailer t2 = new FixedFileTrailer(version, 
126                               HFileBlock.MINOR_VERSION_NO_CHECKSUM);
127       t2.deserialize(dis);
128       assertEquals(-1, bais.read()); // Ensure we have read everything.
129       checkLoadedTrailer(version, t, t2);
130     }
131 
132     // Now check what happens if the trailer is corrupted.
133     Path trailerPath = new Path(util.getDataTestDir(), "trailer_"
134         + version);
135 
136     {
137       for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1,
138           HFile.MAX_FORMAT_VERSION + 1}) {
139         bytes[bytes.length - 1] = invalidVersion;
140         writeTrailer(trailerPath, null, bytes);
141         try {
142           readTrailer(trailerPath);
143           fail("Exception expected");
144         } catch (IllegalArgumentException ex) {
145           // Make it easy to debug this.
146           String msg = ex.getMessage();
147           String cleanMsg = msg.replaceAll(
148               "^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", "");
149           assertEquals("Actual exception message is \"" + msg + "\".\n" +
150               "Cleaned-up message", // will be followed by " expected: ..."
151               "Invalid HFile version: " + invalidVersion, cleanMsg);
152           LOG.info("Got an expected exception: " + msg);
153         }
154       }
155 
156     }
157 
158     // Now write the trailer into a file and auto-detect the version.
159     writeTrailer(trailerPath, t, null);
160 
161     FixedFileTrailer t4 = readTrailer(trailerPath);
162 
163     checkLoadedTrailer(version, t, t4);
164 
165     String trailerStr = t.toString();
166     assertEquals("Invalid number of fields in the string representation "
167         + "of the trailer: " + trailerStr, NUM_FIELDS_BY_VERSION[version - 1],
168         trailerStr.split(", ").length);
169     assertEquals(trailerStr, t4.toString());
170   }
171 
172   private FixedFileTrailer readTrailer(Path trailerPath) throws IOException {
173     FSDataInputStream fsdis = fs.open(trailerPath);
174     FixedFileTrailer trailerRead = FixedFileTrailer.readFromStream(fsdis,
175         fs.getFileStatus(trailerPath).getLen());
176     fsdis.close();
177     return trailerRead;
178   }
179 
180   private void writeTrailer(Path trailerPath, FixedFileTrailer t,
181       byte[] useBytesInstead) throws IOException {
182     assert (t == null) != (useBytesInstead == null); // Expect one non-null.
183 
184     FSDataOutputStream fsdos = fs.create(trailerPath);
185     fsdos.write(135); // to make deserializer's job less trivial
186     if (useBytesInstead != null) {
187       fsdos.write(useBytesInstead);
188     } else {
189       t.serialize(fsdos);
190     }
191     fsdos.close();
192   }
193 
194   private void checkLoadedTrailer(int version, FixedFileTrailer expected,
195       FixedFileTrailer loaded) throws IOException {
196     assertEquals(version, loaded.getMajorVersion());
197     assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());
198 
199     assertEquals(Math.min(expected.getEntryCount(),
200         version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE),
201         loaded.getEntryCount());
202 
203     if (version == 1) {
204       assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset());
205     }
206 
207     if (version == 2) {
208       assertEquals(expected.getLastDataBlockOffset(),
209           loaded.getLastDataBlockOffset());
210       assertEquals(expected.getNumDataIndexLevels(),
211           loaded.getNumDataIndexLevels());
212       assertEquals(expected.createComparator().getClass().getName(),
213           loaded.createComparator().getClass().getName());
214       assertEquals(expected.getFirstDataBlockOffset(),
215           loaded.getFirstDataBlockOffset());
216       assertTrue(
217           expected.createComparator() instanceof KeyValue.KeyComparator);
218       assertEquals(expected.getUncompressedDataIndexSize(),
219           loaded.getUncompressedDataIndexSize());
220     }
221 
222     assertEquals(expected.getLoadOnOpenDataOffset(),
223         loaded.getLoadOnOpenDataOffset());
224     assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount());
225 
226     assertEquals(expected.getTotalUncompressedBytes(),
227         loaded.getTotalUncompressedBytes());
228   }
229 
230 
231   @org.junit.Rule
232   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
233     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
234 }
235