1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.ByteArrayInputStream;
22 import java.io.ByteArrayOutputStream;
23 import java.io.DataInputStream;
24 import java.io.DataOutputStream;
25 import java.io.IOException;
26 import java.util.ArrayList;
27 import java.util.Collection;
28 import java.util.List;
29
30 import org.apache.hadoop.hbase.*;
31 import org.junit.Before;
32 import org.junit.Test;
33 import org.junit.experimental.categories.Category;
34 import org.junit.runner.RunWith;
35 import org.junit.runners.Parameterized;
36 import org.junit.runners.Parameterized.Parameters;
37
38 import static org.junit.Assert.*;
39
40 import org.apache.commons.logging.Log;
41 import org.apache.commons.logging.LogFactory;
42 import org.apache.hadoop.fs.FSDataInputStream;
43 import org.apache.hadoop.fs.FSDataOutputStream;
44 import org.apache.hadoop.fs.FileSystem;
45 import org.apache.hadoop.fs.Path;
46
47 @RunWith(Parameterized.class)
48 @Category(SmallTests.class)
49 public class TestFixedFileTrailer {
50
51 private static final Log LOG = LogFactory.getLog(TestFixedFileTrailer.class);
52
53
54
55
56
57 private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 14 };
58
59 private HBaseTestingUtility util = new HBaseTestingUtility();
60 private FileSystem fs;
61 private ByteArrayOutputStream baos = new ByteArrayOutputStream();
62 private int version;
63
64 static {
65 assert NUM_FIELDS_BY_VERSION.length == HFile.MAX_FORMAT_VERSION
66 - HFile.MIN_FORMAT_VERSION + 1;
67 }
68
69 public TestFixedFileTrailer(int version) {
70 this.version = version;
71 }
72
73 @Parameters
74 public static Collection<Object[]> getParameters() {
75 List<Object[]> versionsToTest = new ArrayList<Object[]>();
76 for (int v = HFile.MIN_FORMAT_VERSION; v <= HFile.MAX_FORMAT_VERSION; ++v)
77 versionsToTest.add(new Integer[] { v } );
78 return versionsToTest;
79 }
80
81 @Before
82 public void setUp() throws IOException {
83 fs = FileSystem.get(util.getConfiguration());
84 }
85
86 @Test
87 public void testTrailer() throws IOException {
88 FixedFileTrailer t = new FixedFileTrailer(version,
89 HFileBlock.MINOR_VERSION_NO_CHECKSUM);
90 t.setDataIndexCount(3);
91 t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
92
93 t.setLastDataBlockOffset(291);
94 t.setNumDataIndexLevels(3);
95 t.setComparatorClass(KeyValue.KEY_COMPARATOR.getClass());
96 t.setFirstDataBlockOffset(9081723123L);
97 t.setUncompressedDataIndexSize(827398717L);
98
99 t.setLoadOnOpenOffset(128);
100 t.setMetaIndexCount(7);
101
102 t.setTotalUncompressedBytes(129731987);
103
104 {
105 DataOutputStream dos = new DataOutputStream(baos);
106 t.serialize(dos);
107 dos.flush();
108 assertEquals(dos.size(), FixedFileTrailer.getTrailerSize(version));
109 }
110
111 byte[] bytes = baos.toByteArray();
112 baos.reset();
113
114 assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
115
116 ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
117
118
119 {
120 DataInputStream dis = new DataInputStream(bais);
121 FixedFileTrailer t2 = new FixedFileTrailer(version,
122 HFileBlock.MINOR_VERSION_NO_CHECKSUM);
123 t2.deserialize(dis);
124 assertEquals(-1, bais.read());
125 checkLoadedTrailer(version, t, t2);
126 }
127
128
129 Path trailerPath = new Path(util.getDataTestDir(), "trailer_" + version);
130
131 {
132 for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1,
133 HFile.MAX_FORMAT_VERSION + 1}) {
134 bytes[bytes.length - 1] = invalidVersion;
135 writeTrailer(trailerPath, null, bytes);
136 try {
137 readTrailer(trailerPath);
138 fail("Exception expected");
139 } catch (IllegalArgumentException ex) {
140
141 String msg = ex.getMessage();
142 String cleanMsg = msg.replaceAll(
143 "^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", "");
144 assertEquals("Actual exception message is \"" + msg + "\".\n" +
145 "Cleaned-up message",
146 "Invalid HFile version: " + invalidVersion, cleanMsg);
147 LOG.info("Got an expected exception: " + msg);
148 }
149 }
150
151 }
152
153
154 writeTrailer(trailerPath, t, null);
155
156 FixedFileTrailer t4 = readTrailer(trailerPath);
157
158 checkLoadedTrailer(version, t, t4);
159
160 String trailerStr = t.toString();
161 assertEquals("Invalid number of fields in the string representation "
162 + "of the trailer: " + trailerStr, NUM_FIELDS_BY_VERSION[version - 2],
163 trailerStr.split(", ").length);
164 assertEquals(trailerStr, t4.toString());
165 }
166
167 private FixedFileTrailer readTrailer(Path trailerPath) throws IOException {
168 FSDataInputStream fsdis = fs.open(trailerPath);
169 FixedFileTrailer trailerRead = FixedFileTrailer.readFromStream(fsdis,
170 fs.getFileStatus(trailerPath).getLen());
171 fsdis.close();
172 return trailerRead;
173 }
174
175 private void writeTrailer(Path trailerPath, FixedFileTrailer t,
176 byte[] useBytesInstead) throws IOException {
177 assert (t == null) != (useBytesInstead == null);
178
179 FSDataOutputStream fsdos = fs.create(trailerPath);
180 fsdos.write(135);
181 if (useBytesInstead != null) {
182 fsdos.write(useBytesInstead);
183 } else {
184 t.serialize(fsdos);
185 }
186 fsdos.close();
187 }
188
189 private void checkLoadedTrailer(int version, FixedFileTrailer expected,
190 FixedFileTrailer loaded) throws IOException {
191 assertEquals(version, loaded.getMajorVersion());
192 assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());
193
194 assertEquals(Math.min(expected.getEntryCount(),
195 version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE),
196 loaded.getEntryCount());
197
198 if (version == 1) {
199 assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset());
200 }
201
202 if (version == 2) {
203 assertEquals(expected.getLastDataBlockOffset(),
204 loaded.getLastDataBlockOffset());
205 assertEquals(expected.getNumDataIndexLevels(),
206 loaded.getNumDataIndexLevels());
207 assertEquals(expected.createComparator().getClass().getName(),
208 loaded.createComparator().getClass().getName());
209 assertEquals(expected.getFirstDataBlockOffset(),
210 loaded.getFirstDataBlockOffset());
211 assertTrue(
212 expected.createComparator() instanceof KeyValue.KeyComparator);
213 assertEquals(expected.getUncompressedDataIndexSize(),
214 loaded.getUncompressedDataIndexSize());
215 }
216
217 assertEquals(expected.getLoadOnOpenDataOffset(),
218 loaded.getLoadOnOpenDataOffset());
219 assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount());
220
221 assertEquals(expected.getTotalUncompressedBytes(),
222 loaded.getTotalUncompressedBytes());
223 }
224
225
226 }
227