1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import java.io.ByteArrayInputStream;
23 import java.io.ByteArrayOutputStream;
24 import java.io.DataInputStream;
25 import java.io.DataOutputStream;
26 import java.io.IOException;
27 import java.util.ArrayList;
28 import java.util.Collection;
29 import java.util.List;
30
31 import org.apache.hadoop.hbase.*;
32 import org.junit.Before;
33 import org.junit.Test;
34 import org.junit.experimental.categories.Category;
35 import org.junit.runner.RunWith;
36 import org.junit.runners.Parameterized;
37 import org.junit.runners.Parameterized.Parameters;
38
39 import static org.junit.Assert.*;
40
41 import org.apache.commons.logging.Log;
42 import org.apache.commons.logging.LogFactory;
43 import org.apache.hadoop.fs.FSDataInputStream;
44 import org.apache.hadoop.fs.FSDataOutputStream;
45 import org.apache.hadoop.fs.FileSystem;
46 import org.apache.hadoop.fs.Path;
47
48 @RunWith(Parameterized.class)
49 @Category(MediumTests.class)
50 public class TestFixedFileTrailer {
51
52 private static final Log LOG = LogFactory.getLog(TestFixedFileTrailer.class);
53
54
55 private static final int[] NUM_FIELDS_BY_VERSION = new int[] { 9, 14 };
56
57 private HBaseTestingUtility util = new HBaseTestingUtility();
58 private FileSystem fs;
59 private ByteArrayOutputStream baos = new ByteArrayOutputStream();
60 private int version;
61
62 static {
63 assert NUM_FIELDS_BY_VERSION.length == HFile.MAX_FORMAT_VERSION
64 - HFile.MIN_FORMAT_VERSION + 1;
65 }
66
67 public TestFixedFileTrailer(int version) {
68 this.version = version;
69 }
70
71 @Parameters
72 public static Collection<Object[]> getParameters() {
73 List<Object[]> versionsToTest = new ArrayList<Object[]>();
74 for (int v = HFile.MIN_FORMAT_VERSION; v <= HFile.MAX_FORMAT_VERSION; ++v)
75 versionsToTest.add(new Integer[] { v } );
76 return versionsToTest;
77 }
78
79 @Before
80 public void setUp() throws IOException {
81 fs = FileSystem.get(util.getConfiguration());
82 }
83
84 @Test
85 public void testTrailer() throws IOException {
86 FixedFileTrailer t = new FixedFileTrailer(version,
87 HFileBlock.MINOR_VERSION_NO_CHECKSUM);
88 t.setDataIndexCount(3);
89 t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
90
91 if (version == 1) {
92 t.setFileInfoOffset(876);
93 }
94
95 if (version == 2) {
96 t.setLastDataBlockOffset(291);
97 t.setNumDataIndexLevels(3);
98 t.setComparatorClass(KeyValue.KEY_COMPARATOR.getClass());
99 t.setFirstDataBlockOffset(9081723123L);
100 t.setUncompressedDataIndexSize(827398717L);
101 }
102
103 t.setLoadOnOpenOffset(128);
104 t.setMetaIndexCount(7);
105
106 t.setTotalUncompressedBytes(129731987);
107
108 {
109 DataOutputStream dos = new DataOutputStream(baos);
110 t.serialize(dos);
111 dos.flush();
112 assertEquals(dos.size(), FixedFileTrailer.getTrailerSize(version));
113 }
114
115 byte[] bytes = baos.toByteArray();
116 baos.reset();
117
118 assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));
119
120 ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
121
122
123 {
124 DataInputStream dis = new DataInputStream(bais);
125 FixedFileTrailer t2 = new FixedFileTrailer(version,
126 HFileBlock.MINOR_VERSION_NO_CHECKSUM);
127 t2.deserialize(dis);
128 assertEquals(-1, bais.read());
129 checkLoadedTrailer(version, t, t2);
130 }
131
132
133 Path trailerPath = new Path(util.getDataTestDir(), "trailer_"
134 + version);
135
136 {
137 for (byte invalidVersion : new byte[] { HFile.MIN_FORMAT_VERSION - 1,
138 HFile.MAX_FORMAT_VERSION + 1}) {
139 bytes[bytes.length - 1] = invalidVersion;
140 writeTrailer(trailerPath, null, bytes);
141 try {
142 readTrailer(trailerPath);
143 fail("Exception expected");
144 } catch (IllegalArgumentException ex) {
145
146 String msg = ex.getMessage();
147 String cleanMsg = msg.replaceAll(
148 "^(java(\\.[a-zA-Z]+)+:\\s+)?|\\s+\\(.*\\)\\s*$", "");
149 assertEquals("Actual exception message is \"" + msg + "\".\n" +
150 "Cleaned-up message",
151 "Invalid HFile version: " + invalidVersion, cleanMsg);
152 LOG.info("Got an expected exception: " + msg);
153 }
154 }
155
156 }
157
158
159 writeTrailer(trailerPath, t, null);
160
161 FixedFileTrailer t4 = readTrailer(trailerPath);
162
163 checkLoadedTrailer(version, t, t4);
164
165 String trailerStr = t.toString();
166 assertEquals("Invalid number of fields in the string representation "
167 + "of the trailer: " + trailerStr, NUM_FIELDS_BY_VERSION[version - 1],
168 trailerStr.split(", ").length);
169 assertEquals(trailerStr, t4.toString());
170 }
171
172 private FixedFileTrailer readTrailer(Path trailerPath) throws IOException {
173 FSDataInputStream fsdis = fs.open(trailerPath);
174 FixedFileTrailer trailerRead = FixedFileTrailer.readFromStream(fsdis,
175 fs.getFileStatus(trailerPath).getLen());
176 fsdis.close();
177 return trailerRead;
178 }
179
180 private void writeTrailer(Path trailerPath, FixedFileTrailer t,
181 byte[] useBytesInstead) throws IOException {
182 assert (t == null) != (useBytesInstead == null);
183
184 FSDataOutputStream fsdos = fs.create(trailerPath);
185 fsdos.write(135);
186 if (useBytesInstead != null) {
187 fsdos.write(useBytesInstead);
188 } else {
189 t.serialize(fsdos);
190 }
191 fsdos.close();
192 }
193
194 private void checkLoadedTrailer(int version, FixedFileTrailer expected,
195 FixedFileTrailer loaded) throws IOException {
196 assertEquals(version, loaded.getMajorVersion());
197 assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());
198
199 assertEquals(Math.min(expected.getEntryCount(),
200 version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE),
201 loaded.getEntryCount());
202
203 if (version == 1) {
204 assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset());
205 }
206
207 if (version == 2) {
208 assertEquals(expected.getLastDataBlockOffset(),
209 loaded.getLastDataBlockOffset());
210 assertEquals(expected.getNumDataIndexLevels(),
211 loaded.getNumDataIndexLevels());
212 assertEquals(expected.createComparator().getClass().getName(),
213 loaded.createComparator().getClass().getName());
214 assertEquals(expected.getFirstDataBlockOffset(),
215 loaded.getFirstDataBlockOffset());
216 assertTrue(
217 expected.createComparator() instanceof KeyValue.KeyComparator);
218 assertEquals(expected.getUncompressedDataIndexSize(),
219 loaded.getUncompressedDataIndexSize());
220 }
221
222 assertEquals(expected.getLoadOnOpenDataOffset(),
223 loaded.getLoadOnOpenDataOffset());
224 assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount());
225
226 assertEquals(expected.getTotalUncompressedBytes(),
227 loaded.getTotalUncompressedBytes());
228 }
229
230
231 @org.junit.Rule
232 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
233 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
234 }
235