1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import java.io.DataInput;
23 import java.io.DataOutput;
24 import java.io.IOException;
25 import java.nio.ByteBuffer;
26 import java.util.Arrays;
27 import java.util.Map;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.fs.FSDataInputStream;
32 import org.apache.hadoop.fs.FSDataOutputStream;
33 import org.apache.hadoop.fs.FileStatus;
34 import org.apache.hadoop.fs.FileSystem;
35 import org.apache.hadoop.fs.Path;
36 import org.apache.hadoop.hbase.HBaseTestCase;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
39 import org.apache.hadoop.hbase.SmallTests;
40 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
41 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.io.Writable;
44 import org.junit.experimental.categories.Category;
45
46
47
48
49
50
51
52
53
54 @Category(SmallTests.class)
55 public class TestHFile extends HBaseTestCase {
56 static final Log LOG = LogFactory.getLog(TestHFile.class);
57
58 private String ROOT_DIR;
59 private final int minBlockSize = 512;
60 private static String localFormatter = "%010d";
61 private static CacheConfig cacheConf = null;
62
63 @Override
64 public void setUp() throws Exception {
65 ROOT_DIR = this.getUnitTestdir("TestHFile").toString();
66 super.setUp();
67 }
68
69 @Override
70 public void tearDown() throws Exception {
71 super.tearDown();
72 }
73
74
75
76
77
78
79
80 public void testEmptyHFile() throws IOException {
81 if (cacheConf == null) cacheConf = new CacheConfig(conf);
82 Path f = new Path(ROOT_DIR, getName());
83 Writer w =
84 HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).create();
85 w.close();
86 Reader r = HFile.createReader(fs, f, cacheConf);
87 r.loadFileInfo();
88 assertNull(r.getFirstKey());
89 assertNull(r.getLastKey());
90 }
91
92
93
94
95 public void testCorrupt0LengthHFile() throws IOException {
96 if (cacheConf == null) cacheConf = new CacheConfig(conf);
97 Path f = new Path(ROOT_DIR, getName());
98 FSDataOutputStream fsos = fs.create(f);
99 fsos.close();
100
101 try {
102 Reader r = HFile.createReader(fs, f, cacheConf);
103 } catch (CorruptHFileException che) {
104
105 return;
106 }
107 fail("Should have thrown exception");
108 }
109
110 public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
111 FileStatus fst = fs.getFileStatus(src);
112 long len = fst.getLen();
113 len = len / 2 ;
114
115
116 FSDataOutputStream fdos = fs.create(dst);
117 byte[] buf = new byte[(int)len];
118 FSDataInputStream fdis = fs.open(src);
119 fdis.read(buf);
120 fdos.write(buf);
121 fdis.close();
122 fdos.close();
123 }
124
125
126
127
128 public void testCorruptTruncatedHFile() throws IOException {
129 if (cacheConf == null) cacheConf = new CacheConfig(conf);
130 Path f = new Path(ROOT_DIR, getName());
131 Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f).create();
132 writeSomeRecords(w, 0, 100);
133 w.close();
134
135 Path trunc = new Path(f.getParent(), "trucated");
136 truncateFile(fs, w.getPath(), trunc);
137
138 try {
139 Reader r = HFile.createReader(fs, trunc, cacheConf);
140 } catch (CorruptHFileException che) {
141
142 return;
143 }
144 fail("Should have thrown exception");
145 }
146
147
148
149 private int writeSomeRecords(Writer writer, int start, int n)
150 throws IOException {
151 String value = "value";
152 for (int i = start; i < (start + n); i++) {
153 String key = String.format(localFormatter, Integer.valueOf(i));
154 writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key));
155 }
156 return (start + n);
157 }
158
159 private void readAllRecords(HFileScanner scanner) throws IOException {
160 readAndCheckbytes(scanner, 0, 100);
161 }
162
163
164 private int readAndCheckbytes(HFileScanner scanner, int start, int n)
165 throws IOException {
166 String value = "value";
167 int i = start;
168 for (; i < (start + n); i++) {
169 ByteBuffer key = scanner.getKey();
170 ByteBuffer val = scanner.getValue();
171 String keyStr = String.format(localFormatter, Integer.valueOf(i));
172 String valStr = value + keyStr;
173 byte [] keyBytes = Bytes.toBytes(key);
174 assertTrue("bytes for keys do not match " + keyStr + " " +
175 Bytes.toString(Bytes.toBytes(key)),
176 Arrays.equals(Bytes.toBytes(keyStr), keyBytes));
177 byte [] valBytes = Bytes.toBytes(val);
178 assertTrue("bytes for vals do not match " + valStr + " " +
179 Bytes.toString(valBytes),
180 Arrays.equals(Bytes.toBytes(valStr), valBytes));
181 if (!scanner.next()) {
182 break;
183 }
184 }
185 assertEquals(i, start + n - 1);
186 return (start + n);
187 }
188
189 private byte[] getSomeKey(int rowId) {
190 return String.format(localFormatter, Integer.valueOf(rowId)).getBytes();
191 }
192
193 private void writeRecords(Writer writer) throws IOException {
194 writeSomeRecords(writer, 0, 100);
195 writer.close();
196 }
197
198 private FSDataOutputStream createFSOutput(Path name) throws IOException {
199
200 FSDataOutputStream fout = fs.create(name);
201 return fout;
202 }
203
204
205
206
207 void basicWithSomeCodec(String codec) throws IOException {
208 if (cacheConf == null) cacheConf = new CacheConfig(conf);
209 Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString());
210 FSDataOutputStream fout = createFSOutput(ncTFile);
211 Writer writer = HFile.getWriterFactory(conf, cacheConf)
212 .withOutputStream(fout)
213 .withBlockSize(minBlockSize)
214 .withCompression(codec)
215 .create();
216 LOG.info(writer);
217 writeRecords(writer);
218 fout.close();
219 FSDataInputStream fin = fs.open(ncTFile);
220 Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
221 fs.getFileStatus(ncTFile).getLen(), cacheConf);
222 System.out.println(cacheConf.toString());
223
224 reader.loadFileInfo();
225
226 HFileScanner scanner = reader.getScanner(true, false);
227
228 scanner.seekTo();
229 readAllRecords(scanner);
230 scanner.seekTo(getSomeKey(50));
231 assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0);
232
233 ByteBuffer readKey = scanner.getKey();
234 assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
235 Bytes.toBytes(readKey)));
236
237 scanner.seekTo(new byte[0]);
238 ByteBuffer val1 = scanner.getValue();
239 scanner.seekTo(new byte[0]);
240 ByteBuffer val2 = scanner.getValue();
241 assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
242
243 reader.close();
244 fin.close();
245 fs.delete(ncTFile, true);
246 }
247
248 public void testTFileFeatures() throws IOException {
249 basicWithSomeCodec("none");
250 basicWithSomeCodec("gz");
251 }
252
253 private void writeNumMetablocks(Writer writer, int n) {
254 for (int i = 0; i < n; i++) {
255 writer.appendMetaBlock("HFileMeta" + i, new Writable() {
256 private int val;
257 public Writable setVal(int val) { this.val = val; return this; }
258
259 @Override
260 public void write(DataOutput out) throws IOException {
261 out.write(("something to test" + val).getBytes());
262 }
263
264 @Override
265 public void readFields(DataInput in) throws IOException { }
266 }.setVal(i));
267 }
268 }
269
270 private void someTestingWithMetaBlock(Writer writer) {
271 writeNumMetablocks(writer, 10);
272 }
273
274 private void readNumMetablocks(Reader reader, int n) throws IOException {
275 for (int i = 0; i < n; i++) {
276 ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
277 ByteBuffer expected =
278 ByteBuffer.wrap(("something to test" + i).getBytes());
279 assertTrue("failed to match metadata", actual.compareTo(expected) == 0);
280 }
281 }
282
283 private void someReadingWithMetaBlock(Reader reader) throws IOException {
284 readNumMetablocks(reader, 10);
285 }
286
287 private void metablocks(final String compress) throws Exception {
288 if (cacheConf == null) cacheConf = new CacheConfig(conf);
289 Path mFile = new Path(ROOT_DIR, "meta.hfile");
290 FSDataOutputStream fout = createFSOutput(mFile);
291 Writer writer = HFile.getWriterFactory(conf, cacheConf)
292 .withOutputStream(fout)
293 .withBlockSize(minBlockSize)
294 .withCompression(compress)
295 .create();
296 someTestingWithMetaBlock(writer);
297 writer.close();
298 fout.close();
299 FSDataInputStream fin = fs.open(mFile);
300 Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
301 this.fs.getFileStatus(mFile).getLen(), cacheConf);
302 reader.loadFileInfo();
303
304 assertFalse(reader.getScanner(false, false).seekTo());
305 someReadingWithMetaBlock(reader);
306 fs.delete(mFile, true);
307 reader.close();
308 fin.close();
309 }
310
311
312 public void testMetaBlocks() throws Exception {
313 metablocks("none");
314 metablocks("gz");
315 }
316
317 public void testNullMetaBlocks() throws Exception {
318 if (cacheConf == null) cacheConf = new CacheConfig(conf);
319 for (Compression.Algorithm compressAlgo :
320 HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
321 Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
322 FSDataOutputStream fout = createFSOutput(mFile);
323 Writer writer = HFile.getWriterFactory(conf, cacheConf)
324 .withOutputStream(fout)
325 .withBlockSize(minBlockSize)
326 .withCompression(compressAlgo)
327 .create();
328 writer.append("foo".getBytes(), "value".getBytes());
329 writer.close();
330 fout.close();
331 Reader reader = HFile.createReader(fs, mFile, cacheConf);
332 reader.loadFileInfo();
333 assertNull(reader.getMetaBlock("non-existant", false));
334 }
335 }
336
337
338
339
340 public void testCompressionOrdinance() {
341 assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
342 assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
343 assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
344 assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
345 assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
346 }
347
348 public void testComparator() throws IOException {
349 if (cacheConf == null) cacheConf = new CacheConfig(conf);
350 Path mFile = new Path(ROOT_DIR, "meta.tfile");
351 FSDataOutputStream fout = createFSOutput(mFile);
352 KeyComparator comparator = new KeyComparator() {
353 @Override
354 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2,
355 int l2) {
356 return -Bytes.compareTo(b1, s1, l1, b2, s2, l2);
357 }
358 @Override
359 public int compare(byte[] o1, byte[] o2) {
360 return compare(o1, 0, o1.length, o2, 0, o2.length);
361 }
362 };
363 Writer writer = HFile.getWriterFactory(conf, cacheConf)
364 .withOutputStream(fout)
365 .withBlockSize(minBlockSize)
366 .withComparator(comparator)
367 .create();
368 writer.append("3".getBytes(), "0".getBytes());
369 writer.append("2".getBytes(), "0".getBytes());
370 writer.append("1".getBytes(), "0".getBytes());
371 writer.close();
372 }
373
374
375 @org.junit.Rule
376 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
377 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
378 }
379