1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.DataInput;
22 import java.io.DataOutput;
23 import java.io.IOException;
24 import java.nio.ByteBuffer;
25 import java.util.Arrays;
26 import java.util.Map;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.fs.FSDataInputStream;
31 import org.apache.hadoop.fs.FSDataOutputStream;
32 import org.apache.hadoop.fs.FileStatus;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestCase;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
38 import org.apache.hadoop.hbase.SmallTests;
39 import org.apache.hadoop.hbase.io.compress.Compression;
40 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
41 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.io.Writable;
44 import org.junit.experimental.categories.Category;
45
46
47
48
49
50
51
52
53
54 @Category(SmallTests.class)
55 public class TestHFile extends HBaseTestCase {
56 static final Log LOG = LogFactory.getLog(TestHFile.class);
57
58 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59 private static String ROOT_DIR =
60 TEST_UTIL.getDataTestDir("TestHFile").toString();
61 private final int minBlockSize = 512;
62 private static String localFormatter = "%010d";
63 private static CacheConfig cacheConf = null;
64 private Map<String, Long> startingMetrics;
65
66 @Override
67 public void setUp() throws Exception {
68 super.setUp();
69 }
70
71 @Override
72 public void tearDown() throws Exception {
73 super.tearDown();
74 }
75
76
77
78
79
80
81
82 public void testEmptyHFile() throws IOException {
83 if (cacheConf == null) cacheConf = new CacheConfig(conf);
84 Path f = new Path(ROOT_DIR, getName());
85 Writer w =
86 HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).create();
87 w.close();
88 Reader r = HFile.createReader(fs, f, cacheConf);
89 r.loadFileInfo();
90 assertNull(r.getFirstKey());
91 assertNull(r.getLastKey());
92 }
93
94
95
96
97 public void testCorrupt0LengthHFile() throws IOException {
98 if (cacheConf == null) cacheConf = new CacheConfig(conf);
99 Path f = new Path(ROOT_DIR, getName());
100 FSDataOutputStream fsos = fs.create(f);
101 fsos.close();
102
103 try {
104 Reader r = HFile.createReader(fs, f, cacheConf);
105 } catch (CorruptHFileException che) {
106
107 return;
108 }
109 fail("Should have thrown exception");
110 }
111
112 public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
113 FileStatus fst = fs.getFileStatus(src);
114 long len = fst.getLen();
115 len = len / 2 ;
116
117
118 FSDataOutputStream fdos = fs.create(dst);
119 byte[] buf = new byte[(int)len];
120 FSDataInputStream fdis = fs.open(src);
121 fdis.read(buf);
122 fdos.write(buf);
123 fdis.close();
124 fdos.close();
125 }
126
127
128
129
130 public void testCorruptTruncatedHFile() throws IOException {
131 if (cacheConf == null) cacheConf = new CacheConfig(conf);
132 Path f = new Path(ROOT_DIR, getName());
133 Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f).create();
134 writeSomeRecords(w, 0, 100);
135 w.close();
136
137 Path trunc = new Path(f.getParent(), "trucated");
138 truncateFile(fs, w.getPath(), trunc);
139
140 try {
141 Reader r = HFile.createReader(fs, trunc, cacheConf);
142 } catch (CorruptHFileException che) {
143
144 return;
145 }
146 fail("Should have thrown exception");
147 }
148
149
150
151 private int writeSomeRecords(Writer writer, int start, int n)
152 throws IOException {
153 String value = "value";
154 for (int i = start; i < (start + n); i++) {
155 String key = String.format(localFormatter, Integer.valueOf(i));
156 writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key));
157 }
158 return (start + n);
159 }
160
161 private void readAllRecords(HFileScanner scanner) throws IOException {
162 readAndCheckbytes(scanner, 0, 100);
163 }
164
165
166 private int readAndCheckbytes(HFileScanner scanner, int start, int n)
167 throws IOException {
168 String value = "value";
169 int i = start;
170 for (; i < (start + n); i++) {
171 ByteBuffer key = scanner.getKey();
172 ByteBuffer val = scanner.getValue();
173 String keyStr = String.format(localFormatter, Integer.valueOf(i));
174 String valStr = value + keyStr;
175 byte [] keyBytes = Bytes.toBytes(key);
176 assertTrue("bytes for keys do not match " + keyStr + " " +
177 Bytes.toString(Bytes.toBytes(key)),
178 Arrays.equals(Bytes.toBytes(keyStr), keyBytes));
179 byte [] valBytes = Bytes.toBytes(val);
180 assertTrue("bytes for vals do not match " + valStr + " " +
181 Bytes.toString(valBytes),
182 Arrays.equals(Bytes.toBytes(valStr), valBytes));
183 if (!scanner.next()) {
184 break;
185 }
186 }
187 assertEquals(i, start + n - 1);
188 return (start + n);
189 }
190
191 private byte[] getSomeKey(int rowId) {
192 return String.format(localFormatter, Integer.valueOf(rowId)).getBytes();
193 }
194
195 private void writeRecords(Writer writer) throws IOException {
196 writeSomeRecords(writer, 0, 100);
197 writer.close();
198 }
199
200 private FSDataOutputStream createFSOutput(Path name) throws IOException {
201
202 FSDataOutputStream fout = fs.create(name);
203 return fout;
204 }
205
206
207
208
209 void basicWithSomeCodec(String codec) throws IOException {
210 if (cacheConf == null) cacheConf = new CacheConfig(conf);
211 Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString());
212 FSDataOutputStream fout = createFSOutput(ncTFile);
213 Writer writer = HFile.getWriterFactory(conf, cacheConf)
214 .withOutputStream(fout)
215 .withBlockSize(minBlockSize)
216 .withCompression(codec)
217 .create();
218 LOG.info(writer);
219 writeRecords(writer);
220 fout.close();
221 FSDataInputStream fin = fs.open(ncTFile);
222 Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
223 fs.getFileStatus(ncTFile).getLen(), cacheConf);
224 System.out.println(cacheConf.toString());
225
226 reader.loadFileInfo();
227
228 HFileScanner scanner = reader.getScanner(true, false);
229
230 scanner.seekTo();
231 readAllRecords(scanner);
232 scanner.seekTo(getSomeKey(50));
233 assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0);
234
235 ByteBuffer readKey = scanner.getKey();
236 assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
237 Bytes.toBytes(readKey)));
238
239 scanner.seekTo(new byte[0]);
240 ByteBuffer val1 = scanner.getValue();
241 scanner.seekTo(new byte[0]);
242 ByteBuffer val2 = scanner.getValue();
243 assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
244
245 reader.close();
246 fin.close();
247 fs.delete(ncTFile, true);
248 }
249
250 public void testTFileFeatures() throws IOException {
251 basicWithSomeCodec("none");
252 basicWithSomeCodec("gz");
253 }
254
255 private void writeNumMetablocks(Writer writer, int n) {
256 for (int i = 0; i < n; i++) {
257 writer.appendMetaBlock("HFileMeta" + i, new Writable() {
258 private int val;
259 public Writable setVal(int val) { this.val = val; return this; }
260
261 @Override
262 public void write(DataOutput out) throws IOException {
263 out.write(("something to test" + val).getBytes());
264 }
265
266 @Override
267 public void readFields(DataInput in) throws IOException { }
268 }.setVal(i));
269 }
270 }
271
272 private void someTestingWithMetaBlock(Writer writer) {
273 writeNumMetablocks(writer, 10);
274 }
275
276 private void readNumMetablocks(Reader reader, int n) throws IOException {
277 for (int i = 0; i < n; i++) {
278 ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
279 ByteBuffer expected =
280 ByteBuffer.wrap(("something to test" + i).getBytes());
281 assertTrue("failed to match metadata", actual.compareTo(expected) == 0);
282 }
283 }
284
285 private void someReadingWithMetaBlock(Reader reader) throws IOException {
286 readNumMetablocks(reader, 10);
287 }
288
289 private void metablocks(final String compress) throws Exception {
290 if (cacheConf == null) cacheConf = new CacheConfig(conf);
291 Path mFile = new Path(ROOT_DIR, "meta.hfile");
292 FSDataOutputStream fout = createFSOutput(mFile);
293 Writer writer = HFile.getWriterFactory(conf, cacheConf)
294 .withOutputStream(fout)
295 .withBlockSize(minBlockSize)
296 .withCompression(compress)
297 .create();
298 someTestingWithMetaBlock(writer);
299 writer.close();
300 fout.close();
301 FSDataInputStream fin = fs.open(mFile);
302 Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
303 this.fs.getFileStatus(mFile).getLen(), cacheConf);
304 reader.loadFileInfo();
305
306 assertFalse(reader.getScanner(false, false).seekTo());
307 someReadingWithMetaBlock(reader);
308 fs.delete(mFile, true);
309 reader.close();
310 fin.close();
311 }
312
313
314 public void testMetaBlocks() throws Exception {
315 metablocks("none");
316 metablocks("gz");
317 }
318
319 public void testNullMetaBlocks() throws Exception {
320 if (cacheConf == null) cacheConf = new CacheConfig(conf);
321 for (Compression.Algorithm compressAlgo :
322 HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
323 Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
324 FSDataOutputStream fout = createFSOutput(mFile);
325 Writer writer = HFile.getWriterFactory(conf, cacheConf)
326 .withOutputStream(fout)
327 .withBlockSize(minBlockSize)
328 .withCompression(compressAlgo)
329 .create();
330 writer.append("foo".getBytes(), "value".getBytes());
331 writer.close();
332 fout.close();
333 Reader reader = HFile.createReader(fs, mFile, cacheConf);
334 reader.loadFileInfo();
335 assertNull(reader.getMetaBlock("non-existant", false));
336 }
337 }
338
339
340
341
342 public void testCompressionOrdinance() {
343 assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
344 assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
345 assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
346 assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
347 assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
348 }
349
350
351
352 static class CustomKeyComparator extends KeyComparator {
353 @Override
354 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2,
355 int l2) {
356 return -Bytes.compareTo(b1, s1, l1, b2, s2, l2);
357 }
358 @Override
359 public int compare(byte[] o1, byte[] o2) {
360 return compare(o1, 0, o1.length, o2, 0, o2.length);
361 }
362 }
363
364 public void testComparator() throws IOException {
365 if (cacheConf == null) cacheConf = new CacheConfig(conf);
366 Path mFile = new Path(ROOT_DIR, "meta.tfile");
367 FSDataOutputStream fout = createFSOutput(mFile);
368 KeyComparator comparator = new CustomKeyComparator();
369 Writer writer = HFile.getWriterFactory(conf, cacheConf)
370 .withOutputStream(fout)
371 .withBlockSize(minBlockSize)
372 .withComparator(comparator)
373 .create();
374 writer.append("3".getBytes(), "0".getBytes());
375 writer.append("2".getBytes(), "0".getBytes());
376 writer.append("1".getBytes(), "0".getBytes());
377 writer.close();
378 }
379
380
381 }
382