1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.io.hfile;
20
21 import java.io.DataInput;
22 import java.io.DataOutput;
23 import java.io.IOException;
24 import java.nio.ByteBuffer;
25 import java.util.Arrays;
26 import java.util.Map;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.fs.FSDataInputStream;
31 import org.apache.hadoop.fs.FSDataOutputStream;
32 import org.apache.hadoop.fs.FileStatus;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.hbase.HBaseTestCase;
36 import org.apache.hadoop.hbase.HBaseTestingUtility;
37 import org.apache.hadoop.hbase.KeyValue;
38 import org.apache.hadoop.hbase.SmallTests;
39 import org.apache.hadoop.hbase.io.compress.Compression;
40 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
41 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.io.Writable;
44 import org.junit.experimental.categories.Category;
45
46
47
48
49
50
51
52
53
54 @Category(SmallTests.class)
55 public class TestHFile extends HBaseTestCase {
56 static final Log LOG = LogFactory.getLog(TestHFile.class);
57
58 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
59 private static String ROOT_DIR =
60 TEST_UTIL.getDataTestDir("TestHFile").toString();
61 private final int minBlockSize = 512;
62 private static String localFormatter = "%010d";
63 private static CacheConfig cacheConf = null;
64 private Map<String, Long> startingMetrics;
65
66 @Override
67 public void setUp() throws Exception {
68 super.setUp();
69 }
70
71 @Override
72 public void tearDown() throws Exception {
73 super.tearDown();
74 }
75
76
77
78
79
80
81
82 public void testEmptyHFile() throws IOException {
83 if (cacheConf == null) cacheConf = new CacheConfig(conf);
84 Path f = new Path(ROOT_DIR, getName());
85 Writer w =
86 HFile.getWriterFactory(conf, cacheConf).withPath(fs, f).create();
87 w.close();
88 Reader r = HFile.createReader(fs, f, cacheConf);
89 r.loadFileInfo();
90 assertNull(r.getFirstKey());
91 assertNull(r.getLastKey());
92 }
93
94
95
96
97 public void testCorrupt0LengthHFile() throws IOException {
98 if (cacheConf == null) cacheConf = new CacheConfig(conf);
99 Path f = new Path(ROOT_DIR, getName());
100 FSDataOutputStream fsos = fs.create(f);
101 fsos.close();
102
103 try {
104 Reader r = HFile.createReader(fs, f, cacheConf);
105 } catch (CorruptHFileException che) {
106
107 return;
108 }
109 fail("Should have thrown exception");
110 }
111
112 public static void truncateFile(FileSystem fs, Path src, Path dst) throws IOException {
113 FileStatus fst = fs.getFileStatus(src);
114 long len = fst.getLen();
115 len = len / 2 ;
116
117
118 FSDataOutputStream fdos = fs.create(dst);
119 byte[] buf = new byte[(int)len];
120 FSDataInputStream fdis = fs.open(src);
121 fdis.read(buf);
122 fdos.write(buf);
123 fdis.close();
124 fdos.close();
125 }
126
127
128
129
130 public void testCorruptTruncatedHFile() throws IOException {
131 if (cacheConf == null) cacheConf = new CacheConfig(conf);
132 Path f = new Path(ROOT_DIR, getName());
133 Writer w = HFile.getWriterFactory(conf, cacheConf).withPath(this.fs, f).create();
134 writeSomeRecords(w, 0, 100);
135 w.close();
136
137 Path trunc = new Path(f.getParent(), "trucated");
138 truncateFile(fs, w.getPath(), trunc);
139
140 try {
141 Reader r = HFile.createReader(fs, trunc, cacheConf);
142 } catch (CorruptHFileException che) {
143
144 return;
145 }
146 fail("Should have thrown exception");
147 }
148
149
150
151 private int writeSomeRecords(Writer writer, int start, int n)
152 throws IOException {
153 String value = "value";
154 for (int i = start; i < (start + n); i++) {
155 String key = String.format(localFormatter, Integer.valueOf(i));
156 writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key));
157 }
158 return (start + n);
159 }
160
161 private void readAllRecords(HFileScanner scanner) throws IOException {
162 readAndCheckbytes(scanner, 0, 100);
163 }
164
165
166 private int readAndCheckbytes(HFileScanner scanner, int start, int n)
167 throws IOException {
168 String value = "value";
169 int i = start;
170 for (; i < (start + n); i++) {
171 ByteBuffer key = scanner.getKey();
172 ByteBuffer val = scanner.getValue();
173 String keyStr = String.format(localFormatter, Integer.valueOf(i));
174 String valStr = value + keyStr;
175 byte [] keyBytes = Bytes.toBytes(key);
176 assertTrue("bytes for keys do not match " + keyStr + " " +
177 Bytes.toString(Bytes.toBytes(key)),
178 Arrays.equals(Bytes.toBytes(keyStr), keyBytes));
179 byte [] valBytes = Bytes.toBytes(val);
180 assertTrue("bytes for vals do not match " + valStr + " " +
181 Bytes.toString(valBytes),
182 Arrays.equals(Bytes.toBytes(valStr), valBytes));
183 if (!scanner.next()) {
184 break;
185 }
186 }
187 assertEquals(i, start + n - 1);
188 return (start + n);
189 }
190
191 private byte[] getSomeKey(int rowId) {
192 return String.format(localFormatter, Integer.valueOf(rowId)).getBytes();
193 }
194
195 private void writeRecords(Writer writer) throws IOException {
196 writeSomeRecords(writer, 0, 100);
197 writer.close();
198 }
199
200 private FSDataOutputStream createFSOutput(Path name) throws IOException {
201
202 FSDataOutputStream fout = fs.create(name);
203 return fout;
204 }
205
206
207
208
209 void basicWithSomeCodec(String codec) throws IOException {
210 if (cacheConf == null) cacheConf = new CacheConfig(conf);
211 Path ncTFile = new Path(ROOT_DIR, "basic.hfile." + codec.toString());
212 FSDataOutputStream fout = createFSOutput(ncTFile);
213 Writer writer = HFile.getWriterFactory(conf, cacheConf)
214 .withOutputStream(fout)
215 .withBlockSize(minBlockSize)
216 .withCompression(codec)
217
218 .withComparator(new KeyValue.RawBytesComparator())
219 .create();
220 LOG.info(writer);
221 writeRecords(writer);
222 fout.close();
223 FSDataInputStream fin = fs.open(ncTFile);
224 Reader reader = HFile.createReaderFromStream(ncTFile, fs.open(ncTFile),
225 fs.getFileStatus(ncTFile).getLen(), cacheConf);
226 System.out.println(cacheConf.toString());
227
228 reader.loadFileInfo();
229
230 HFileScanner scanner = reader.getScanner(true, false);
231
232 scanner.seekTo();
233 readAllRecords(scanner);
234 scanner.seekTo(getSomeKey(50));
235 assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0);
236
237 ByteBuffer readKey = scanner.getKey();
238 assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
239 Bytes.toBytes(readKey)));
240
241 scanner.seekTo(new byte[0]);
242 ByteBuffer val1 = scanner.getValue();
243 scanner.seekTo(new byte[0]);
244 ByteBuffer val2 = scanner.getValue();
245 assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
246
247 reader.close();
248 fin.close();
249 fs.delete(ncTFile, true);
250 }
251
252 public void testTFileFeatures() throws IOException {
253 basicWithSomeCodec("none");
254 basicWithSomeCodec("gz");
255 }
256
257 private void writeNumMetablocks(Writer writer, int n) {
258 for (int i = 0; i < n; i++) {
259 writer.appendMetaBlock("HFileMeta" + i, new Writable() {
260 private int val;
261 public Writable setVal(int val) { this.val = val; return this; }
262
263 @Override
264 public void write(DataOutput out) throws IOException {
265 out.write(("something to test" + val).getBytes());
266 }
267
268 @Override
269 public void readFields(DataInput in) throws IOException { }
270 }.setVal(i));
271 }
272 }
273
274 private void someTestingWithMetaBlock(Writer writer) {
275 writeNumMetablocks(writer, 10);
276 }
277
278 private void readNumMetablocks(Reader reader, int n) throws IOException {
279 for (int i = 0; i < n; i++) {
280 ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
281 ByteBuffer expected =
282 ByteBuffer.wrap(("something to test" + i).getBytes());
283 assertTrue("failed to match metadata", actual.compareTo(expected) == 0);
284 }
285 }
286
287 private void someReadingWithMetaBlock(Reader reader) throws IOException {
288 readNumMetablocks(reader, 10);
289 }
290
291 private void metablocks(final String compress) throws Exception {
292 if (cacheConf == null) cacheConf = new CacheConfig(conf);
293 Path mFile = new Path(ROOT_DIR, "meta.hfile");
294 FSDataOutputStream fout = createFSOutput(mFile);
295 Writer writer = HFile.getWriterFactory(conf, cacheConf)
296 .withOutputStream(fout)
297 .withBlockSize(minBlockSize)
298 .withCompression(compress)
299 .create();
300 someTestingWithMetaBlock(writer);
301 writer.close();
302 fout.close();
303 FSDataInputStream fin = fs.open(mFile);
304 Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile),
305 this.fs.getFileStatus(mFile).getLen(), cacheConf);
306 reader.loadFileInfo();
307
308 assertFalse(reader.getScanner(false, false).seekTo());
309 someReadingWithMetaBlock(reader);
310 fs.delete(mFile, true);
311 reader.close();
312 fin.close();
313 }
314
315
316 public void testMetaBlocks() throws Exception {
317 metablocks("none");
318 metablocks("gz");
319 }
320
321 public void testNullMetaBlocks() throws Exception {
322 if (cacheConf == null) cacheConf = new CacheConfig(conf);
323 for (Compression.Algorithm compressAlgo :
324 HBaseTestingUtility.COMPRESSION_ALGORITHMS) {
325 Path mFile = new Path(ROOT_DIR, "nometa_" + compressAlgo + ".hfile");
326 FSDataOutputStream fout = createFSOutput(mFile);
327 Writer writer = HFile.getWriterFactory(conf, cacheConf)
328 .withOutputStream(fout)
329 .withBlockSize(minBlockSize)
330 .withCompression(compressAlgo)
331 .create();
332 writer.append("foo".getBytes(), "value".getBytes());
333 writer.close();
334 fout.close();
335 Reader reader = HFile.createReader(fs, mFile, cacheConf);
336 reader.loadFileInfo();
337 assertNull(reader.getMetaBlock("non-existant", false));
338 }
339 }
340
341
342
343
344 public void testCompressionOrdinance() {
345 assertTrue(Compression.Algorithm.LZO.ordinal() == 0);
346 assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
347 assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
348 assertTrue(Compression.Algorithm.SNAPPY.ordinal() == 3);
349 assertTrue(Compression.Algorithm.LZ4.ordinal() == 4);
350 }
351
352 }
353