1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import java.io.DataInput;
23 import java.io.DataOutput;
24 import java.io.IOException;
25 import java.nio.ByteBuffer;
26 import java.util.Arrays;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.fs.FSDataInputStream;
31 import org.apache.hadoop.fs.FSDataOutputStream;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.HBaseTestCase;
34 import org.apache.hadoop.hbase.HBaseTestingUtility;
35 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
36 import org.apache.hadoop.hbase.io.hfile.HFile.BlockIndex;
37 import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
38 import org.apache.hadoop.hbase.io.hfile.HFile.Writer;
39 import org.apache.hadoop.hbase.util.Bytes;
40 import org.apache.hadoop.hbase.util.ClassSize;
41 import org.apache.hadoop.io.Writable;
42
43
44
45
46
47
48
49
50
51 public class TestHFile extends HBaseTestCase {
52 static final Log LOG = LogFactory.getLog(TestHFile.class);
53
54 private static String ROOT_DIR =
55 HBaseTestingUtility.getTestDir("TestHFile").toString();
56 private final int minBlockSize = 512;
57 private static String localFormatter = "%010d";
58
59
60
61
62
63
64 public void testEmptyHFile() throws IOException {
65 Path f = new Path(ROOT_DIR, getName());
66 Writer w = new Writer(this.fs, f);
67 w.close();
68 Reader r = new Reader(fs, f, null, false);
69 r.loadFileInfo();
70 assertNull(r.getFirstKey());
71 assertNull(r.getLastKey());
72 }
73
74
75
76 private int writeSomeRecords(Writer writer, int start, int n)
77 throws IOException {
78 String value = "value";
79 for (int i = start; i < (start + n); i++) {
80 String key = String.format(localFormatter, Integer.valueOf(i));
81 writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key));
82 }
83 return (start + n);
84 }
85
86 private void readAllRecords(HFileScanner scanner) throws IOException {
87 readAndCheckbytes(scanner, 0, 100);
88 }
89
90
91 private int readAndCheckbytes(HFileScanner scanner, int start, int n)
92 throws IOException {
93 String value = "value";
94 int i = start;
95 for (; i < (start + n); i++) {
96 ByteBuffer key = scanner.getKey();
97 ByteBuffer val = scanner.getValue();
98 String keyStr = String.format(localFormatter, Integer.valueOf(i));
99 String valStr = value + keyStr;
100 byte [] keyBytes = Bytes.toBytes(key);
101 assertTrue("bytes for keys do not match " + keyStr + " " +
102 Bytes.toString(Bytes.toBytes(key)),
103 Arrays.equals(Bytes.toBytes(keyStr), keyBytes));
104 byte [] valBytes = Bytes.toBytes(val);
105 assertTrue("bytes for vals do not match " + valStr + " " +
106 Bytes.toString(valBytes),
107 Arrays.equals(Bytes.toBytes(valStr), valBytes));
108 if (!scanner.next()) {
109 break;
110 }
111 }
112 assertEquals(i, start + n - 1);
113 return (start + n);
114 }
115
116 private byte[] getSomeKey(int rowId) {
117 return String.format(localFormatter, Integer.valueOf(rowId)).getBytes();
118 }
119
120 private void writeRecords(Writer writer) throws IOException {
121 writeSomeRecords(writer, 0, 100);
122 writer.close();
123 }
124
125 private FSDataOutputStream createFSOutput(Path name) throws IOException {
126 if (fs.exists(name)) fs.delete(name, true);
127 FSDataOutputStream fout = fs.create(name);
128 return fout;
129 }
130
131
132
133
134 void basicWithSomeCodec(String codec) throws IOException {
135 Path ncTFile = new Path(ROOT_DIR, "basic.hfile");
136 FSDataOutputStream fout = createFSOutput(ncTFile);
137 Writer writer = new Writer(fout, minBlockSize,
138 Compression.getCompressionAlgorithmByName(codec), null);
139 LOG.info(writer);
140 writeRecords(writer);
141 fout.close();
142 FSDataInputStream fin = fs.open(ncTFile);
143 Reader reader = new Reader(fs.open(ncTFile),
144 fs.getFileStatus(ncTFile).getLen(), null, false);
145
146 reader.loadFileInfo();
147
148 HFileScanner scanner = reader.getScanner(true, false);
149
150 scanner.seekTo();
151 readAllRecords(scanner);
152 scanner.seekTo(getSomeKey(50));
153 assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0);
154
155 ByteBuffer readKey = scanner.getKey();
156 assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50),
157 Bytes.toBytes(readKey)));
158
159 scanner.seekTo(new byte[0]);
160 ByteBuffer val1 = scanner.getValue();
161 scanner.seekTo(new byte[0]);
162 ByteBuffer val2 = scanner.getValue();
163 assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
164
165 reader.close();
166 fin.close();
167 fs.delete(ncTFile, true);
168 }
169
170 public void testTFileFeatures() throws IOException {
171 basicWithSomeCodec("none");
172 basicWithSomeCodec("gz");
173 }
174
175 private void writeNumMetablocks(Writer writer, int n) {
176 for (int i = 0; i < n; i++) {
177 writer.appendMetaBlock("HFileMeta" + i, new Writable() {
178 private int val;
179 public Writable setVal(int val) { this.val = val; return this; }
180
181 @Override
182 public void write(DataOutput out) throws IOException {
183 out.write(("something to test" + val).getBytes());
184 }
185
186 @Override
187 public void readFields(DataInput in) throws IOException { }
188 }.setVal(i));
189 }
190 }
191
192 private void someTestingWithMetaBlock(Writer writer) {
193 writeNumMetablocks(writer, 10);
194 }
195
196 private void readNumMetablocks(Reader reader, int n) throws IOException {
197 for (int i = 0; i < n; i++) {
198 ByteBuffer actual = reader.getMetaBlock("HFileMeta" + i, false);
199 ByteBuffer expected =
200 ByteBuffer.wrap(("something to test" + i).getBytes());
201 assertTrue("failed to match metadata", actual.compareTo(expected) == 0);
202 }
203 }
204
205 private void someReadingWithMetaBlock(Reader reader) throws IOException {
206 readNumMetablocks(reader, 10);
207 }
208
209 private void metablocks(final String compress) throws Exception {
210 Path mFile = new Path(ROOT_DIR, "meta.hfile");
211 FSDataOutputStream fout = createFSOutput(mFile);
212 Writer writer = new Writer(fout, minBlockSize,
213 Compression.getCompressionAlgorithmByName(compress), null);
214 someTestingWithMetaBlock(writer);
215 writer.close();
216 fout.close();
217 FSDataInputStream fin = fs.open(mFile);
218 Reader reader = new Reader(fs.open(mFile), this.fs.getFileStatus(mFile)
219 .getLen(), null, false);
220 reader.loadFileInfo();
221
222 assertFalse(reader.getScanner(false, false).seekTo());
223 someReadingWithMetaBlock(reader);
224 fs.delete(mFile, true);
225 reader.close();
226 fin.close();
227 }
228
229
230 public void testMetaBlocks() throws Exception {
231 metablocks("none");
232 metablocks("gz");
233 }
234
235 public void testNullMetaBlocks() throws Exception {
236 Path mFile = new Path(ROOT_DIR, "nometa.hfile");
237 FSDataOutputStream fout = createFSOutput(mFile);
238 Writer writer = new Writer(fout, minBlockSize,
239 Compression.Algorithm.NONE, null);
240 writer.append("foo".getBytes(), "value".getBytes());
241 writer.close();
242 fout.close();
243 Reader reader = new Reader(fs, mFile, null, false);
244 reader.loadFileInfo();
245 assertNull(reader.getMetaBlock("non-existant", false));
246 }
247
248
249
250
251 public void testCompressionOrdinance() {
252
253 assertTrue(Compression.Algorithm.GZ.ordinal() == 1);
254 assertTrue(Compression.Algorithm.NONE.ordinal() == 2);
255 }
256
257
258 public void testComparator() throws IOException {
259 Path mFile = new Path(ROOT_DIR, "meta.tfile");
260 FSDataOutputStream fout = createFSOutput(mFile);
261 Writer writer = new Writer(fout, minBlockSize, (Compression.Algorithm) null,
262 new KeyComparator() {
263 @Override
264 public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2,
265 int l2) {
266 return -Bytes.compareTo(b1, s1, l1, b2, s2, l2);
267
268 }
269 @Override
270 public int compare(byte[] o1, byte[] o2) {
271 return compare(o1, 0, o1.length, o2, 0, o2.length);
272 }
273 });
274 writer.append("3".getBytes(), "0".getBytes());
275 writer.append("2".getBytes(), "0".getBytes());
276 writer.append("1".getBytes(), "0".getBytes());
277 writer.close();
278 }
279
280
281
282
283 @SuppressWarnings("unchecked")
284 public void testHeapSizeForBlockIndex() throws IOException{
285 Class cl = null;
286 long expected = 0L;
287 long actual = 0L;
288
289 cl = BlockIndex.class;
290 expected = ClassSize.estimateBase(cl, false);
291 BlockIndex bi = new BlockIndex(Bytes.BYTES_RAWCOMPARATOR);
292 actual = bi.heapSize();
293
294
295
296 expected -= ClassSize.align(3 * ClassSize.ARRAY);
297 if(expected != actual) {
298 ClassSize.estimateBase(cl, true);
299 assertEquals(expected, actual);
300 }
301 }
302
303 }