1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.ByteArrayInputStream;
27 import java.io.DataInputStream;
28 import java.io.IOException;
29 import java.nio.ByteBuffer;
30 import java.util.ArrayList;
31 import java.util.List;
32 import java.util.Random;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataInputStream;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.KeyValue;
42 import org.apache.hadoop.hbase.KeyValue.KVComparator;
43 import org.apache.hadoop.hbase.SmallTests;
44 import org.apache.hadoop.hbase.io.compress.Compression;
45 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
46 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.apache.hadoop.hbase.util.Writables;
49 import org.apache.hadoop.io.Text;
50 import org.apache.hadoop.io.WritableUtils;
51 import org.junit.Before;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55
56
57
58
59 @Category(SmallTests.class)
60 public class TestHFileWriterV2 {
61
62 private static final Log LOG = LogFactory.getLog(TestHFileWriterV2.class);
63
64 private static final HBaseTestingUtility TEST_UTIL =
65 new HBaseTestingUtility();
66
67 private Configuration conf;
68 private FileSystem fs;
69
70 @Before
71 public void setUp() throws IOException {
72 conf = TEST_UTIL.getConfiguration();
73 fs = FileSystem.get(conf);
74 }
75
76 @Test
77 public void testHFileFormatV2() throws IOException {
78 Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "testHFileFormatV2");
79 final Compression.Algorithm compressAlgo = Compression.Algorithm.GZ;
80 final int entryCount = 10000;
81 writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, false);
82 }
83
84 @Test
85 public void testMidKeyInHFile() throws IOException{
86 Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
87 "testMidKeyInHFile");
88 Compression.Algorithm compressAlgo = Compression.Algorithm.NONE;
89 int entryCount = 50000;
90 writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, true);
91 }
92
93 private void writeDataAndReadFromHFile(Path hfilePath,
94 Algorithm compressAlgo, int entryCount, boolean findMidKey) throws IOException {
95
96 HFileWriterV2 writer = (HFileWriterV2)
97 new HFileWriterV2.WriterFactoryV2(conf, new CacheConfig(conf))
98 .withPath(fs, hfilePath)
99 .withBlockSize(4096)
100 .withCompression(compressAlgo)
101 .create();
102
103 Random rand = new Random(9713312);
104 List<KeyValue> keyValues = new ArrayList<KeyValue>(entryCount);
105
106 for (int i = 0; i < entryCount; ++i) {
107 byte[] keyBytes = randomOrderedKey(rand, i);
108
109
110 byte[] valueBytes = randomValue(rand);
111 KeyValue keyValue = new KeyValue(keyBytes, null, null, valueBytes);
112 writer.append(keyValue);
113 keyValues.add(keyValue);
114 }
115
116
117
118 writer.appendMetaBlock("CAPITAL_OF_USA", new Text("Washington, D.C."));
119 writer.appendMetaBlock("CAPITAL_OF_RUSSIA", new Text("Moscow"));
120 writer.appendMetaBlock("CAPITAL_OF_FRANCE", new Text("Paris"));
121
122 writer.close();
123
124
125 FSDataInputStream fsdis = fs.open(hfilePath);
126
127
128
129
130 long fileSize = fs.getFileStatus(hfilePath).getLen();
131 FixedFileTrailer trailer =
132 FixedFileTrailer.readFromStream(fsdis, fileSize);
133
134 assertEquals(2, trailer.getMajorVersion());
135 assertEquals(entryCount, trailer.getEntryCount());
136
137 HFileBlock.FSReader blockReader =
138 new HFileBlock.FSReaderV2(fsdis, compressAlgo, fileSize);
139
140 KVComparator comparator = trailer.createComparator();
141 HFileBlockIndex.BlockIndexReader dataBlockIndexReader =
142 new HFileBlockIndex.BlockIndexReader(comparator,
143 trailer.getNumDataIndexLevels());
144 HFileBlockIndex.BlockIndexReader metaBlockIndexReader =
145 new HFileBlockIndex.BlockIndexReader(
146 KeyValue.RAW_COMPARATOR, 1);
147
148 HFileBlock.BlockIterator blockIter = blockReader.blockRange(
149 trailer.getLoadOnOpenDataOffset(),
150 fileSize - trailer.getTrailerSize());
151
152
153 dataBlockIndexReader.readMultiLevelIndexRoot(
154 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
155 trailer.getDataIndexCount());
156
157 if (findMidKey) {
158 byte[] midkey = dataBlockIndexReader.midkey();
159 assertNotNull("Midkey should not be null", midkey);
160 }
161
162
163 metaBlockIndexReader.readRootIndex(
164 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX).getByteStream(),
165 trailer.getMetaIndexCount());
166
167 FileInfo fileInfo = new FileInfo();
168 fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
169 byte [] keyValueFormatVersion = fileInfo.get(
170 HFileWriterV2.KEY_VALUE_VERSION);
171 boolean includeMemstoreTS = keyValueFormatVersion != null &&
172 Bytes.toInt(keyValueFormatVersion) > 0;
173
174
175 int entriesRead = 0;
176 int blocksRead = 0;
177 long memstoreTS = 0;
178
179
180 fsdis.seek(0);
181 long curBlockPos = 0;
182 while (curBlockPos <= trailer.getLastDataBlockOffset()) {
183 HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false);
184 assertEquals(BlockType.DATA, block.getBlockType());
185 ByteBuffer buf = block.getBufferWithoutHeader();
186 while (buf.hasRemaining()) {
187 int keyLen = buf.getInt();
188 int valueLen = buf.getInt();
189
190 byte[] key = new byte[keyLen];
191 buf.get(key);
192
193 byte[] value = new byte[valueLen];
194 buf.get(value);
195
196 if (includeMemstoreTS) {
197 ByteArrayInputStream byte_input = new ByteArrayInputStream(buf.array(),
198 buf.arrayOffset() + buf.position(), buf.remaining());
199 DataInputStream data_input = new DataInputStream(byte_input);
200
201 memstoreTS = WritableUtils.readVLong(data_input);
202 buf.position(buf.position() + WritableUtils.getVIntSize(memstoreTS));
203 }
204
205
206 assertTrue(Bytes.compareTo(key, keyValues.get(entriesRead).getKey()) == 0);
207 assertTrue(Bytes.compareTo(value, keyValues.get(entriesRead).getValue()) == 0);
208
209 ++entriesRead;
210 }
211 ++blocksRead;
212 curBlockPos += block.getOnDiskSizeWithHeader();
213 }
214 LOG.info("Finished reading: entries=" + entriesRead + ", blocksRead="
215 + blocksRead);
216 assertEquals(entryCount, entriesRead);
217
218
219
220
221
222 int metaCounter = 0;
223 while (fsdis.getPos() < trailer.getLoadOnOpenDataOffset()) {
224 LOG.info("Current offset: " + fsdis.getPos() + ", scanning until " +
225 trailer.getLoadOnOpenDataOffset());
226 HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false);
227 assertEquals(BlockType.META, block.getBlockType());
228 Text t = new Text();
229 ByteBuffer buf = block.getBufferWithoutHeader();
230 if (Writables.getWritable(buf.array(), buf.arrayOffset(), buf.limit(), t) == null) {
231 throw new IOException("Failed to deserialize block " + this + " into a " + t.getClass().getSimpleName());
232 }
233 Text expectedText =
234 (metaCounter == 0 ? new Text("Paris") : metaCounter == 1 ? new Text(
235 "Moscow") : new Text("Washington, D.C."));
236 assertEquals(expectedText, t);
237 LOG.info("Read meta block data: " + t);
238 ++metaCounter;
239 curBlockPos += block.getOnDiskSizeWithHeader();
240 }
241
242 fsdis.close();
243 }
244
245
246
247
248 private static final String COLUMN_FAMILY_NAME = "_-myColumnFamily-_";
249 private static final int MIN_ROW_OR_QUALIFIER_LENGTH = 64;
250 private static final int MAX_ROW_OR_QUALIFIER_LENGTH = 128;
251
252
253
254
255
256
257
258
259
260
261
262 public static byte[] randomOrderedKey(Random rand, int i) {
263 StringBuilder k = new StringBuilder();
264
265
266 for (int bitIndex = 31; bitIndex >= 0; --bitIndex) {
267 if ((i & (1 << bitIndex)) == 0)
268 k.append("a");
269 else
270 k.append("b");
271 }
272
273
274 for (int j = 0; j < rand.nextInt(50); ++j)
275 k.append(randomReadableChar(rand));
276
277 byte[] keyBytes = k.toString().getBytes();
278 return keyBytes;
279 }
280
281 public static byte[] randomValue(Random rand) {
282 StringBuilder v = new StringBuilder();
283 for (int j = 0; j < 1 + rand.nextInt(2000); ++j) {
284 v.append((char) (32 + rand.nextInt(95)));
285 }
286
287 byte[] valueBytes = v.toString().getBytes();
288 return valueBytes;
289 }
290
291 public static final char randomReadableChar(Random rand) {
292 int i = rand.nextInt(26 * 2 + 10 + 1);
293 if (i < 26)
294 return (char) ('A' + i);
295 i -= 26;
296
297 if (i < 26)
298 return (char) ('a' + i);
299 i -= 26;
300
301 if (i < 10)
302 return (char) ('0' + i);
303 i -= 10;
304
305 assert i == 0;
306 return '_';
307 }
308
309 public static byte[] randomRowOrQualifier(Random rand) {
310 StringBuilder field = new StringBuilder();
311 int fieldLen = MIN_ROW_OR_QUALIFIER_LENGTH
312 + rand.nextInt(MAX_ROW_OR_QUALIFIER_LENGTH
313 - MIN_ROW_OR_QUALIFIER_LENGTH + 1);
314 for (int i = 0; i < fieldLen; ++i)
315 field.append(randomReadableChar(rand));
316 return field.toString().getBytes();
317 }
318
319 public static KeyValue randomKeyValue(Random rand) {
320 return new KeyValue(randomRowOrQualifier(rand),
321 COLUMN_FAMILY_NAME.getBytes(), randomRowOrQualifier(rand),
322 randomValue(rand));
323 }
324
325
326 }
327