1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.io.hfile;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.ByteArrayInputStream;
27 import java.io.DataInputStream;
28 import java.io.IOException;
29 import java.nio.ByteBuffer;
30 import java.util.ArrayList;
31 import java.util.List;
32 import java.util.Random;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FSDataInputStream;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.KeyValue;
42 import org.apache.hadoop.hbase.SmallTests;
43 import org.apache.hadoop.hbase.io.compress.Compression;
44 import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
45 import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.Writables;
48 import org.apache.hadoop.io.RawComparator;
49 import org.apache.hadoop.io.Text;
50 import org.apache.hadoop.io.WritableUtils;
51 import org.junit.Before;
52 import org.junit.Test;
53 import org.junit.experimental.categories.Category;
54
55
56
57
58
59 @Category(SmallTests.class)
60 public class TestHFileWriterV2 {
61
62 private static final Log LOG = LogFactory.getLog(TestHFileWriterV2.class);
63
64 private static final HBaseTestingUtility TEST_UTIL =
65 new HBaseTestingUtility();
66
67 private Configuration conf;
68 private FileSystem fs;
69
70 @Before
71 public void setUp() throws IOException {
72 conf = TEST_UTIL.getConfiguration();
73 fs = FileSystem.get(conf);
74 }
75
76 @Test
77 public void testHFileFormatV2() throws IOException {
78 Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "testHFileFormatV2");
79 final Compression.Algorithm compressAlgo = Compression.Algorithm.GZ;
80 final int entryCount = 10000;
81 writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, false);
82 }
83
84 @Test
85 public void testMidKeyInHFile() throws IOException{
86 Path hfilePath = new Path(TEST_UTIL.getDataTestDir(),
87 "testMidKeyInHFile");
88 Compression.Algorithm compressAlgo = Compression.Algorithm.NONE;
89 int entryCount = 50000;
90 writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, true);
91 }
92
93 private void writeDataAndReadFromHFile(Path hfilePath,
94 Algorithm compressAlgo, int entryCount, boolean findMidKey) throws IOException {
95
96 HFileWriterV2 writer = (HFileWriterV2)
97 new HFileWriterV2.WriterFactoryV2(conf, new CacheConfig(conf))
98 .withPath(fs, hfilePath)
99 .withBlockSize(4096)
100 .withCompression(compressAlgo)
101 .withComparator(KeyValue.KEY_COMPARATOR)
102 .create();
103
104 Random rand = new Random(9713312);
105 List<KeyValue> keyValues = new ArrayList<KeyValue>(entryCount);
106
107 for (int i = 0; i < entryCount; ++i) {
108 byte[] keyBytes = randomOrderedKey(rand, i);
109
110
111 byte[] valueBytes = randomValue(rand);
112 KeyValue keyValue = new KeyValue(keyBytes, null, null, valueBytes);
113 writer.append(keyValue);
114 keyValues.add(keyValue);
115 }
116
117
118
119 writer.appendMetaBlock("CAPITAL_OF_USA", new Text("Washington, D.C."));
120 writer.appendMetaBlock("CAPITAL_OF_RUSSIA", new Text("Moscow"));
121 writer.appendMetaBlock("CAPITAL_OF_FRANCE", new Text("Paris"));
122
123 writer.close();
124
125
126 FSDataInputStream fsdis = fs.open(hfilePath);
127
128
129
130
131 long fileSize = fs.getFileStatus(hfilePath).getLen();
132 FixedFileTrailer trailer =
133 FixedFileTrailer.readFromStream(fsdis, fileSize);
134
135 assertEquals(2, trailer.getMajorVersion());
136 assertEquals(entryCount, trailer.getEntryCount());
137
138 HFileBlock.FSReader blockReader =
139 new HFileBlock.FSReaderV2(fsdis, compressAlgo, fileSize);
140
141 RawComparator<byte []> comparator = trailer.createComparator();
142 HFileBlockIndex.BlockIndexReader dataBlockIndexReader =
143 new HFileBlockIndex.BlockIndexReader(comparator,
144 trailer.getNumDataIndexLevels());
145 HFileBlockIndex.BlockIndexReader metaBlockIndexReader =
146 new HFileBlockIndex.BlockIndexReader(
147 Bytes.BYTES_RAWCOMPARATOR, 1);
148
149 HFileBlock.BlockIterator blockIter = blockReader.blockRange(
150 trailer.getLoadOnOpenDataOffset(),
151 fileSize - trailer.getTrailerSize());
152
153
154 dataBlockIndexReader.readMultiLevelIndexRoot(
155 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX),
156 trailer.getDataIndexCount());
157
158 if (findMidKey) {
159 byte[] midkey = dataBlockIndexReader.midkey();
160 assertNotNull("Midkey should not be null", midkey);
161 }
162
163
164 metaBlockIndexReader.readRootIndex(
165 blockIter.nextBlockWithBlockType(BlockType.ROOT_INDEX).getByteStream(),
166 trailer.getMetaIndexCount());
167
168 FileInfo fileInfo = new FileInfo();
169 fileInfo.read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
170 byte [] keyValueFormatVersion = fileInfo.get(
171 HFileWriterV2.KEY_VALUE_VERSION);
172 boolean includeMemstoreTS = keyValueFormatVersion != null &&
173 Bytes.toInt(keyValueFormatVersion) > 0;
174
175
176 int entriesRead = 0;
177 int blocksRead = 0;
178 long memstoreTS = 0;
179
180
181 fsdis.seek(0);
182 long curBlockPos = 0;
183 while (curBlockPos <= trailer.getLastDataBlockOffset()) {
184 HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false);
185 assertEquals(BlockType.DATA, block.getBlockType());
186 ByteBuffer buf = block.getBufferWithoutHeader();
187 while (buf.hasRemaining()) {
188 int keyLen = buf.getInt();
189 int valueLen = buf.getInt();
190
191 byte[] key = new byte[keyLen];
192 buf.get(key);
193
194 byte[] value = new byte[valueLen];
195 buf.get(value);
196
197 if (includeMemstoreTS) {
198 ByteArrayInputStream byte_input = new ByteArrayInputStream(buf.array(),
199 buf.arrayOffset() + buf.position(), buf.remaining());
200 DataInputStream data_input = new DataInputStream(byte_input);
201
202 memstoreTS = WritableUtils.readVLong(data_input);
203 buf.position(buf.position() + WritableUtils.getVIntSize(memstoreTS));
204 }
205
206
207 assertTrue(Bytes.compareTo(key, keyValues.get(entriesRead).getKey()) == 0);
208 assertTrue(Bytes.compareTo(value, keyValues.get(entriesRead).getValue()) == 0);
209
210 ++entriesRead;
211 }
212 ++blocksRead;
213 curBlockPos += block.getOnDiskSizeWithHeader();
214 }
215 LOG.info("Finished reading: entries=" + entriesRead + ", blocksRead="
216 + blocksRead);
217 assertEquals(entryCount, entriesRead);
218
219
220
221
222
223 int metaCounter = 0;
224 while (fsdis.getPos() < trailer.getLoadOnOpenDataOffset()) {
225 LOG.info("Current offset: " + fsdis.getPos() + ", scanning until " +
226 trailer.getLoadOnOpenDataOffset());
227 HFileBlock block = blockReader.readBlockData(curBlockPos, -1, -1, false);
228 assertEquals(BlockType.META, block.getBlockType());
229 Text t = new Text();
230 ByteBuffer buf = block.getBufferWithoutHeader();
231 if (Writables.getWritable(buf.array(), buf.arrayOffset(), buf.limit(), t) == null) {
232 throw new IOException("Failed to deserialize block " + this + " into a " + t.getClass().getSimpleName());
233 }
234 Text expectedText =
235 (metaCounter == 0 ? new Text("Paris") : metaCounter == 1 ? new Text(
236 "Moscow") : new Text("Washington, D.C."));
237 assertEquals(expectedText, t);
238 LOG.info("Read meta block data: " + t);
239 ++metaCounter;
240 curBlockPos += block.getOnDiskSizeWithHeader();
241 }
242
243 fsdis.close();
244 }
245
246
247
248
249 private static final String COLUMN_FAMILY_NAME = "_-myColumnFamily-_";
250 private static final int MIN_ROW_OR_QUALIFIER_LENGTH = 64;
251 private static final int MAX_ROW_OR_QUALIFIER_LENGTH = 128;
252
253
254
255
256
257
258
259
260
261
262
263 public static byte[] randomOrderedKey(Random rand, int i) {
264 StringBuilder k = new StringBuilder();
265
266
267 for (int bitIndex = 31; bitIndex >= 0; --bitIndex) {
268 if ((i & (1 << bitIndex)) == 0)
269 k.append("a");
270 else
271 k.append("b");
272 }
273
274
275 for (int j = 0; j < rand.nextInt(50); ++j)
276 k.append(randomReadableChar(rand));
277
278 byte[] keyBytes = k.toString().getBytes();
279 return keyBytes;
280 }
281
282 public static byte[] randomValue(Random rand) {
283 StringBuilder v = new StringBuilder();
284 for (int j = 0; j < 1 + rand.nextInt(2000); ++j) {
285 v.append((char) (32 + rand.nextInt(95)));
286 }
287
288 byte[] valueBytes = v.toString().getBytes();
289 return valueBytes;
290 }
291
292 public static final char randomReadableChar(Random rand) {
293 int i = rand.nextInt(26 * 2 + 10 + 1);
294 if (i < 26)
295 return (char) ('A' + i);
296 i -= 26;
297
298 if (i < 26)
299 return (char) ('a' + i);
300 i -= 26;
301
302 if (i < 10)
303 return (char) ('0' + i);
304 i -= 10;
305
306 assert i == 0;
307 return '_';
308 }
309
310 public static byte[] randomRowOrQualifier(Random rand) {
311 StringBuilder field = new StringBuilder();
312 int fieldLen = MIN_ROW_OR_QUALIFIER_LENGTH
313 + rand.nextInt(MAX_ROW_OR_QUALIFIER_LENGTH
314 - MIN_ROW_OR_QUALIFIER_LENGTH + 1);
315 for (int i = 0; i < fieldLen; ++i)
316 field.append(randomReadableChar(rand));
317 return field.toString().getBytes();
318 }
319
320 public static KeyValue randomKeyValue(Random rand) {
321 return new KeyValue(randomRowOrQualifier(rand),
322 COLUMN_FAMILY_NAME.getBytes(), randomRowOrQualifier(rand),
323 randomValue(rand));
324 }
325
326
327 }
328