1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.codec.prefixtree.column;
20
21 import java.io.ByteArrayOutputStream;
22 import java.io.IOException;
23 import java.util.Collection;
24 import java.util.List;
25
26 import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta;
27 import org.apache.hadoop.hbase.codec.prefixtree.decode.column.ColumnReader;
28 import org.apache.hadoop.hbase.codec.prefixtree.encode.column.ColumnSectionWriter;
29 import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.Tokenizer;
30 import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.TokenizerNode;
31 import org.apache.hadoop.hbase.util.ByteRange;
32 import org.apache.hadoop.hbase.util.ByteRangeUtils;
33 import org.apache.hadoop.hbase.util.Bytes;
34 import org.apache.hadoop.hbase.util.byterange.impl.ByteRangeTreeSet;
35 import org.junit.Assert;
36 import org.junit.Test;
37 import org.junit.runner.RunWith;
38 import org.junit.runners.Parameterized;
39 import org.junit.runners.Parameterized.Parameters;
40
41 import com.google.common.collect.Lists;
42
43 @RunWith(Parameterized.class)
44 public class TestColumnBuilder {
45
46 @Parameters
47 public static Collection<Object[]> parameters() {
48 return new TestColumnData.InMemory().getAllAsObjectArray();
49 }
50
51
52
53 protected TestColumnData columns;
54 protected ByteRangeTreeSet columnSorter;
55 protected List<ByteRange> sortedUniqueColumns;
56 protected PrefixTreeBlockMeta blockMeta;
57 protected Tokenizer builder;
58 protected ColumnSectionWriter writer;
59 protected byte[] bytes;
60 protected byte[] buffer;
61 protected ColumnReader reader;
62
63
64
65 public TestColumnBuilder(TestColumnData columns) {
66 this.columns = columns;
67 List<ByteRange> inputs = columns.getInputs();
68 this.columnSorter = new ByteRangeTreeSet(inputs);
69 this.sortedUniqueColumns = columnSorter.compile().getSortedRanges();
70 List<byte[]> copies = ByteRangeUtils.copyToNewArrays(sortedUniqueColumns);
71 Assert.assertTrue(Bytes.isSorted(copies));
72 this.blockMeta = new PrefixTreeBlockMeta();
73 this.blockMeta.setNumMetaBytes(0);
74 this.blockMeta.setNumRowBytes(0);
75 this.builder = new Tokenizer();
76 }
77
78
79
80 @Test
81 public void testReaderRoundTrip() throws IOException {
82 for (int i = 0; i < sortedUniqueColumns.size(); ++i) {
83 ByteRange column = sortedUniqueColumns.get(i);
84 builder.addSorted(column);
85 }
86 List<byte[]> builderOutputArrays = builder.getArrays();
87 for (int i = 0; i < builderOutputArrays.size(); ++i) {
88 byte[] inputArray = sortedUniqueColumns.get(i).deepCopyToNewArray();
89 byte[] outputArray = builderOutputArrays.get(i);
90 boolean same = Bytes.equals(inputArray, outputArray);
91 Assert.assertTrue(same);
92 }
93 Assert.assertEquals(sortedUniqueColumns.size(), builderOutputArrays.size());
94
95 writer = new ColumnSectionWriter(blockMeta, builder, false);
96 ByteArrayOutputStream baos = new ByteArrayOutputStream();
97 writer.compile().writeBytes(baos);
98 bytes = baos.toByteArray();
99 buffer = new byte[blockMeta.getMaxQualifierLength()];
100 reader = new ColumnReader(buffer, false);
101 reader.initOnBlock(blockMeta, bytes);
102
103 List<TokenizerNode> builderNodes = Lists.newArrayList();
104 builder.appendNodes(builderNodes, true, true);
105 int i = 0;
106 for (TokenizerNode builderNode : builderNodes) {
107 if (!builderNode.hasOccurrences()) {
108 continue;
109 }
110 Assert.assertEquals(1, builderNode.getNumOccurrences());
111
112 int position = builderNode.getOutputArrayOffset();
113 byte[] output = reader.populateBuffer(position).copyBufferToNewArray();
114 boolean same = Bytes.equals(sortedUniqueColumns.get(i).deepCopyToNewArray(), output);
115 Assert.assertTrue(same);
116 ++i;
117 }
118 }
119
120 }