View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree.column;
20  
21  import java.io.ByteArrayOutputStream;
22  import java.io.IOException;
23  import java.util.Collection;
24  import java.util.List;
25  
26  import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta;
27  import org.apache.hadoop.hbase.codec.prefixtree.decode.column.ColumnReader;
28  import org.apache.hadoop.hbase.codec.prefixtree.encode.column.ColumnSectionWriter;
29  import org.apache.hadoop.hbase.codec.prefixtree.encode.other.ColumnNodeType;
30  import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.Tokenizer;
31  import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.TokenizerNode;
32  import org.apache.hadoop.hbase.util.ByteRange;
33  import org.apache.hadoop.hbase.util.ByteRangeUtils;
34  import org.apache.hadoop.hbase.util.Bytes;
35  import org.apache.hadoop.hbase.util.byterange.impl.ByteRangeTreeSet;
36  import org.junit.Assert;
37  import org.junit.Test;
38  import org.junit.runner.RunWith;
39  import org.junit.runners.Parameterized;
40  import org.junit.runners.Parameterized.Parameters;
41  
42  import com.google.common.collect.Lists;
43  
44  @RunWith(Parameterized.class)
45  public class TestColumnBuilder {
46  
47    @Parameters
48    public static Collection<Object[]> parameters() {
49      return new TestColumnData.InMemory().getAllAsObjectArray();
50    }
51  
52    /*********** fields **********************************/
53  
54    protected TestColumnData columns;
55    protected ByteRangeTreeSet columnSorter;
56    protected List<ByteRange> sortedUniqueColumns;
57    protected PrefixTreeBlockMeta blockMeta;
58    protected Tokenizer builder;
59    protected ColumnSectionWriter writer;
60    protected byte[] bytes;
61    protected byte[] buffer;
62    protected ColumnReader reader;
63  
64    /*************** construct ****************************/
65  
66    public TestColumnBuilder(TestColumnData columns) {
67      this.columns = columns;
68      List<ByteRange> inputs = columns.getInputs();
69      this.columnSorter = new ByteRangeTreeSet(inputs);
70      this.sortedUniqueColumns = columnSorter.compile().getSortedRanges();
71      List<byte[]> copies = ByteRangeUtils.copyToNewArrays(sortedUniqueColumns);
72      Assert.assertTrue(Bytes.isSorted(copies));
73      this.blockMeta = new PrefixTreeBlockMeta();
74      this.blockMeta.setNumMetaBytes(0);
75      this.blockMeta.setNumRowBytes(0);
76      this.builder = new Tokenizer();
77    }
78  
79    /************* methods ********************************/
80  
81    @Test
82    public void testReaderRoundTrip() throws IOException {
83      for (int i = 0; i < sortedUniqueColumns.size(); ++i) {
84        ByteRange column = sortedUniqueColumns.get(i);
85        builder.addSorted(column);
86      }
87      List<byte[]> builderOutputArrays = builder.getArrays();
88      for (int i = 0; i < builderOutputArrays.size(); ++i) {
89        byte[] inputArray = sortedUniqueColumns.get(i).deepCopyToNewArray();
90        byte[] outputArray = builderOutputArrays.get(i);
91        boolean same = Bytes.equals(inputArray, outputArray);
92        Assert.assertTrue(same);
93      }
94      Assert.assertEquals(sortedUniqueColumns.size(), builderOutputArrays.size());
95  
96      writer = new ColumnSectionWriter(blockMeta, builder, ColumnNodeType.QUALIFIER);
97      ByteArrayOutputStream baos = new ByteArrayOutputStream();
98      writer.compile().writeBytes(baos);
99      bytes = baos.toByteArray();
100     buffer = new byte[blockMeta.getMaxQualifierLength()];
101     reader = new ColumnReader(buffer, ColumnNodeType.QUALIFIER);
102     reader.initOnBlock(blockMeta, bytes);
103 
104     List<TokenizerNode> builderNodes = Lists.newArrayList();
105     builder.appendNodes(builderNodes, true, true);
106     int i = 0;
107     for (TokenizerNode builderNode : builderNodes) {
108       if (!builderNode.hasOccurrences()) {
109         continue;
110       }
111       Assert.assertEquals(1, builderNode.getNumOccurrences());// we de-duped before adding to
112                                                               // builder
113       int position = builderNode.getOutputArrayOffset();
114       byte[] output = reader.populateBuffer(position).copyBufferToNewArray();
115       boolean same = Bytes.equals(sortedUniqueColumns.get(i).deepCopyToNewArray(), output);
116       Assert.assertTrue(same);
117       ++i;
118     }
119   }
120 
121 }