View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree.column;
20  
21  import java.io.ByteArrayOutputStream;
22  import java.io.IOException;
23  import java.util.Collection;
24  import java.util.List;
25  
26  import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta;
27  import org.apache.hadoop.hbase.codec.prefixtree.decode.column.ColumnReader;
28  import org.apache.hadoop.hbase.codec.prefixtree.encode.column.ColumnSectionWriter;
29  import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.Tokenizer;
30  import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.TokenizerNode;
31  import org.apache.hadoop.hbase.util.ByteRange;
32  import org.apache.hadoop.hbase.util.ByteRangeUtils;
33  import org.apache.hadoop.hbase.util.Bytes;
34  import org.apache.hadoop.hbase.util.byterange.impl.ByteRangeTreeSet;
35  import org.junit.Assert;
36  import org.junit.Test;
37  import org.junit.runner.RunWith;
38  import org.junit.runners.Parameterized;
39  import org.junit.runners.Parameterized.Parameters;
40  
41  import com.google.common.collect.Lists;
42  
43  @RunWith(Parameterized.class)
44  public class TestColumnBuilder {
45  
46    @Parameters
47    public static Collection<Object[]> parameters() {
48      return new TestColumnData.InMemory().getAllAsObjectArray();
49    }
50  
51    /*********** fields **********************************/
52  
53    protected TestColumnData columns;
54    protected ByteRangeTreeSet columnSorter;
55    protected List<ByteRange> sortedUniqueColumns;
56    protected PrefixTreeBlockMeta blockMeta;
57    protected Tokenizer builder;
58    protected ColumnSectionWriter writer;
59    protected byte[] bytes;
60    protected byte[] buffer;
61    protected ColumnReader reader;
62  
63    /*************** construct ****************************/
64  
65    public TestColumnBuilder(TestColumnData columns) {
66      this.columns = columns;
67      List<ByteRange> inputs = columns.getInputs();
68      this.columnSorter = new ByteRangeTreeSet(inputs);
69      this.sortedUniqueColumns = columnSorter.compile().getSortedRanges();
70      List<byte[]> copies = ByteRangeUtils.copyToNewArrays(sortedUniqueColumns);
71      Assert.assertTrue(Bytes.isSorted(copies));
72      this.blockMeta = new PrefixTreeBlockMeta();
73      this.blockMeta.setNumMetaBytes(0);
74      this.blockMeta.setNumRowBytes(0);
75      this.builder = new Tokenizer();
76    }
77  
78    /************* methods ********************************/
79  
80    @Test
81    public void testReaderRoundTrip() throws IOException {
82      for (int i = 0; i < sortedUniqueColumns.size(); ++i) {
83        ByteRange column = sortedUniqueColumns.get(i);
84        builder.addSorted(column);
85      }
86      List<byte[]> builderOutputArrays = builder.getArrays();
87      for (int i = 0; i < builderOutputArrays.size(); ++i) {
88        byte[] inputArray = sortedUniqueColumns.get(i).deepCopyToNewArray();
89        byte[] outputArray = builderOutputArrays.get(i);
90        boolean same = Bytes.equals(inputArray, outputArray);
91        Assert.assertTrue(same);
92      }
93      Assert.assertEquals(sortedUniqueColumns.size(), builderOutputArrays.size());
94  
95      writer = new ColumnSectionWriter(blockMeta, builder, false);
96      ByteArrayOutputStream baos = new ByteArrayOutputStream();
97      writer.compile().writeBytes(baos);
98      bytes = baos.toByteArray();
99      buffer = new byte[blockMeta.getMaxQualifierLength()];
100     reader = new ColumnReader(buffer, false);
101     reader.initOnBlock(blockMeta, bytes);
102 
103     List<TokenizerNode> builderNodes = Lists.newArrayList();
104     builder.appendNodes(builderNodes, true, true);
105     int i = 0;
106     for (TokenizerNode builderNode : builderNodes) {
107       if (!builderNode.hasOccurrences()) {
108         continue;
109       }
110       Assert.assertEquals(1, builderNode.getNumOccurrences());// we de-duped before adding to
111                                                               // builder
112       int position = builderNode.getOutputArrayOffset();
113       byte[] output = reader.populateBuffer(position).copyBufferToNewArray();
114       boolean same = Bytes.equals(sortedUniqueColumns.get(i).deepCopyToNewArray(), output);
115       Assert.assertTrue(same);
116       ++i;
117     }
118   }
119 
120 }