View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree.encode.row;
20  
21  import java.io.IOException;
22  import java.io.OutputStream;
23  import java.util.ArrayList;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.classification.InterfaceAudience;
28  import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta;
29  import org.apache.hadoop.hbase.codec.prefixtree.encode.PrefixTreeEncoder;
30  import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.TokenizerNode;
31  import org.apache.hadoop.hbase.util.ByteRangeUtils;
32  import org.apache.hadoop.hbase.util.CollectionUtils;
33  import org.apache.hadoop.hbase.util.vint.UFIntTool;
34  import org.apache.hadoop.hbase.util.vint.UVIntTool;
35  
36  /**
37   * Serializes the fields comprising one node of the row trie, which can be a branch, nub, or leaf.
38   * Please see the write() method for the order in which data is written.
39   */
40  @InterfaceAudience.Private
41  public class RowNodeWriter{
42    protected static final Log LOG = LogFactory.getLog(RowNodeWriter.class);
43  
44    /********************* fields ******************************/
45  
46    protected PrefixTreeEncoder prefixTreeEncoder;
47    protected PrefixTreeBlockMeta blockMeta;
48    protected TokenizerNode tokenizerNode;
49  
50    protected int tokenWidth;
51    protected int fanOut;
52    protected int numCells;
53  
54    protected int width;
55  
56  
57    /*********************** construct *************************/
58  
59    public RowNodeWriter(PrefixTreeEncoder keyValueBuilder, TokenizerNode tokenizerNode) {
60      reconstruct(keyValueBuilder, tokenizerNode);
61    }
62  
63    public void reconstruct(PrefixTreeEncoder prefixTreeEncoder, TokenizerNode tokenizerNode) {
64      this.prefixTreeEncoder = prefixTreeEncoder;
65      reset(tokenizerNode);
66    }
67  
68    public void reset(TokenizerNode node) {
69      this.blockMeta = prefixTreeEncoder.getBlockMeta();// changes between blocks
70      this.tokenizerNode = node;
71      this.tokenWidth = 0;
72      this.fanOut = 0;
73      this.numCells = 0;
74      this.width = 0;
75      calculateOffsetsAndLengths();
76    }
77  
78  
79    /********************* methods ****************************/
80  
81    protected void calculateOffsetsAndLengths(){
82      tokenWidth = tokenizerNode.getTokenLength();
83      if(!tokenizerNode.isRoot()){
84        --tokenWidth;//root has no parent
85      }
86      fanOut = CollectionUtils.nullSafeSize(tokenizerNode.getChildren());
87      numCells = tokenizerNode.getNumOccurrences();
88    }
89  
90    public int calculateWidth(){
91      calculateWidthOverrideOffsetWidth(blockMeta.getNextNodeOffsetWidth());
92      return width;
93    }
94  
95    public int calculateWidthOverrideOffsetWidth(int offsetWidth){
96      width = 0;
97      width += UVIntTool.numBytes(tokenWidth);
98      width += tokenWidth;
99  
100     width += UVIntTool.numBytes(fanOut);
101     width += fanOut;
102 
103     width += UVIntTool.numBytes(numCells);
104 
105     if(tokenizerNode.hasOccurrences()){
106       int fixedBytesPerCell = blockMeta.getFamilyOffsetWidth()
107         + blockMeta.getQualifierOffsetWidth()
108         + blockMeta.getTimestampIndexWidth()
109         + blockMeta.getMvccVersionIndexWidth()
110         + blockMeta.getKeyValueTypeWidth()
111         + blockMeta.getValueOffsetWidth()
112         + blockMeta.getValueLengthWidth();
113       width += numCells * fixedBytesPerCell;
114     }
115 
116     if( ! tokenizerNode.isLeaf()){
117       width += fanOut * offsetWidth;
118     }
119 
120     return width;
121   }
122 
123 
124   /*********************** writing the compiled structure to the OutputStream ***************/
125 
126   public void write(OutputStream os) throws IOException{
127     //info about this row trie node
128     writeRowToken(os);
129     writeFan(os);
130     writeNumCells(os);
131 
132     //UFInt indexes and offsets for each cell in the row (if nub or leaf)
133     writeFamilyNodeOffsets(os);
134     writeQualifierNodeOffsets(os);
135     writeTimestampIndexes(os);
136     writeMvccVersionIndexes(os);
137     writeCellTypes(os);
138     writeValueOffsets(os);
139     writeValueLengths(os);
140 
141     //offsets to the children of this row trie node (if branch or nub)
142     writeNextRowTrieNodeOffsets(os);
143   }
144 
145 
146   /**
147    * Row node token, fan, and numCells. Written once at the beginning of each row node. These 3
148    * fields can reproduce all the row keys that compose the block.
149    */
150 
151   /**
152    * UVInt: tokenWidth
153    * bytes: token
154    */
155   protected void writeRowToken(OutputStream os) throws IOException {
156     UVIntTool.writeBytes(tokenWidth, os);
157     int tokenStartIndex = tokenizerNode.isRoot() ? 0 : 1;
158     ByteRangeUtils.write(os, tokenizerNode.getToken(), tokenStartIndex);
159   }
160 
161   /**
162    * UVInt: numFanBytes/fanOut
163    * bytes: each fan byte
164    */
165   public void writeFan(OutputStream os) throws IOException {
166     UVIntTool.writeBytes(fanOut, os);
167     if (fanOut <= 0) {
168       return;
169     }
170     ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
171     for (int i = 0; i < children.size(); ++i) {
172       TokenizerNode child = children.get(i);
173       os.write(child.getToken().get(0));// first byte of each child's token
174     }
175   }
176 
177   /**
178    * UVInt: numCells, the number of cells in this row which will be 0 for branch nodes
179    */
180   protected void writeNumCells(OutputStream os) throws IOException {
181     UVIntTool.writeBytes(numCells, os);
182   }
183 
184 
185   /**
186    * The following methods write data for each cell in the row, mostly consisting of indexes or
187    * offsets into the timestamp/column data structures that are written in the middle of the block.
188    * We use {@link UFIntTool} to encode these indexes/offsets to allow random access during a binary
189    * search of a particular column/timestamp combination.
190    * <p/>
191    * Branch nodes will not have any data in these sections.
192    */
193 
194   protected void writeFamilyNodeOffsets(OutputStream os) throws IOException {
195     if (blockMeta.getFamilyOffsetWidth() <= 0) {
196       return;
197     }
198     for (int i = 0; i < numCells; ++i) {
199       int cellInsertionIndex = PrefixTreeEncoder.MULITPLE_FAMILIES_POSSIBLE ? tokenizerNode
200           .getFirstInsertionIndex() + i : 0;
201       int sortedIndex = prefixTreeEncoder.getFamilySorter().getSortedIndexForInsertionId(
202         cellInsertionIndex);
203       int indexedFamilyOffset = prefixTreeEncoder.getFamilyWriter().getOutputArrayOffset(
204         sortedIndex);
205       UFIntTool.writeBytes(blockMeta.getFamilyOffsetWidth(), indexedFamilyOffset, os);
206     }
207   }
208 
209   protected void writeQualifierNodeOffsets(OutputStream os) throws IOException {
210     if (blockMeta.getQualifierOffsetWidth() <= 0) {
211       return;
212     }
213     for (int i = 0; i < numCells; ++i) {
214       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
215       int sortedIndex = prefixTreeEncoder.getQualifierSorter().getSortedIndexForInsertionId(
216         cellInsertionIndex);
217       int indexedQualifierOffset = prefixTreeEncoder.getQualifierWriter().getOutputArrayOffset(
218         sortedIndex);
219       UFIntTool.writeBytes(blockMeta.getQualifierOffsetWidth(), indexedQualifierOffset, os);
220     }
221   }
222 
223   protected void writeTimestampIndexes(OutputStream os) throws IOException {
224     if (blockMeta.getTimestampIndexWidth() <= 0) {
225       return;
226     }
227     for (int i = 0; i < numCells; ++i) {
228       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
229       long timestamp = prefixTreeEncoder.getTimestamps()[cellInsertionIndex];
230       int timestampIndex = prefixTreeEncoder.getTimestampEncoder().getIndex(timestamp);
231       UFIntTool.writeBytes(blockMeta.getTimestampIndexWidth(), timestampIndex, os);
232     }
233   }
234 
235   protected void writeMvccVersionIndexes(OutputStream os) throws IOException {
236     if (blockMeta.getMvccVersionIndexWidth() <= 0) {
237       return;
238     }
239     for (int i = 0; i < numCells; ++i) {
240       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
241       long mvccVersion = prefixTreeEncoder.getMvccVersions()[cellInsertionIndex];
242       int mvccVersionIndex = prefixTreeEncoder.getMvccVersionEncoder().getIndex(mvccVersion);
243       UFIntTool.writeBytes(blockMeta.getMvccVersionIndexWidth(), mvccVersionIndex, os);
244     }
245   }
246 
247   protected void writeCellTypes(OutputStream os) throws IOException {
248     if (blockMeta.isAllSameType()) {
249       return;
250     }
251     for (int i = 0; i < numCells; ++i) {
252       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
253       os.write(prefixTreeEncoder.getTypeBytes()[cellInsertionIndex]);
254     }
255   }
256 
257   protected void writeValueOffsets(OutputStream os) throws IOException {
258     for (int i = 0; i < numCells; ++i) {
259       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
260       long valueStartIndex = prefixTreeEncoder.getValueOffset(cellInsertionIndex);
261       UFIntTool.writeBytes(blockMeta.getValueOffsetWidth(), valueStartIndex, os);
262     }
263   }
264 
265   protected void writeValueLengths(OutputStream os) throws IOException {
266     for (int i = 0; i < numCells; ++i) {
267       int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
268       int valueLength = prefixTreeEncoder.getValueLength(cellInsertionIndex);
269       UFIntTool.writeBytes(blockMeta.getValueLengthWidth(), valueLength, os);
270     }
271   }
272 
273 
274   /**
275    * If a branch or a nub, the last thing we append are the UFInt offsets to the child row nodes.
276    */
277   protected void writeNextRowTrieNodeOffsets(OutputStream os) throws IOException {
278     ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
279     for (int i = 0; i < children.size(); ++i) {
280       TokenizerNode child = children.get(i);
281       int distanceToChild = tokenizerNode.getNegativeIndex() - child.getNegativeIndex();
282       UFIntTool.writeBytes(blockMeta.getNextNodeOffsetWidth(), distanceToChild, os);
283     }
284   }
285 }