1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.codec.prefixtree.encode.row;
20
21 import java.io.IOException;
22 import java.io.OutputStream;
23 import java.util.ArrayList;
24
25 import org.apache.commons.logging.Log;
26 import org.apache.commons.logging.LogFactory;
27 import org.apache.hadoop.classification.InterfaceAudience;
28 import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta;
29 import org.apache.hadoop.hbase.codec.prefixtree.encode.PrefixTreeEncoder;
30 import org.apache.hadoop.hbase.codec.prefixtree.encode.tokenize.TokenizerNode;
31 import org.apache.hadoop.hbase.util.ByteRangeTool;
32 import org.apache.hadoop.hbase.util.CollectionUtils;
33 import org.apache.hadoop.hbase.util.vint.UFIntTool;
34 import org.apache.hadoop.hbase.util.vint.UVIntTool;
35
36
37
38
39
40 @InterfaceAudience.Private
41 public class RowNodeWriter{
42 protected static final Log LOG = LogFactory.getLog(RowNodeWriter.class);
43
44
45
46 protected PrefixTreeEncoder prefixTreeEncoder;
47 protected PrefixTreeBlockMeta blockMeta;
48 protected TokenizerNode tokenizerNode;
49
50 protected int tokenWidth;
51 protected int fanOut;
52 protected int numCells;
53
54 protected int width;
55
56
57
58
59 public RowNodeWriter(PrefixTreeEncoder keyValueBuilder, TokenizerNode tokenizerNode) {
60 reconstruct(keyValueBuilder, tokenizerNode);
61 }
62
63 public void reconstruct(PrefixTreeEncoder prefixTreeEncoder, TokenizerNode tokenizerNode) {
64 this.prefixTreeEncoder = prefixTreeEncoder;
65 reset(tokenizerNode);
66 }
67
68 public void reset(TokenizerNode node) {
69 this.blockMeta = prefixTreeEncoder.getBlockMeta();
70 this.tokenizerNode = node;
71 this.tokenWidth = 0;
72 this.fanOut = 0;
73 this.numCells = 0;
74 this.width = 0;
75 calculateOffsetsAndLengths();
76 }
77
78
79
80
81 protected void calculateOffsetsAndLengths(){
82 tokenWidth = tokenizerNode.getTokenLength();
83 if(!tokenizerNode.isRoot()){
84 --tokenWidth;
85 }
86 fanOut = CollectionUtils.nullSafeSize(tokenizerNode.getChildren());
87 numCells = tokenizerNode.getNumOccurrences();
88 }
89
90 public int calculateWidth(){
91 calculateWidthOverrideOffsetWidth(blockMeta.getNextNodeOffsetWidth());
92 return width;
93 }
94
95 public int calculateWidthOverrideOffsetWidth(int offsetWidth){
96 width = 0;
97 width += UVIntTool.numBytes(tokenWidth);
98 width += tokenWidth;
99
100 width += UVIntTool.numBytes(fanOut);
101 width += fanOut;
102
103 width += UVIntTool.numBytes(numCells);
104
105 if(tokenizerNode.hasOccurrences()){
106 int fixedBytesPerCell = blockMeta.getFamilyOffsetWidth()
107 + blockMeta.getQualifierOffsetWidth()
108 + blockMeta.getTimestampIndexWidth()
109 + blockMeta.getMvccVersionIndexWidth()
110 + blockMeta.getKeyValueTypeWidth()
111 + blockMeta.getValueOffsetWidth()
112 + blockMeta.getValueLengthWidth();
113 width += numCells * fixedBytesPerCell;
114 }
115
116 if( ! tokenizerNode.isLeaf()){
117 width += fanOut * offsetWidth;
118 }
119
120 return width;
121 }
122
123
124
125
126 public void write(OutputStream os) throws IOException{
127
128 writeRowToken(os);
129 writeFan(os);
130 writeNumCells(os);
131
132
133 writeFamilyNodeOffsets(os);
134 writeQualifierNodeOffsets(os);
135 writeTimestampIndexes(os);
136 writeMvccVersionIndexes(os);
137 writeCellTypes(os);
138 writeValueOffsets(os);
139 writeValueLengths(os);
140
141
142 writeNextRowTrieNodeOffsets(os);
143 }
144
145
146
147
148
149
150
151
152
153
154
155 protected void writeRowToken(OutputStream os) throws IOException {
156 UVIntTool.writeBytes(tokenWidth, os);
157 int tokenStartIndex = tokenizerNode.isRoot() ? 0 : 1;
158 ByteRangeTool.write(os, tokenizerNode.getToken(), tokenStartIndex);
159 }
160
161
162
163
164
165 public void writeFan(OutputStream os) throws IOException {
166 UVIntTool.writeBytes(fanOut, os);
167 if (fanOut <= 0) {
168 return;
169 }
170 ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
171 for (int i = 0; i < children.size(); ++i) {
172 TokenizerNode child = children.get(i);
173 os.write(child.getToken().get(0));
174 }
175 }
176
177
178
179
180 protected void writeNumCells(OutputStream os) throws IOException {
181 UVIntTool.writeBytes(numCells, os);
182 }
183
184
185
186
187
188
189
190
191
192
193
194 protected void writeFamilyNodeOffsets(OutputStream os) throws IOException {
195 if (blockMeta.getFamilyOffsetWidth() <= 0) {
196 return;
197 }
198 for (int i = 0; i < numCells; ++i) {
199 int cellInsertionIndex = PrefixTreeEncoder.MULITPLE_FAMILIES_POSSIBLE ? tokenizerNode
200 .getFirstInsertionIndex() + i : 0;
201 int sortedIndex = prefixTreeEncoder.getFamilySorter().getSortedIndexForInsertionId(
202 cellInsertionIndex);
203 int indexedFamilyOffset = prefixTreeEncoder.getFamilyWriter().getOutputArrayOffset(
204 sortedIndex);
205 UFIntTool.writeBytes(blockMeta.getFamilyOffsetWidth(), indexedFamilyOffset, os);
206 }
207 }
208
209 protected void writeQualifierNodeOffsets(OutputStream os) throws IOException {
210 if (blockMeta.getQualifierOffsetWidth() <= 0) {
211 return;
212 }
213 for (int i = 0; i < numCells; ++i) {
214 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
215 int sortedIndex = prefixTreeEncoder.getQualifierSorter().getSortedIndexForInsertionId(
216 cellInsertionIndex);
217 int indexedQualifierOffset = prefixTreeEncoder.getQualifierWriter().getOutputArrayOffset(
218 sortedIndex);
219 UFIntTool.writeBytes(blockMeta.getQualifierOffsetWidth(), indexedQualifierOffset, os);
220 }
221 }
222
223 protected void writeTimestampIndexes(OutputStream os) throws IOException {
224 if (blockMeta.getTimestampIndexWidth() <= 0) {
225 return;
226 }
227 for (int i = 0; i < numCells; ++i) {
228 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
229 long timestamp = prefixTreeEncoder.getTimestamps()[cellInsertionIndex];
230 int timestampIndex = prefixTreeEncoder.getTimestampEncoder().getIndex(timestamp);
231 UFIntTool.writeBytes(blockMeta.getTimestampIndexWidth(), timestampIndex, os);
232 }
233 }
234
235 protected void writeMvccVersionIndexes(OutputStream os) throws IOException {
236 if (blockMeta.getMvccVersionIndexWidth() <= 0) {
237 return;
238 }
239 for (int i = 0; i < numCells; ++i) {
240 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
241 long mvccVersion = prefixTreeEncoder.getMvccVersions()[cellInsertionIndex];
242 int mvccVersionIndex = prefixTreeEncoder.getMvccVersionEncoder().getIndex(mvccVersion);
243 UFIntTool.writeBytes(blockMeta.getMvccVersionIndexWidth(), mvccVersionIndex, os);
244 }
245 }
246
247 protected void writeCellTypes(OutputStream os) throws IOException {
248 if (blockMeta.isAllSameType()) {
249 return;
250 }
251 for (int i = 0; i < numCells; ++i) {
252 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
253 os.write(prefixTreeEncoder.getTypeBytes()[cellInsertionIndex]);
254 }
255 }
256
257 protected void writeValueOffsets(OutputStream os) throws IOException {
258 for (int i = 0; i < numCells; ++i) {
259 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
260 long valueStartIndex = prefixTreeEncoder.getValueOffset(cellInsertionIndex);
261 UFIntTool.writeBytes(blockMeta.getValueOffsetWidth(), valueStartIndex, os);
262 }
263 }
264
265 protected void writeValueLengths(OutputStream os) throws IOException {
266 for (int i = 0; i < numCells; ++i) {
267 int cellInsertionIndex = tokenizerNode.getFirstInsertionIndex() + i;
268 int valueLength = prefixTreeEncoder.getValueLength(cellInsertionIndex);
269 UFIntTool.writeBytes(blockMeta.getValueLengthWidth(), valueLength, os);
270 }
271 }
272
273
274
275
276
277 protected void writeNextRowTrieNodeOffsets(OutputStream os) throws IOException {
278 ArrayList<TokenizerNode> children = tokenizerNode.getChildren();
279 for (int i = 0; i < children.size(); ++i) {
280 TokenizerNode child = children.get(i);
281 int distanceToChild = tokenizerNode.getNegativeIndex() - child.getNegativeIndex();
282 UFIntTool.writeBytes(blockMeta.getNextNodeOffsetWidth(), distanceToChild, os);
283 }
284 }
285 }