View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree.decode;
20  
21  import org.apache.hadoop.classification.InterfaceAudience;
22  import org.apache.hadoop.hbase.Cell;
23  import org.apache.hadoop.hbase.CellComparator;
24  import org.apache.hadoop.hbase.CellScanner;
25  import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta;
26  import org.apache.hadoop.hbase.codec.prefixtree.decode.column.ColumnReader;
27  import org.apache.hadoop.hbase.codec.prefixtree.decode.row.RowNodeReader;
28  import org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp.MvccVersionDecoder;
29  import org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp.TimestampDecoder;
30  
31  /**
32   * Extends PtCell and manipulates its protected fields.  Could alternatively contain a PtCell and
33   * call get/set methods.
34   *
35   * This is an "Array" scanner to distinguish from a future "ByteBuffer" scanner.  This
36   * implementation requires that the bytes be in a normal java byte[] for performance.  The
37   * alternative ByteBuffer implementation would allow for accessing data in an off-heap ByteBuffer
38   * without copying the whole buffer on-heap.
39   */
40  @InterfaceAudience.Private
41  public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanner {
42  
43    /***************** fields ********************************/
44  
45    protected PrefixTreeBlockMeta blockMeta;
46  
47    protected boolean beforeFirst;
48    protected boolean afterLast;
49  
50    protected RowNodeReader[] rowNodes;
51    protected int rowNodeStackIndex;
52  
53    protected RowNodeReader currentRowNode;
54    protected ColumnReader familyReader;
55    protected ColumnReader qualifierReader;
56    protected TimestampDecoder timestampDecoder;
57    protected MvccVersionDecoder mvccVersionDecoder;
58  
59    protected boolean nubCellsRemain;
60    protected int currentCellIndex;
61  
62  
63    /*********************** construct ******************************/
64  
65    // pass in blockMeta so we can initialize buffers big enough for all cells in the block
66    public PrefixTreeArrayScanner(PrefixTreeBlockMeta blockMeta, int rowTreeDepth, 
67        int rowBufferLength, int qualifierBufferLength) {
68      this.rowNodes = new RowNodeReader[rowTreeDepth];
69      for (int i = 0; i < rowNodes.length; ++i) {
70        rowNodes[i] = new RowNodeReader();
71      }
72      this.rowBuffer = new byte[rowBufferLength];
73      this.familyBuffer = new byte[PrefixTreeBlockMeta.MAX_FAMILY_LENGTH];
74      this.familyReader = new ColumnReader(familyBuffer, true);
75      this.qualifierBuffer = new byte[qualifierBufferLength];
76      this.qualifierReader = new ColumnReader(qualifierBuffer, false);
77      this.timestampDecoder = new TimestampDecoder();
78      this.mvccVersionDecoder = new MvccVersionDecoder();
79    }
80  
81  
82    /**************** init helpers ***************************************/
83  
84    /**
85     * Call when first accessing a block.
86     * @return entirely new scanner if false
87     */
88    public boolean areBuffersBigEnough() {
89      if (rowNodes.length < blockMeta.getRowTreeDepth()) {
90        return false;
91      }
92      if (rowBuffer.length < blockMeta.getMaxRowLength()) {
93        return false;
94      }
95      if (qualifierBuffer.length < blockMeta.getMaxQualifierLength()) {
96        return false;
97      }
98      return true;
99    }
100 
101   public void initOnBlock(PrefixTreeBlockMeta blockMeta, byte[] block,
102       boolean includeMvccVersion) {
103     this.block = block;
104     this.blockMeta = blockMeta;
105     this.familyOffset = familyBuffer.length;
106     this.familyReader.initOnBlock(blockMeta, block);
107     this.qualifierOffset = qualifierBuffer.length;
108     this.qualifierReader.initOnBlock(blockMeta, block);
109     this.timestampDecoder.initOnBlock(blockMeta, block);
110     this.mvccVersionDecoder.initOnBlock(blockMeta, block);
111     this.includeMvccVersion = includeMvccVersion;
112     resetToBeforeFirstEntry();
113   }
114 
115   // Does this have to be in the CellScanner Interface?  TODO
116   public void resetToBeforeFirstEntry() {
117     beforeFirst = true;
118     afterLast = false;
119     rowNodeStackIndex = -1;
120     currentRowNode = null;
121     rowLength = 0;
122     familyOffset = familyBuffer.length;
123     familyLength = 0;
124     qualifierOffset = blockMeta.getMaxQualifierLength();
125     qualifierLength = 0;
126     nubCellsRemain = false;
127     currentCellIndex = -1;
128     timestamp = -1L;
129     type = DEFAULT_TYPE;
130     absoluteValueOffset = 0;//use 0 vs -1 so the cell is valid when value hasn't been initialized
131     valueLength = 0;// had it at -1, but that causes null Cell to add up to the wrong length
132   }
133 
134   /**
135    * Call this before putting the scanner back into a pool so it doesn't hold the last used block
136    * in memory.
137    */
138   public void releaseBlockReference(){
139     block = null;
140   }
141 
142 
143   /********************** CellScanner **********************/
144 
145   @Override
146   public Cell current() {
147     if(isOutOfBounds()){
148       return null;
149     }
150     return (Cell)this;
151   }
152 
153   /******************* Object methods ************************/
154 
155   @Override
156   public boolean equals(Object obj) {
157     //trivial override to confirm intent (findbugs)
158     return super.equals(obj);
159   }
160   
161   @Override
162   public int hashCode() {
163     return super.hashCode();
164   }
165 
166   /**
167    * Override PrefixTreeCell.toString() with a check to see if the current cell is valid.
168    */
169   @Override
170   public String toString() {
171     Cell currentCell = current();
172     if(currentCell==null){
173       return "null";
174     }
175     return ((PrefixTreeCell)currentCell).getKeyValueString();
176   }
177 
178 
179   /******************* advance ***************************/
180 
181   public boolean positionAtFirstCell() {
182     reInitFirstNode();
183     return advance();
184   }
185 
186   @Override
187   public boolean advance() {
188     if (afterLast) {
189       return false;
190     }
191     if (!hasOccurrences()) {
192       resetToBeforeFirstEntry();
193     }
194     if (beforeFirst || isLastCellInRow()) {
195       nextRow();
196       if (afterLast) {
197         return false;
198       }
199     } else {
200       ++currentCellIndex;
201     }
202 
203     populateNonRowFields(currentCellIndex);
204     return true;
205   }
206 
207 
208   public boolean nextRow() {
209     nextRowInternal();
210     if (afterLast) {
211       return false;
212     }
213     populateNonRowFields(currentCellIndex);
214     return true;
215   }
216 
217 
218   /**
219    * This method is safe to call when the scanner is not on a fully valid row node, as in the case
220    * of a row token miss in the Searcher
221    * @return true if we are positioned on a valid row, false if past end of block
222    */
223   protected boolean nextRowInternal() {
224     if (afterLast) {
225       return false;
226     }
227     if (beforeFirst) {
228       initFirstNode();
229       if (currentRowNode.hasOccurrences()) {
230         if (currentRowNode.isNub()) {
231           nubCellsRemain = true;
232         }
233         currentCellIndex = 0;
234         return true;
235       }
236     }
237     if (currentRowNode.isLeaf()) {
238       discardCurrentRowNode(true);
239     }
240     while (!afterLast) {
241       if (nubCellsRemain) {
242         nubCellsRemain = false;
243       }
244       if (currentRowNode.hasMoreFanNodes()) {
245         followNextFan();
246         if (currentRowNode.hasOccurrences()) {
247           currentCellIndex = 0;
248           return true;
249         }// found some values
250       } else {
251         discardCurrentRowNode(true);
252       }
253     }
254     return false;// went past the end
255   }
256 
257 
258   /**************** secondary traversal methods ******************************/
259 
260   protected void reInitFirstNode() {
261     resetToBeforeFirstEntry();
262     initFirstNode();
263   }
264 
265   protected void initFirstNode() {
266     int offsetIntoUnderlyingStructure = blockMeta.getAbsoluteRowOffset();
267     rowNodeStackIndex = 0;
268     currentRowNode = rowNodes[0];
269     currentRowNode.initOnBlock(blockMeta, block, offsetIntoUnderlyingStructure);
270     appendCurrentTokenToRowBuffer();
271     beforeFirst = false;
272   }
273 
274   protected void followFirstFan() {
275     followFan(0);
276   }
277 
278   protected void followPreviousFan() {
279     int nextFanPosition = currentRowNode.getFanIndex() - 1;
280     followFan(nextFanPosition);
281   }
282 
283   protected void followCurrentFan() {
284     int currentFanPosition = currentRowNode.getFanIndex();
285     followFan(currentFanPosition);
286   }
287 
288   protected void followNextFan() {
289     int nextFanPosition = currentRowNode.getFanIndex() + 1;
290     followFan(nextFanPosition);
291   }
292 
293   protected void followLastFan() {
294     followFan(currentRowNode.getLastFanIndex());
295   }
296 
297   protected void followFan(int fanIndex) {
298     currentRowNode.setFanIndex(fanIndex);
299     appendToRowBuffer(currentRowNode.getFanByte(fanIndex));
300 
301     int nextOffsetIntoUnderlyingStructure = currentRowNode.getOffset()
302         + currentRowNode.getNextNodeOffset(fanIndex, blockMeta);
303     ++rowNodeStackIndex;
304 
305     currentRowNode = rowNodes[rowNodeStackIndex];
306     currentRowNode.initOnBlock(blockMeta, block, nextOffsetIntoUnderlyingStructure);
307 
308     //TODO getToken is spewing garbage
309     appendCurrentTokenToRowBuffer();
310     if (currentRowNode.isNub()) {
311       nubCellsRemain = true;
312     }
313     currentCellIndex = 0;
314   }
315 
316   /**
317    * @param forwards which marker to set if we overflow
318    */
319   protected void discardCurrentRowNode(boolean forwards) {
320     RowNodeReader rowNodeBeingPopped = currentRowNode;
321     --rowNodeStackIndex;// pop it off the stack
322     if (rowNodeStackIndex < 0) {
323       currentRowNode = null;
324       if (forwards) {
325         markAfterLast();
326       } else {
327         markBeforeFirst();
328       }
329       return;
330     }
331     popFromRowBuffer(rowNodeBeingPopped);
332     currentRowNode = rowNodes[rowNodeStackIndex];
333   }
334 
335   protected void markBeforeFirst() {
336     beforeFirst = true;
337     afterLast = false;
338     currentRowNode = null;
339   }
340 
341   protected void markAfterLast() {
342     beforeFirst = false;
343     afterLast = true;
344     currentRowNode = null;
345   }
346 
347 
348   /***************** helper methods **************************/
349 
350   protected void appendCurrentTokenToRowBuffer() {
351     System.arraycopy(block, currentRowNode.getTokenArrayOffset(), rowBuffer, rowLength, 
352       currentRowNode.getTokenLength());
353     rowLength += currentRowNode.getTokenLength();
354   }
355 
356   protected void appendToRowBuffer(byte b) {
357     rowBuffer[rowLength] = b;
358     ++rowLength;
359   }
360 
361   protected void popFromRowBuffer(RowNodeReader rowNodeBeingPopped) {
362     rowLength -= rowNodeBeingPopped.getTokenLength();
363     --rowLength; // pop the parent's fan byte
364   }
365 
366   protected boolean hasOccurrences() {
367     return currentRowNode != null && currentRowNode.hasOccurrences();
368   }
369 
370   protected boolean isBranch() {
371     return currentRowNode != null && !currentRowNode.hasOccurrences()
372         && currentRowNode.hasChildren();
373   }
374 
375   protected boolean isNub() {
376     return currentRowNode != null && currentRowNode.hasOccurrences()
377         && currentRowNode.hasChildren();
378   }
379 
380   protected boolean isLeaf() {
381     return currentRowNode != null && currentRowNode.hasOccurrences()
382         && !currentRowNode.hasChildren();
383   }
384 
385   //TODO expose this in a PrefixTreeScanner interface
386   public boolean isBeforeFirst(){
387     return beforeFirst;
388   }
389 
390   public boolean isAfterLast(){
391     return afterLast;
392   }
393 
394   protected boolean isOutOfBounds(){
395     return beforeFirst || afterLast;
396   }
397 
398   protected boolean isFirstCellInRow() {
399     return currentCellIndex == 0;
400   }
401 
402   protected boolean isLastCellInRow() {
403     return currentCellIndex == currentRowNode.getLastCellIndex();
404   }
405 
406 
407   /********************* fill in family/qualifier/ts/type/value ************/
408 
409   protected int populateNonRowFieldsAndCompareTo(int cellNum, Cell key) {
410     populateNonRowFields(cellNum);
411     return CellComparator.compareStatic(this, key);
412   }
413 
414   protected void populateFirstNonRowFields() {
415     populateNonRowFields(0);
416   }
417 
418   protected void populatePreviousNonRowFields() {
419     populateNonRowFields(currentCellIndex - 1);
420   }
421 
422   protected void populateLastNonRowFields() {
423     populateNonRowFields(currentRowNode.getLastCellIndex());
424   }
425 
426   protected void populateNonRowFields(int cellIndex) {
427     currentCellIndex = cellIndex;
428     populateFamily();
429     populateQualifier();
430     populateTimestamp();
431     populateMvccVersion();
432     populateType();
433     populateValueOffsets();
434   }
435 
436   protected void populateFamily() {
437     int familyTreeIndex = currentRowNode.getFamilyOffset(currentCellIndex, blockMeta);
438     familyOffset = familyReader.populateBuffer(familyTreeIndex).getColumnOffset();
439     familyLength = familyReader.getColumnLength();
440   }
441 
442   protected void populateQualifier() {
443     int qualifierTreeIndex = currentRowNode.getColumnOffset(currentCellIndex, blockMeta);
444     qualifierOffset = qualifierReader.populateBuffer(qualifierTreeIndex).getColumnOffset();
445     qualifierLength = qualifierReader.getColumnLength();
446   }
447 
448   protected void populateTimestamp() {
449     if (blockMeta.isAllSameTimestamp()) {
450       timestamp = blockMeta.getMinTimestamp();
451     } else {
452       int timestampIndex = currentRowNode.getTimestampIndex(currentCellIndex, blockMeta);
453       timestamp = timestampDecoder.getLong(timestampIndex);
454     }
455   }
456 
457   protected void populateMvccVersion() {
458     if (blockMeta.isAllSameMvccVersion()) {
459       mvccVersion = blockMeta.getMinMvccVersion();
460     } else {
461       int mvccVersionIndex = currentRowNode.getMvccVersionIndex(currentCellIndex,
462         blockMeta);
463       mvccVersion = mvccVersionDecoder.getMvccVersion(mvccVersionIndex);
464     }
465   }
466 
467   protected void populateType() {
468     int typeInt;
469     if (blockMeta.isAllSameType()) {
470       typeInt = blockMeta.getAllTypes();
471     } else {
472       typeInt = currentRowNode.getType(currentCellIndex, blockMeta);
473     }
474     type = PrefixTreeCell.TYPES[typeInt];
475   }
476 
477   protected void populateValueOffsets() {
478     int offsetIntoValueSection = currentRowNode.getValueOffset(currentCellIndex, blockMeta);
479     absoluteValueOffset = blockMeta.getAbsoluteValueOffset() + offsetIntoValueSection;
480     valueLength = currentRowNode.getValueLength(currentCellIndex, blockMeta);
481   }
482 
483 
484   /**************** getters ***************************/
485 
486   public byte[] getTreeBytes() {
487     return block;
488   }
489 
490   public PrefixTreeBlockMeta getBlockMeta() {
491     return blockMeta;
492   }
493 
494   public int getMaxRowTreeStackNodes() {
495     return rowNodes.length;
496   }
497 
498   public int getRowBufferLength() {
499     return rowBuffer.length;
500   }
501 
502   public int getQualifierBufferLength() {
503     return qualifierBuffer.length;
504   }
505 
506 }