View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.codec.prefixtree.decode;
20  
21  import org.apache.hadoop.classification.InterfaceAudience;
22  import org.apache.hadoop.hbase.Cell;
23  import org.apache.hadoop.hbase.CellComparator;
24  import org.apache.hadoop.hbase.CellScanner;
25  import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta;
26  import org.apache.hadoop.hbase.codec.prefixtree.decode.column.ColumnReader;
27  import org.apache.hadoop.hbase.codec.prefixtree.decode.row.RowNodeReader;
28  import org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp.MvccVersionDecoder;
29  import org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp.TimestampDecoder;
30  import org.apache.hadoop.hbase.codec.prefixtree.encode.other.ColumnNodeType;
31  import org.apache.hadoop.hbase.util.Bytes;
32  
33  /**
34   * Extends PtCell and manipulates its protected fields.  Could alternatively contain a PtCell and
35   * call get/set methods.
36   *
37   * This is an "Array" scanner to distinguish from a future "ByteBuffer" scanner.  This
38   * implementation requires that the bytes be in a normal java byte[] for performance.  The
39   * alternative ByteBuffer implementation would allow for accessing data in an off-heap ByteBuffer
40   * without copying the whole buffer on-heap.
41   */
42  @InterfaceAudience.Private
43  public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanner {
44  
45    /***************** fields ********************************/
46  
47    protected PrefixTreeBlockMeta blockMeta;
48  
49    protected boolean beforeFirst;
50    protected boolean afterLast;
51  
52    protected RowNodeReader[] rowNodes;
53    protected int rowNodeStackIndex;
54  
55    protected RowNodeReader currentRowNode;
56    protected ColumnReader familyReader;
57    protected ColumnReader qualifierReader;
58    protected ColumnReader tagsReader;
59    protected TimestampDecoder timestampDecoder;
60    protected MvccVersionDecoder mvccVersionDecoder;
61  
62    protected boolean nubCellsRemain;
63    protected int currentCellIndex;
64    protected boolean movedToPrevious;
65  
66  
67    /*********************** construct ******************************/
68  
69    // pass in blockMeta so we can initialize buffers big enough for all cells in the block
70    public PrefixTreeArrayScanner(PrefixTreeBlockMeta blockMeta, int rowTreeDepth,
71        int rowBufferLength, int qualifierBufferLength, int tagsBufferLength) {
72      this.rowNodes = new RowNodeReader[rowTreeDepth];
73      for (int i = 0; i < rowNodes.length; ++i) {
74        rowNodes[i] = new RowNodeReader();
75      }
76      this.rowBuffer = new byte[rowBufferLength];
77      this.familyBuffer = new byte[PrefixTreeBlockMeta.MAX_FAMILY_LENGTH];
78      this.familyReader = new ColumnReader(familyBuffer, ColumnNodeType.FAMILY);
79      this.qualifierBuffer = new byte[qualifierBufferLength];
80      this.tagsBuffer = new byte[tagsBufferLength];
81      this.qualifierReader = new ColumnReader(qualifierBuffer, ColumnNodeType.QUALIFIER);
82      this.tagsReader = new ColumnReader(tagsBuffer, ColumnNodeType.TAGS);
83      this.timestampDecoder = new TimestampDecoder();
84      this.mvccVersionDecoder = new MvccVersionDecoder();
85    }
86  
87  
88    /**************** init helpers ***************************************/
89  
90    /**
91     * Call when first accessing a block.
92     * @return entirely new scanner if false
93     */
94    public boolean areBuffersBigEnough() {
95      if (rowNodes.length < blockMeta.getRowTreeDepth()) {
96        return false;
97      }
98      if (rowBuffer.length < blockMeta.getMaxRowLength()) {
99        return false;
100     }
101     if (qualifierBuffer.length < blockMeta.getMaxQualifierLength()) {
102       return false;
103     }
104     if(tagsBuffer.length < blockMeta.getMaxTagsLength()) {
105       return false;
106     }
107     return true;
108   }
109 
110   public void initOnBlock(PrefixTreeBlockMeta blockMeta, byte[] block,
111       boolean includeMvccVersion) {
112     this.block = block;
113     this.blockMeta = blockMeta;
114     this.familyOffset = familyBuffer.length;
115     this.familyReader.initOnBlock(blockMeta, block);
116     this.qualifierOffset = qualifierBuffer.length;
117     this.qualifierReader.initOnBlock(blockMeta, block);
118     this.tagsOffset = tagsBuffer.length;
119     this.tagsReader.initOnBlock(blockMeta, block);
120     this.timestampDecoder.initOnBlock(blockMeta, block);
121     this.mvccVersionDecoder.initOnBlock(blockMeta, block);
122     this.includeMvccVersion = includeMvccVersion;
123     resetToBeforeFirstEntry();
124   }
125 
126   // Does this have to be in the CellScanner Interface?  TODO
127   public void resetToBeforeFirstEntry() {
128     beforeFirst = true;
129     afterLast = false;
130     rowNodeStackIndex = -1;
131     currentRowNode = null;
132     rowLength = 0;
133     familyOffset = familyBuffer.length;
134     familyLength = 0;
135     qualifierOffset = blockMeta.getMaxQualifierLength();
136     qualifierLength = 0;
137     nubCellsRemain = false;
138     currentCellIndex = -1;
139     timestamp = -1L;
140     type = DEFAULT_TYPE;
141     absoluteValueOffset = 0;//use 0 vs -1 so the cell is valid when value hasn't been initialized
142     valueLength = 0;// had it at -1, but that causes null Cell to add up to the wrong length
143     tagsOffset = blockMeta.getMaxTagsLength();
144     tagsLength = 0;
145   }
146 
147   /**
148    * Call this before putting the scanner back into a pool so it doesn't hold the last used block
149    * in memory.
150    */
151   public void releaseBlockReference(){
152     block = null;
153   }
154 
155 
156   /********************** CellScanner **********************/
157 
158   @Override
159   public Cell current() {
160     if(isOutOfBounds()){
161       return null;
162     }
163     return (Cell)this;
164   }
165 
166   /******************* Object methods ************************/
167 
168   @Override
169   public boolean equals(Object obj) {
170     //trivial override to confirm intent (findbugs)
171     return super.equals(obj);
172   }
173   
174   @Override
175   public int hashCode() {
176     return super.hashCode();
177   }
178 
179   /**
180    * Override PrefixTreeCell.toString() with a check to see if the current cell is valid.
181    */
182   @Override
183   public String toString() {
184     Cell currentCell = current();
185     if(currentCell==null){
186       return "null";
187     }
188     return ((PrefixTreeCell)currentCell).getKeyValueString();
189   }
190 
191 
192   /******************* advance ***************************/
193 
194   public boolean positionAtFirstCell() {
195     reInitFirstNode();
196     return advance();
197   }
198 
199   @Override
200   public boolean advance() {
201     if (afterLast) {
202       return false;
203     }
204     if (!hasOccurrences()) {
205       resetToBeforeFirstEntry();
206     }
207     if (beforeFirst || isLastCellInRow()) {
208       nextRow();
209       if (afterLast) {
210         return false;
211       }
212     } else {
213       ++currentCellIndex;
214     }
215 
216     populateNonRowFields(currentCellIndex);
217     return true;
218   }
219 
220 
221   public boolean nextRow() {
222     nextRowInternal();
223     if (afterLast) {
224       return false;
225     }
226     populateNonRowFields(currentCellIndex);
227     return true;
228   }
229 
230 
231   /**
232    * This method is safe to call when the scanner is not on a fully valid row node, as in the case
233    * of a row token miss in the Searcher
234    * @return true if we are positioned on a valid row, false if past end of block
235    */
236   protected boolean nextRowInternal() {
237     if (afterLast) {
238       return false;
239     }
240     if (beforeFirst) {
241       initFirstNode();
242       if (currentRowNode.hasOccurrences()) {
243         if (currentRowNode.isNub()) {
244           nubCellsRemain = true;
245         }
246         currentCellIndex = 0;
247         return true;
248       }
249     }
250     if (currentRowNode.isLeaf()) {
251       discardCurrentRowNode(true);
252     }
253     while (!afterLast) {
254       if (nubCellsRemain) {
255         nubCellsRemain = false;
256       }
257       if (currentRowNode.hasMoreFanNodes()) {
258         followNextFan();
259         if (currentRowNode.hasOccurrences()) {
260           // found some values
261           currentCellIndex = 0;
262           return true;
263         }
264       } else {
265         if (movedToPrevious && currentRowNode.hasOccurrences()
266             && currentRowNode.getFanIndex() == getNextFanIndex()) {
267           followFan(getNextFanIndex());
268         } else {
269           discardCurrentRowNode(true);
270         }
271       }
272     }
273     return false;// went past the end
274   }
275 
276 
277   /**************** secondary traversal methods ******************************/
278 
279   protected void reInitFirstNode() {
280     resetToBeforeFirstEntry();
281     initFirstNode();
282   }
283 
284   protected void initFirstNode() {
285     int offsetIntoUnderlyingStructure = blockMeta.getAbsoluteRowOffset();
286     rowNodeStackIndex = 0;
287     currentRowNode = rowNodes[0];
288     currentRowNode.initOnBlock(blockMeta, block, offsetIntoUnderlyingStructure);
289     appendCurrentTokenToRowBuffer();
290     beforeFirst = false;
291   }
292 
293   protected void followFirstFan() {
294     followFan(0);
295   }
296 
297   protected void followPreviousFan() {
298     int nextFanPosition = currentRowNode.getFanIndex() - 1;
299     followFan(nextFanPosition);
300   }
301 
302   protected void followCurrentFan() {
303     int currentFanPosition = currentRowNode.getFanIndex();
304     followFan(currentFanPosition);
305   }
306 
307   protected int getNextFanIndex() {
308     return rowNodes[rowNodeStackIndex + 1].getFanIndex();
309   }
310 
311   protected void followNextFan() {
312     int nextFanPosition = currentRowNode.getFanIndex() + 1;
313     followFan(nextFanPosition);
314   }
315 
316   protected void followLastFan() {
317     followFan(currentRowNode.getLastFanIndex());
318   }
319 
320   protected void followFan(int fanIndex) {
321     currentRowNode.setFanIndex(fanIndex);
322     appendToRowBuffer(currentRowNode.getFanByte(fanIndex));
323 
324     int nextOffsetIntoUnderlyingStructure = currentRowNode.getOffset()
325         + currentRowNode.getNextNodeOffset(fanIndex, blockMeta);
326     ++rowNodeStackIndex;
327 
328     currentRowNode = rowNodes[rowNodeStackIndex];
329     currentRowNode.initOnBlock(blockMeta, block, nextOffsetIntoUnderlyingStructure);
330 
331     //TODO getToken is spewing garbage
332     appendCurrentTokenToRowBuffer();
333     if (currentRowNode.isNub()) {
334       nubCellsRemain = true;
335     }
336     currentCellIndex = 0;
337   }
338 
339   /**
340    * @param forwards which marker to set if we overflow
341    */
342   protected void discardCurrentRowNode(boolean forwards) {
343     RowNodeReader rowNodeBeingPopped = currentRowNode;
344     --rowNodeStackIndex;// pop it off the stack
345     if (rowNodeStackIndex < 0) {
346       currentRowNode = null;
347       if (forwards) {
348         markAfterLast();
349       } else {
350         markBeforeFirst();
351       }
352       return;
353     }
354     popFromRowBuffer(rowNodeBeingPopped);
355     currentRowNode = rowNodes[rowNodeStackIndex];
356   }
357 
358   protected void markBeforeFirst() {
359     beforeFirst = true;
360     afterLast = false;
361     currentRowNode = null;
362   }
363 
364   protected void markAfterLast() {
365     beforeFirst = false;
366     afterLast = true;
367     currentRowNode = null;
368   }
369 
370 
371   /***************** helper methods **************************/
372 
373   protected void appendCurrentTokenToRowBuffer() {
374     System.arraycopy(block, currentRowNode.getTokenArrayOffset(), rowBuffer, rowLength, 
375       currentRowNode.getTokenLength());
376     rowLength += currentRowNode.getTokenLength();
377   }
378 
379   protected void appendToRowBuffer(byte b) {
380     rowBuffer[rowLength] = b;
381     ++rowLength;
382   }
383 
384   protected void popFromRowBuffer(RowNodeReader rowNodeBeingPopped) {
385     rowLength -= rowNodeBeingPopped.getTokenLength();
386     --rowLength; // pop the parent's fan byte
387   }
388 
389   protected boolean hasOccurrences() {
390     return currentRowNode != null && currentRowNode.hasOccurrences();
391   }
392 
393   protected boolean isBranch() {
394     return currentRowNode != null && !currentRowNode.hasOccurrences()
395         && currentRowNode.hasChildren();
396   }
397 
398   protected boolean isNub() {
399     return currentRowNode != null && currentRowNode.hasOccurrences()
400         && currentRowNode.hasChildren();
401   }
402 
403   protected boolean isLeaf() {
404     return currentRowNode != null && currentRowNode.hasOccurrences()
405         && !currentRowNode.hasChildren();
406   }
407 
408   //TODO expose this in a PrefixTreeScanner interface
409   public boolean isBeforeFirst(){
410     return beforeFirst;
411   }
412 
413   public boolean isAfterLast(){
414     return afterLast;
415   }
416 
417   protected boolean isOutOfBounds(){
418     return beforeFirst || afterLast;
419   }
420 
421   protected boolean isFirstCellInRow() {
422     return currentCellIndex == 0;
423   }
424 
425   protected boolean isLastCellInRow() {
426     return currentCellIndex == currentRowNode.getLastCellIndex();
427   }
428 
429 
430   /********************* fill in family/qualifier/ts/type/value ************/
431 
432   protected int populateNonRowFieldsAndCompareTo(int cellNum, Cell key) {
433     populateNonRowFields(cellNum);
434     return CellComparator.compareStatic(this, key, true);
435   }
436 
437   protected void populateFirstNonRowFields() {
438     populateNonRowFields(0);
439   }
440 
441   protected void populatePreviousNonRowFields() {
442     populateNonRowFields(currentCellIndex - 1);
443   }
444 
445   protected void populateLastNonRowFields() {
446     populateNonRowFields(currentRowNode.getLastCellIndex());
447   }
448 
449   protected void populateNonRowFields(int cellIndex) {
450     currentCellIndex = cellIndex;
451     populateFamily();
452     populateQualifier();
453     // Read tags only if there are tags in the meta
454     if(blockMeta.getNumTagsBytes() != 0) {
455       populateTag();
456     }
457     populateTimestamp();
458     populateMvccVersion();
459     populateType();
460     populateValueOffsets();
461   }
462 
463   protected void populateFamily() {
464     int familyTreeIndex = currentRowNode.getFamilyOffset(currentCellIndex, blockMeta);
465     familyOffset = familyReader.populateBuffer(familyTreeIndex).getColumnOffset();
466     familyLength = familyReader.getColumnLength();
467   }
468 
469   protected void populateQualifier() {
470     int qualifierTreeIndex = currentRowNode.getColumnOffset(currentCellIndex, blockMeta);
471     qualifierOffset = qualifierReader.populateBuffer(qualifierTreeIndex).getColumnOffset();
472     qualifierLength = qualifierReader.getColumnLength();
473   }
474 
475   protected void populateTag() {
476     int tagTreeIndex = currentRowNode.getTagOffset(currentCellIndex, blockMeta);
477     tagsOffset = tagsReader.populateBuffer(tagTreeIndex).getColumnOffset();
478     tagsLength = tagsReader.getColumnLength();
479   }
480 
481   protected void populateTimestamp() {
482     if (blockMeta.isAllSameTimestamp()) {
483       timestamp = blockMeta.getMinTimestamp();
484     } else {
485       int timestampIndex = currentRowNode.getTimestampIndex(currentCellIndex, blockMeta);
486       timestamp = timestampDecoder.getLong(timestampIndex);
487     }
488   }
489 
490   protected void populateMvccVersion() {
491     if (blockMeta.isAllSameMvccVersion()) {
492       mvccVersion = blockMeta.getMinMvccVersion();
493     } else {
494       int mvccVersionIndex = currentRowNode.getMvccVersionIndex(currentCellIndex,
495         blockMeta);
496       mvccVersion = mvccVersionDecoder.getMvccVersion(mvccVersionIndex);
497     }
498   }
499 
500   protected void populateType() {
501     int typeInt;
502     if (blockMeta.isAllSameType()) {
503       typeInt = blockMeta.getAllTypes();
504     } else {
505       typeInt = currentRowNode.getType(currentCellIndex, blockMeta);
506     }
507     type = PrefixTreeCell.TYPES[typeInt];
508   }
509 
510   protected void populateValueOffsets() {
511     int offsetIntoValueSection = currentRowNode.getValueOffset(currentCellIndex, blockMeta);
512     absoluteValueOffset = blockMeta.getAbsoluteValueOffset() + offsetIntoValueSection;
513     valueLength = currentRowNode.getValueLength(currentCellIndex, blockMeta);
514   }
515 
516   /**************** getters ***************************/
517 
518   public byte[] getTreeBytes() {
519     return block;
520   }
521 
522   public PrefixTreeBlockMeta getBlockMeta() {
523     return blockMeta;
524   }
525 
526   public int getMaxRowTreeStackNodes() {
527     return rowNodes.length;
528   }
529 
530   public int getRowBufferLength() {
531     return rowBuffer.length;
532   }
533 
534   public int getQualifierBufferLength() {
535     return qualifierBuffer.length;
536   }
537 
538   public int getTagBufferLength() {
539     return tagsBuffer.length;
540   }
541 
542   public void setMovedToPreviousAsPartOfSeek(boolean movedToPrevious) {
543     this.movedToPrevious = movedToPrevious;
544   }
545 
546   public boolean hasMovedToPreviousAsPartOfSeek() {
547     return this.movedToPrevious;
548   }
549 
550 }