View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.hadoop.hbase.classification.InterfaceAudience;
26  import org.apache.hadoop.hbase.util.ByteBufferUtils;
27  import org.apache.hadoop.hbase.util.Bytes;
28  import org.apache.hadoop.hbase.util.IterableUtils;
29  import org.apache.hadoop.hbase.util.SimpleByteRange;
30  import org.apache.hadoop.io.WritableUtils;
31  
32  import com.google.common.base.Function;
33  import com.google.common.collect.Lists;
34  
35  /**
36   * static convenience methods for dealing with KeyValues and collections of KeyValues
37   */
38  @InterfaceAudience.Private
39  public class KeyValueUtil {
40  
41    /**************** length *********************/
42  
43    public static int length(final Cell cell) {
44      return (int) (KeyValue.getKeyValueDataStructureSize(cell.getRowLength(),
45          cell.getFamilyLength(), cell.getQualifierLength(), cell.getValueLength(),
46          cell.getTagsLengthUnsigned()));
47    }
48  
49    public static int keyLength(final Cell cell) {
50      return (int)KeyValue.getKeyDataStructureSize(cell.getRowLength(), cell.getFamilyLength(),
51        cell.getQualifierLength());
52    }
53  
54    public static int lengthWithMvccVersion(final KeyValue kv, final boolean includeMvccVersion) {
55      int length = kv.getLength();
56      if (includeMvccVersion) {
57        length += WritableUtils.getVIntSize(kv.getMvccVersion());
58      }
59      return length;
60    }
61  
62    public static int totalLengthWithMvccVersion(final Iterable<? extends KeyValue> kvs,
63        final boolean includeMvccVersion) {
64      int length = 0;
65      for (KeyValue kv : IterableUtils.nullSafe(kvs)) {
66        length += lengthWithMvccVersion(kv, includeMvccVersion);
67      }
68      return length;
69    }
70  
71  
72    /**************** copy key only *********************/
73  
74    public static KeyValue copyToNewKeyValue(final Cell cell) {
75      byte[] bytes = copyToNewByteArray(cell);
76      KeyValue kvCell = new KeyValue(bytes, 0, bytes.length);
77      kvCell.setMvccVersion(cell.getMvccVersion());
78      return kvCell;
79    }
80  
81    public static ByteBuffer copyKeyToNewByteBuffer(final Cell cell) {
82      byte[] bytes = new byte[keyLength(cell)];
83      appendKeyToByteArrayWithoutValue(cell, bytes, 0);
84      ByteBuffer buffer = ByteBuffer.wrap(bytes);
85      return buffer;
86    }
87  
88    public static byte[] copyToNewByteArray(final Cell cell) {
89      int v1Length = length(cell);
90      byte[] backingBytes = new byte[v1Length];
91      appendToByteArray(cell, backingBytes, 0);
92      return backingBytes;
93    }
94  
95    protected static int appendKeyToByteArrayWithoutValue(final Cell cell, final byte[] output,
96        final int offset) {
97      int nextOffset = offset;
98      nextOffset = Bytes.putShort(output, nextOffset, cell.getRowLength());
99      nextOffset = CellUtil.copyRowTo(cell, output, nextOffset);
100     nextOffset = Bytes.putByte(output, nextOffset, cell.getFamilyLength());
101     nextOffset = CellUtil.copyFamilyTo(cell, output, nextOffset);
102     nextOffset = CellUtil.copyQualifierTo(cell, output, nextOffset);
103     nextOffset = Bytes.putLong(output, nextOffset, cell.getTimestamp());
104     nextOffset = Bytes.putByte(output, nextOffset, cell.getTypeByte());
105     return nextOffset;
106   }
107 
108 
109   /**************** copy key and value *********************/
110 
111   public static int appendToByteArray(final Cell cell, final byte[] output, final int offset) {
112     int pos = offset;
113     pos = Bytes.putInt(output, pos, keyLength(cell));
114     pos = Bytes.putInt(output, pos, cell.getValueLength());
115     pos = appendKeyToByteArrayWithoutValue(cell, output, pos);
116     pos = CellUtil.copyValueTo(cell, output, pos);
117     if ((cell.getTagsLengthUnsigned() > 0)) {
118       pos = Bytes.putAsShort(output, pos, cell.getTagsLengthUnsigned());
119       pos = CellUtil.copyTagTo(cell, output, pos);
120     }
121     return pos;
122   }
123 
124   public static ByteBuffer copyToNewByteBuffer(final Cell cell) {
125     byte[] bytes = new byte[length(cell)];
126     appendToByteArray(cell, bytes, 0);
127     ByteBuffer buffer = ByteBuffer.wrap(bytes);
128     return buffer;
129   }
130 
131   public static void appendToByteBuffer(final ByteBuffer bb, final KeyValue kv,
132       final boolean includeMvccVersion) {
133     // keep pushing the limit out. assume enough capacity
134     bb.limit(bb.position() + kv.getLength());
135     bb.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
136     if (includeMvccVersion) {
137       int numMvccVersionBytes = WritableUtils.getVIntSize(kv.getMvccVersion());
138       ByteBufferUtils.extendLimit(bb, numMvccVersionBytes);
139       ByteBufferUtils.writeVLong(bb, kv.getMvccVersion());
140     }
141   }
142 
143 
144   /**************** iterating *******************************/
145 
146   /**
147    * Creates a new KeyValue object positioned in the supplied ByteBuffer and sets the ByteBuffer's
148    * position to the start of the next KeyValue. Does not allocate a new array or copy data.
149    * @param bb
150    * @param includesMvccVersion
151    * @param includesTags
152    */
153   public static KeyValue nextShallowCopy(final ByteBuffer bb, final boolean includesMvccVersion,
154       boolean includesTags) {
155     if (bb.isDirect()) {
156       throw new IllegalArgumentException("only supports heap buffers");
157     }
158     if (bb.remaining() < 1) {
159       return null;
160     }
161     KeyValue keyValue = null;
162     int underlyingArrayOffset = bb.arrayOffset() + bb.position();
163     int keyLength = bb.getInt();
164     int valueLength = bb.getInt();
165     ByteBufferUtils.skip(bb, keyLength + valueLength);
166     int tagsLength = 0;
167     if (includesTags) {
168       // Read short as unsigned, high byte first
169       tagsLength = ((bb.get() & 0xff) << 8) ^ (bb.get() & 0xff);
170       ByteBufferUtils.skip(bb, tagsLength);
171     }
172     int kvLength = (int) KeyValue.getKeyValueDataStructureSize(keyLength, valueLength, tagsLength);
173     keyValue = new KeyValue(bb.array(), underlyingArrayOffset, kvLength);
174     if (includesMvccVersion) {
175       long mvccVersion = ByteBufferUtils.readVLong(bb);
176       keyValue.setMvccVersion(mvccVersion);
177     }
178     return keyValue;
179   }
180 
181 
182   /*************** next/previous **********************************/
183 
184   /**
185    * Append single byte 0x00 to the end of the input row key
186    */
187   public static KeyValue createFirstKeyInNextRow(final Cell in){
188     byte[] nextRow = new byte[in.getRowLength() + 1];
189     System.arraycopy(in.getRowArray(), in.getRowOffset(), nextRow, 0, in.getRowLength());
190     nextRow[nextRow.length - 1] = 0;//maybe not necessary
191     return KeyValue.createFirstOnRow(nextRow);
192   }
193 
194   /**
195    * Increment the row bytes and clear the other fields
196    */
197   public static KeyValue createFirstKeyInIncrementedRow(final Cell in){
198     byte[] thisRow = new SimpleByteRange(in.getRowArray(), in.getRowOffset(), in.getRowLength())
199         .deepCopyToNewArray();
200     byte[] nextRow = Bytes.unsignedCopyAndIncrement(thisRow);
201     return KeyValue.createFirstOnRow(nextRow);
202   }
203 
204   /**
205    * Decrement the timestamp.  For tests (currently wasteful)
206    *
207    * Remember timestamps are sorted reverse chronologically.
208    * @param in
209    * @return previous key
210    */
211   public static KeyValue previousKey(final KeyValue in) {
212     return KeyValue.createFirstOnRow(CellUtil.cloneRow(in), CellUtil.cloneFamily(in),
213       CellUtil.cloneQualifier(in), in.getTimestamp() - 1);
214   }
215 
216   /*************** misc **********************************/
217   /**
218    * @param cell
219    * @return <code>cell<code> if it is an instance of {@link KeyValue} else we will return a
220    * new {@link KeyValue} instance made from <code>cell</code>
221    */
222   public static KeyValue ensureKeyValue(final Cell cell) {
223     if (cell == null) return null;
224     return cell instanceof KeyValue? (KeyValue)cell: copyToNewKeyValue(cell);
225   }
226 
227   public static List<KeyValue> ensureKeyValues(List<Cell> cells) {
228     List<KeyValue> lazyList = Lists.transform(cells, new Function<Cell, KeyValue>() {
229       @Override
230       public KeyValue apply(Cell arg0) {
231         return KeyValueUtil.ensureKeyValue(arg0);
232       }
233     });
234     return new ArrayList<KeyValue>(lazyList);
235   }
236 }