View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import java.util.UUID;
30  
31  import org.apache.hadoop.hbase.classification.InterfaceAudience;
32  import org.apache.hadoop.hbase.classification.InterfaceStability;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.CellScannable;
35  import org.apache.hadoop.hbase.CellScanner;
36  import org.apache.hadoop.hbase.CellUtil;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.KeyValueUtil;
40  import org.apache.hadoop.hbase.Tag;
41  import org.apache.hadoop.hbase.exceptions.DeserializationException;
42  import org.apache.hadoop.hbase.io.HeapSize;
43  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44  import org.apache.hadoop.hbase.security.access.AccessControlConstants;
45  import org.apache.hadoop.hbase.security.access.Permission;
46  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
47  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.ClassSize;
50  
51  import com.google.common.collect.ArrayListMultimap;
52  import com.google.common.collect.ListMultimap;
53  import com.google.common.collect.Lists;
54  import com.google.common.io.ByteArrayDataInput;
55  import com.google.common.io.ByteArrayDataOutput;
56  import com.google.common.io.ByteStreams;
57  
58  @InterfaceAudience.Public
59  @InterfaceStability.Evolving
60  public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable,
61      HeapSize {
62    public static final long MUTATION_OVERHEAD = ClassSize.align(
63        // This
64        ClassSize.OBJECT +
65        // row + OperationWithAttributes.attributes
66        2 * ClassSize.REFERENCE +
67        // Timestamp
68        1 * Bytes.SIZEOF_LONG +
69        // durability
70        ClassSize.REFERENCE +
71        // familyMap
72        ClassSize.REFERENCE +
73        // familyMap
74        ClassSize.TREEMAP);
75  
76    /**
77     * The attribute for storing the list of clusters that have consumed the change.
78     */
79    private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
80  
81    /**
82     * The attribute for storing TTL for the result of the mutation.
83     */
84    private static final String OP_ATTRIBUTE_TTL = "_ttl";
85  
86    protected byte [] row = null;
87    protected long ts = HConstants.LATEST_TIMESTAMP;
88    protected Durability durability = Durability.USE_DEFAULT;
89  
90    // A Map sorted by column family.
91    protected NavigableMap<byte [], List<Cell>> familyMap =
92      new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
93  
94    @Override
95    public CellScanner cellScanner() {
96      return CellUtil.createCellScanner(getFamilyCellMap());
97    }
98  
99    /**
100    * Creates an empty list if one doesn't exist for the given column family
101    * or else it returns the associated list of Cell objects.
102    *
103    * @param family column family
104    * @return a list of Cell objects, returns an empty list if one doesn't exist.
105    */
106   List<Cell> getCellList(byte[] family) {
107     List<Cell> list = this.familyMap.get(family);
108     if (list == null) {
109       list = new ArrayList<Cell>();
110     }
111     return list;
112   }
113 
114   /*
115    * Create a KeyValue with this objects row key and the Put identifier.
116    *
117    * @return a KeyValue with this objects row key and the Put identifier.
118    */
119   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
120     return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
121   }
122 
123   /**
124    * Create a KeyValue with this objects row key and the Put identifier.
125    * @param family
126    * @param qualifier
127    * @param ts
128    * @param value
129    * @param tags - Specify the Tags as an Array {@link KeyValue.Tag}
130    * @return a KeyValue with this objects row key and the Put identifier.
131    */
132   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) {
133     KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags);
134     return kvWithTag;
135   }
136 
137   /*
138    * Create a KeyValue with this objects row key and the Put identifier.
139    *
140    * @return a KeyValue with this objects row key and the Put identifier.
141    */
142   KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value,
143                              Tag[] tags) {
144     return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
145         family, 0, family == null ? 0 : family.length,
146         qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null);
147   }
148 
149   /**
150    * Compile the column family (i.e. schema) information
151    * into a Map. Useful for parsing and aggregation by debugging,
152    * logging, and administration tools.
153    * @return Map
154    */
155   @Override
156   public Map<String, Object> getFingerprint() {
157     Map<String, Object> map = new HashMap<String, Object>();
158     List<String> families = new ArrayList<String>();
159     // ideally, we would also include table information, but that information
160     // is not stored in each Operation instance.
161     map.put("families", families);
162     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
163       families.add(Bytes.toStringBinary(entry.getKey()));
164     }
165     return map;
166   }
167 
168   /**
169    * Compile the details beyond the scope of getFingerprint (row, columns,
170    * timestamps, etc.) into a Map along with the fingerprinted information.
171    * Useful for debugging, logging, and administration tools.
172    * @param maxCols a limit on the number of columns output prior to truncation
173    * @return Map
174    */
175   @Override
176   public Map<String, Object> toMap(int maxCols) {
177     // we start with the fingerprint map and build on top of it.
178     Map<String, Object> map = getFingerprint();
179     // replace the fingerprint's simple list of families with a
180     // map from column families to lists of qualifiers and kv details
181     Map<String, List<Map<String, Object>>> columns =
182       new HashMap<String, List<Map<String, Object>>>();
183     map.put("families", columns);
184     map.put("row", Bytes.toStringBinary(this.row));
185     int colCount = 0;
186     // iterate through all column families affected
187     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
188       // map from this family to details for each cell affected within the family
189       List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
190       columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
191       colCount += entry.getValue().size();
192       if (maxCols <= 0) {
193         continue;
194       }
195       // add details for each cell
196       for (Cell cell: entry.getValue()) {
197         if (--maxCols <= 0 ) {
198           continue;
199         }
200         Map<String, Object> cellMap = cellToStringMap(cell);
201         qualifierDetails.add(cellMap);
202       }
203     }
204     map.put("totalColumns", colCount);
205     // add the id if set
206     if (getId() != null) {
207       map.put("id", getId());
208     }
209     // Add the TTL if set
210     // Long.MAX_VALUE is the default, and is interpreted to mean this attribute
211     // has not been set.
212     if (getTTL() != Long.MAX_VALUE) {
213       map.put("ttl", getTTL());
214     }
215     return map;
216   }
217 
218   private static Map<String, Object> cellToStringMap(Cell c) {
219     Map<String, Object> stringMap = new HashMap<String, Object>();
220     stringMap.put("qualifier", Bytes.toStringBinary(c.getQualifierArray(), c.getQualifierOffset(),
221                 c.getQualifierLength()));
222     stringMap.put("timestamp", c.getTimestamp());
223     stringMap.put("vlen", c.getValueLength());
224     List<Tag> tags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength());
225     if (tags != null) {
226       List<String> tagsString = new ArrayList<String>();
227       for (Tag t : tags) {
228         tagsString.add((t.getType()) + ":" + Bytes.toStringBinary(t.getValue()));
229       }
230       stringMap.put("tag", tagsString);
231     }
232     return stringMap;
233   }
234 
235   /**
236    * @deprecated Use {@link #getDurability()} instead.
237    * @return true if edits should be applied to WAL, false if not
238    */
239   @Deprecated
240   public boolean getWriteToWAL() {
241     return this.durability == Durability.SKIP_WAL;
242   }
243 
244   /**
245    * Set whether this Delete should be written to the WAL or not.
246    * Not writing the WAL means you may lose edits on server crash.
247    * This method will reset any changes made via {@link #setDurability(Durability)}
248    * @param write true if edits should be written to WAL, false if not
249    * @deprecated Use {@link #setDurability(Durability)} instead.
250    */
251   @Deprecated
252   public Mutation setWriteToWAL(boolean write) {
253     setDurability(write ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
254     return this;
255   }
256 
257   /**
258    * Set the durability for this mutation
259    * @param d
260    */
261   public Mutation setDurability(Durability d) {
262     this.durability = d;
263     return this;
264   }
265 
266   /** Get the current durability */
267   public Durability getDurability() {
268     return this.durability;
269   }
270 
271   /**
272    * Method for retrieving the put's familyMap
273    * @return familyMap
274    */
275   public NavigableMap<byte [], List<Cell>> getFamilyCellMap() {
276     return this.familyMap;
277   }
278 
279   /**
280    * Method for setting the put's familyMap
281    */
282   public Mutation setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) {
283     // TODO: Shut this down or move it up to be a Constructor.  Get new object rather than change
284     // this internal data member.
285     this.familyMap = map;
286     return this;
287   }
288 
289   /**
290    * Method for retrieving the put's familyMap that is deprecated and inefficient.
291    * @return the map
292    * @deprecated use {@link #getFamilyCellMap()} instead.
293    */
294   @Deprecated
295   public NavigableMap<byte [], List<KeyValue>> getFamilyMap() {
296     TreeMap<byte[], List<KeyValue>> fm =
297         new TreeMap<byte[], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
298     for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {
299       List<KeyValue> kvl = new ArrayList<KeyValue>(e.getValue().size());
300       for (Cell c : e.getValue()) {
301         kvl.add(KeyValueUtil.ensureKeyValue(c));
302       }
303       fm.put(e.getKey(), kvl);
304     }
305     return fm;
306   }
307 
308   /**
309    * Method for setting the put's familyMap that is deprecated and inefficient.
310    * @deprecated use {@link #setFamilyCellMap(NavigableMap)} instead.
311    */
312   @Deprecated
313   public Mutation setFamilyMap(NavigableMap<byte [], List<KeyValue>> map) {
314     TreeMap<byte[], List<Cell>> fm = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
315     for (Map.Entry<byte[], List<KeyValue>> e : map.entrySet()) {
316       fm.put(e.getKey(), Lists.<Cell>newArrayList(e.getValue()));
317     }
318     this.familyMap = fm;
319     return this;
320   }
321 
322   /**
323    * Method to check if the familyMap is empty
324    * @return true if empty, false otherwise
325    */
326   public boolean isEmpty() {
327     return familyMap.isEmpty();
328   }
329 
330   /**
331    * Method for retrieving the delete's row
332    * @return row
333    */
334   @Override
335   public byte [] getRow() {
336     return this.row;
337   }
338 
339   @Override
340   public int compareTo(final Row d) {
341     return Bytes.compareTo(this.getRow(), d.getRow());
342   }
343 
344   /**
345    * Method for retrieving the timestamp
346    * @return timestamp
347    */
348   public long getTimeStamp() {
349     return this.ts;
350   }
351 
352   /**
353    * Marks that the clusters with the given clusterIds have consumed the mutation
354    * @param clusterIds of the clusters that have consumed the mutation
355    */
356   public Mutation setClusterIds(List<UUID> clusterIds) {
357     ByteArrayDataOutput out = ByteStreams.newDataOutput();
358     out.writeInt(clusterIds.size());
359     for (UUID clusterId : clusterIds) {
360       out.writeLong(clusterId.getMostSignificantBits());
361       out.writeLong(clusterId.getLeastSignificantBits());
362     }
363     setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
364     return this;
365   }
366 
367   /**
368    * @return the set of clusterIds that have consumed the mutation
369    */
370   public List<UUID> getClusterIds() {
371     List<UUID> clusterIds = new ArrayList<UUID>();
372     byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
373     if(bytes != null) {
374       ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
375       int numClusters = in.readInt();
376       for(int i=0; i<numClusters; i++){
377         clusterIds.add(new UUID(in.readLong(), in.readLong()));
378       }
379     }
380     return clusterIds;
381   }
382 
383   /**
384    * Sets the visibility expression associated with cells in this Mutation.
385    * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation.
386    * @param expression
387    */
388   public Mutation setCellVisibility(CellVisibility expression) {
389     this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil
390         .toCellVisibility(expression).toByteArray());
391     return this;
392   }
393 
394   /**
395    * @return CellVisibility associated with cells in this Mutation.
396    * @throws DeserializationException
397    */
398   public CellVisibility getCellVisibility() throws DeserializationException {
399     byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY);
400     if (cellVisibilityBytes == null) return null;
401     return ProtobufUtil.toCellVisibility(cellVisibilityBytes);
402   }
403 
404   /**
405    * Number of KeyValues carried by this Mutation.
406    * @return the total number of KeyValues
407    */
408   public int size() {
409     int size = 0;
410     for (List<Cell> cells : this.familyMap.values()) {
411       size += cells.size();
412     }
413     return size;
414   }
415 
416   /**
417    * @return the number of different families
418    */
419   public int numFamilies() {
420     return familyMap.size();
421   }
422 
423   /**
424    * @return Calculate what Mutation adds to class heap size.
425    */
426   @Override
427   public long heapSize() {
428     long heapsize = MUTATION_OVERHEAD;
429     // Adding row
430     heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
431 
432     // Adding map overhead
433     heapsize +=
434       ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
435     for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
436       //Adding key overhead
437       heapsize +=
438         ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
439 
440       //This part is kinds tricky since the JVM can reuse references if you
441       //store the same value, but have a good match with SizeOf at the moment
442       //Adding value overhead
443       heapsize += ClassSize.align(ClassSize.ARRAYLIST);
444       int size = entry.getValue().size();
445       heapsize += ClassSize.align(ClassSize.ARRAY +
446           size * ClassSize.REFERENCE);
447 
448       for(Cell cell : entry.getValue()) {
449         heapsize += CellUtil.estimatedHeapSizeOf(cell);
450       }
451     }
452     heapsize += getAttributeSize();
453     heapsize += extraHeapSize();
454     return ClassSize.align(heapsize);
455   }
456 
457   /**
458    * @return The serialized ACL for this operation, or null if none
459    */
460   public byte[] getACL() {
461     return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
462   }
463 
464   /**
465    * @param user User short name
466    * @param perms Permissions for the user
467    */
468   public Mutation setACL(String user, Permission perms) {
469     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
470       ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray());
471     return this;
472   }
473 
474   /**
475    * @param perms A map of permissions for a user or users
476    */
477   public Mutation setACL(Map<String, Permission> perms) {
478     ListMultimap<String, Permission> permMap = ArrayListMultimap.create();
479     for (Map.Entry<String, Permission> entry : perms.entrySet()) {
480       permMap.put(entry.getKey(), entry.getValue());
481     }
482     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
483       ProtobufUtil.toUsersAndPermissions(permMap).toByteArray());
484     return this;
485   }
486 
487   /**
488    * Return the TTL requested for the result of the mutation, in milliseconds.
489    * @return the TTL requested for the result of the mutation, in milliseconds,
490    * or Long.MAX_VALUE if unset
491    */
492   public long getTTL() {
493     byte[] ttlBytes = getAttribute(OP_ATTRIBUTE_TTL);
494     if (ttlBytes != null) {
495       return Bytes.toLong(ttlBytes);
496     }
497     return Long.MAX_VALUE;
498   }
499 
500   /**
501    * Set the TTL desired for the result of the mutation, in milliseconds.
502    * @param ttl the TTL desired for the result of the mutation, in milliseconds
503    * @return this
504    */
505   public Mutation setTTL(long ttl) {
506     setAttribute(OP_ATTRIBUTE_TTL, Bytes.toBytes(ttl));
507     return this;
508   }
509 
510   /**
511    * Subclasses should override this method to add the heap size of their own fields.
512    * @return the heap size to add (will be aligned).
513    */
514   protected long extraHeapSize(){
515     return 0L;
516   }
517 
518 
519   /**
520    * @param row Row to check
521    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
522    * &gt; {@link HConstants#MAX_ROW_LENGTH}
523    * @return <code>row</code>
524    */
525   static byte [] checkRow(final byte [] row) {
526     return checkRow(row, 0, row == null? 0: row.length);
527   }
528 
529   /**
530    * @param row Row to check
531    * @param offset
532    * @param length
533    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
534    * &gt; {@link HConstants#MAX_ROW_LENGTH}
535    * @return <code>row</code>
536    */
537   static byte [] checkRow(final byte [] row, final int offset, final int length) {
538     if (row == null) {
539       throw new IllegalArgumentException("Row buffer is null");
540     }
541     if (length == 0) {
542       throw new IllegalArgumentException("Row length is 0");
543     }
544     if (length > HConstants.MAX_ROW_LENGTH) {
545       throw new IllegalArgumentException("Row length " + length + " is > " +
546         HConstants.MAX_ROW_LENGTH);
547     }
548     return row;
549   }
550 
551   static void checkRow(ByteBuffer row) {
552     if (row == null) {
553       throw new IllegalArgumentException("Row buffer is null");
554     }
555     if (row.remaining() == 0) {
556       throw new IllegalArgumentException("Row length is 0");
557     }
558     if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
559       throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
560           HConstants.MAX_ROW_LENGTH);
561     }
562   }
563 }