View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.HashMap;
25  import java.util.List;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import java.util.UUID;
30  
31  import org.apache.hadoop.hbase.classification.InterfaceAudience;
32  import org.apache.hadoop.hbase.classification.InterfaceStability;
33  import org.apache.hadoop.hbase.Cell;
34  import org.apache.hadoop.hbase.CellScannable;
35  import org.apache.hadoop.hbase.CellScanner;
36  import org.apache.hadoop.hbase.CellUtil;
37  import org.apache.hadoop.hbase.HConstants;
38  import org.apache.hadoop.hbase.KeyValue;
39  import org.apache.hadoop.hbase.KeyValueUtil;
40  import org.apache.hadoop.hbase.Tag;
41  import org.apache.hadoop.hbase.exceptions.DeserializationException;
42  import org.apache.hadoop.hbase.io.HeapSize;
43  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44  import org.apache.hadoop.hbase.security.access.AccessControlConstants;
45  import org.apache.hadoop.hbase.security.access.Permission;
46  import org.apache.hadoop.hbase.security.visibility.CellVisibility;
47  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
48  import org.apache.hadoop.hbase.util.Bytes;
49  import org.apache.hadoop.hbase.util.ClassSize;
50  
51  import com.google.common.collect.ArrayListMultimap;
52  import com.google.common.collect.ListMultimap;
53  import com.google.common.collect.Lists;
54  import com.google.common.io.ByteArrayDataInput;
55  import com.google.common.io.ByteArrayDataOutput;
56  import com.google.common.io.ByteStreams;
57  
58  @InterfaceAudience.Public
59  @InterfaceStability.Evolving
60  public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable,
61      HeapSize {
62    public static final long MUTATION_OVERHEAD = ClassSize.align(
63        // This
64        ClassSize.OBJECT +
65        // row + OperationWithAttributes.attributes
66        2 * ClassSize.REFERENCE +
67        // Timestamp
68        1 * Bytes.SIZEOF_LONG +
69        // durability
70        ClassSize.REFERENCE +
71        // familyMap
72        ClassSize.REFERENCE +
73        // familyMap
74        ClassSize.TREEMAP);
75  
76    /**
77     * The attribute for storing the list of clusters that have consumed the change.
78     */
79    private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
80  
81    /**
82     * The attribute for storing TTL for the result of the mutation.
83     */
84    private static final String OP_ATTRIBUTE_TTL = "_ttl";
85  
86    protected byte [] row = null;
87    protected long ts = HConstants.LATEST_TIMESTAMP;
88    protected Durability durability = Durability.USE_DEFAULT;
89  
90    // A Map sorted by column family.
91    protected NavigableMap<byte [], List<Cell>> familyMap =
92      new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
93  
94    @Override
95    public CellScanner cellScanner() {
96      return CellUtil.createCellScanner(getFamilyCellMap());
97    }
98  
99    /**
100    * Creates an empty list if one doesn't exist for the given column family
101    * or else it returns the associated list of Cell objects.
102    *
103    * @param family column family
104    * @return a list of Cell objects, returns an empty list if one doesn't exist.
105    */
106   List<Cell> getCellList(byte[] family) {
107     List<Cell> list = this.familyMap.get(family);
108     if (list == null) {
109       list = new ArrayList<Cell>();
110     }
111     return list;
112   }
113 
114   /*
115    * Create a KeyValue with this objects row key and the Put identifier.
116    *
117    * @return a KeyValue with this objects row key and the Put identifier.
118    */
119   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
120     return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
121   }
122 
123   /**
124    * Create a KeyValue with this objects row key and the Put identifier.
125    * @param family
126    * @param qualifier
127    * @param ts
128    * @param value
129    * @param tags - Specify the Tags as an Array {@link KeyValue.Tag}
130    * @return a KeyValue with this objects row key and the Put identifier.
131    */
132   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) {
133     KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags);
134     return kvWithTag;
135   }
136 
137   /*
138    * Create a KeyValue with this objects row key and the Put identifier.
139    *
140    * @return a KeyValue with this objects row key and the Put identifier.
141    */
142   KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value,
143                              Tag[] tags) {
144     return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
145         family, 0, family == null ? 0 : family.length,
146         qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null);
147   }
148 
149   /**
150    * Compile the column family (i.e. schema) information
151    * into a Map. Useful for parsing and aggregation by debugging,
152    * logging, and administration tools.
153    * @return Map
154    */
155   @Override
156   public Map<String, Object> getFingerprint() {
157     Map<String, Object> map = new HashMap<String, Object>();
158     List<String> families = new ArrayList<String>();
159     // ideally, we would also include table information, but that information
160     // is not stored in each Operation instance.
161     map.put("families", families);
162     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
163       families.add(Bytes.toStringBinary(entry.getKey()));
164     }
165     return map;
166   }
167 
168   /**
169    * Compile the details beyond the scope of getFingerprint (row, columns,
170    * timestamps, etc.) into a Map along with the fingerprinted information.
171    * Useful for debugging, logging, and administration tools.
172    * @param maxCols a limit on the number of columns output prior to truncation
173    * @return Map
174    */
175   @Override
176   public Map<String, Object> toMap(int maxCols) {
177     // we start with the fingerprint map and build on top of it.
178     Map<String, Object> map = getFingerprint();
179     // replace the fingerprint's simple list of families with a
180     // map from column families to lists of qualifiers and kv details
181     Map<String, List<Map<String, Object>>> columns =
182       new HashMap<String, List<Map<String, Object>>>();
183     map.put("families", columns);
184     map.put("row", Bytes.toStringBinary(this.row));
185     int colCount = 0;
186     // iterate through all column families affected
187     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
188       // map from this family to details for each cell affected within the family
189       List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
190       columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
191       colCount += entry.getValue().size();
192       if (maxCols <= 0) {
193         continue;
194       }
195       // add details for each cell
196       for (Cell cell: entry.getValue()) {
197         if (--maxCols <= 0 ) {
198           continue;
199         }
200         // KeyValue v1 expectation.  Cast for now until we go all Cell all the time.
201         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
202         Map<String, Object> kvMap = kv.toStringMap();
203         // row and family information are already available in the bigger map
204         kvMap.remove("row");
205         kvMap.remove("family");
206         qualifierDetails.add(kvMap);
207       }
208     }
209     map.put("totalColumns", colCount);
210     // add the id if set
211     if (getId() != null) {
212       map.put("id", getId());
213     }
214     // Add the TTL if set
215     // Long.MAX_VALUE is the default, and is interpreted to mean this attribute
216     // has not been set.
217     if (getTTL() != Long.MAX_VALUE) {
218       map.put("ttl", getTTL());
219     }
220     return map;
221   }
222 
223   /**
224    * @deprecated Use {@link #getDurability()} instead.
225    * @return true if edits should be applied to WAL, false if not
226    */
227   @Deprecated
228   public boolean getWriteToWAL() {
229     return this.durability == Durability.SKIP_WAL;
230   }
231 
232   /**
233    * Set whether this Delete should be written to the WAL or not.
234    * Not writing the WAL means you may lose edits on server crash.
235    * This method will reset any changes made via {@link #setDurability(Durability)}
236    * @param write true if edits should be written to WAL, false if not
237    * @deprecated Use {@link #setDurability(Durability)} instead.
238    */
239   @Deprecated
240   public void setWriteToWAL(boolean write) {
241     setDurability(write ? Durability.USE_DEFAULT : Durability.SKIP_WAL);
242   }
243 
244   /**
245    * Set the durability for this mutation
246    * @param d
247    */
248   public void setDurability(Durability d) {
249     this.durability = d;
250   }
251 
252   /** Get the current durability */
253   public Durability getDurability() {
254     return this.durability;
255   }
256 
257   /**
258    * Method for retrieving the put's familyMap
259    * @return familyMap
260    */
261   public NavigableMap<byte [], List<Cell>> getFamilyCellMap() {
262     return this.familyMap;
263   }
264 
265   /**
266    * Method for setting the put's familyMap
267    */
268   public void setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) {
269     // TODO: Shut this down or move it up to be a Constructor.  Get new object rather than change
270     // this internal data member.
271     this.familyMap = map;
272   }
273 
274   /**
275    * Method for retrieving the put's familyMap that is deprecated and inefficient.
276    * @return the map
277    * @deprecated use {@link #getFamilyCellMap()} instead.
278    */
279   @Deprecated
280   public NavigableMap<byte [], List<KeyValue>> getFamilyMap() {
281     TreeMap<byte[], List<KeyValue>> fm =
282         new TreeMap<byte[], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
283     for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) {
284       List<KeyValue> kvl = new ArrayList<KeyValue>(e.getValue().size());
285       for (Cell c : e.getValue()) {
286         kvl.add(KeyValueUtil.ensureKeyValue(c));
287       }
288       fm.put(e.getKey(), kvl);
289     }
290     return fm;
291   }
292 
293   /**
294    * Method for setting the put's familyMap that is deprecated and inefficient.
295    * @deprecated use {@link #setFamilyCellMap(NavigableMap)} instead.
296    */
297   @Deprecated
298   public void setFamilyMap(NavigableMap<byte [], List<KeyValue>> map) {
299     TreeMap<byte[], List<Cell>> fm = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
300     for (Map.Entry<byte[], List<KeyValue>> e : map.entrySet()) {
301       fm.put(e.getKey(), Lists.<Cell>newArrayList(e.getValue()));
302     }
303     this.familyMap = fm;
304   }
305 
306   /**
307    * Method to check if the familyMap is empty
308    * @return true if empty, false otherwise
309    */
310   public boolean isEmpty() {
311     return familyMap.isEmpty();
312   }
313 
314   /**
315    * Method for retrieving the delete's row
316    * @return row
317    */
318   @Override
319   public byte [] getRow() {
320     return this.row;
321   }
322 
323   @Override
324   public int compareTo(final Row d) {
325     return Bytes.compareTo(this.getRow(), d.getRow());
326   }
327 
328   /**
329    * Method for retrieving the timestamp
330    * @return timestamp
331    */
332   public long getTimeStamp() {
333     return this.ts;
334   }
335 
336   /**
337    * Marks that the clusters with the given clusterIds have consumed the mutation
338    * @param clusterIds of the clusters that have consumed the mutation
339    */
340   public void setClusterIds(List<UUID> clusterIds) {
341     ByteArrayDataOutput out = ByteStreams.newDataOutput();
342     out.writeInt(clusterIds.size());
343     for (UUID clusterId : clusterIds) {
344       out.writeLong(clusterId.getMostSignificantBits());
345       out.writeLong(clusterId.getLeastSignificantBits());
346     }
347     setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
348   }
349 
350   /**
351    * @return the set of clusterIds that have consumed the mutation
352    */
353   public List<UUID> getClusterIds() {
354     List<UUID> clusterIds = new ArrayList<UUID>();
355     byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
356     if(bytes != null) {
357       ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
358       int numClusters = in.readInt();
359       for(int i=0; i<numClusters; i++){
360         clusterIds.add(new UUID(in.readLong(), in.readLong()));
361       }
362     }
363     return clusterIds;
364   }
365 
366   /**
367    * Sets the visibility expression associated with cells in this Mutation.
368    * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation.
369    * @param expression
370    */
371   public void setCellVisibility(CellVisibility expression) {
372     this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil
373         .toCellVisibility(expression).toByteArray());
374   }
375 
376   /**
377    * @return CellVisibility associated with cells in this Mutation.
378    * @throws DeserializationException
379    */
380   public CellVisibility getCellVisibility() throws DeserializationException {
381     byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY);
382     if (cellVisibilityBytes == null) return null;
383     return ProtobufUtil.toCellVisibility(cellVisibilityBytes);
384   }
385 
386   /**
387    * Number of KeyValues carried by this Mutation.
388    * @return the total number of KeyValues
389    */
390   public int size() {
391     int size = 0;
392     for (List<Cell> cells : this.familyMap.values()) {
393       size += cells.size();
394     }
395     return size;
396   }
397 
398   /**
399    * @return the number of different families
400    */
401   public int numFamilies() {
402     return familyMap.size();
403   }
404 
405   /**
406    * @return Calculate what Mutation adds to class heap size.
407    */
408   @Override
409   public long heapSize() {
410     long heapsize = MUTATION_OVERHEAD;
411     // Adding row
412     heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
413 
414     // Adding map overhead
415     heapsize +=
416       ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
417     for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
418       //Adding key overhead
419       heapsize +=
420         ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
421 
422       //This part is kinds tricky since the JVM can reuse references if you
423       //store the same value, but have a good match with SizeOf at the moment
424       //Adding value overhead
425       heapsize += ClassSize.align(ClassSize.ARRAYLIST);
426       int size = entry.getValue().size();
427       heapsize += ClassSize.align(ClassSize.ARRAY +
428           size * ClassSize.REFERENCE);
429 
430       for(Cell cell : entry.getValue()) {
431         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
432         heapsize += kv.heapSize();
433       }
434     }
435     heapsize += getAttributeSize();
436     heapsize += extraHeapSize();
437     return ClassSize.align(heapsize);
438   }
439 
440   /**
441    * @return The serialized ACL for this operation, or null if none
442    */
443   public byte[] getACL() {
444     return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL);
445   }
446 
447   /**
448    * @param user User short name
449    * @param perms Permissions for the user
450    */
451   public void setACL(String user, Permission perms) {
452     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
453       ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray());
454   }
455 
456   /**
457    * @param perms A map of permissions for a user or users
458    */
459   public void setACL(Map<String, Permission> perms) {
460     ListMultimap<String, Permission> permMap = ArrayListMultimap.create();
461     for (Map.Entry<String, Permission> entry : perms.entrySet()) {
462       permMap.put(entry.getKey(), entry.getValue());
463     }
464     setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL,
465       ProtobufUtil.toUsersAndPermissions(permMap).toByteArray());
466   }
467 
468   /**
469    * @deprecated No effect
470    */
471   @Deprecated
472   public boolean getACLStrategy() {
473     return false;
474   }
475 
476   /**
477    * @deprecated No effect
478    */
479   @Deprecated
480   public void setACLStrategy(boolean cellFirstStrategy) {
481   }
482 
483   /**
484    * Return the TTL requested for the result of the mutation, in milliseconds.
485    * @return the TTL requested for the result of the mutation, in milliseconds,
486    * or Long.MAX_VALUE if unset
487    */
488   public long getTTL() {
489     byte[] ttlBytes = getAttribute(OP_ATTRIBUTE_TTL);
490     if (ttlBytes != null) {
491       return Bytes.toLong(ttlBytes);
492     }
493     return Long.MAX_VALUE;
494   }
495 
496   /**
497    * Set the TTL desired for the result of the mutation, in milliseconds.
498    * @param ttl the TTL desired for the result of the mutation, in milliseconds
499    * @return this
500    */
501   public Mutation setTTL(long ttl) {
502     setAttribute(OP_ATTRIBUTE_TTL, Bytes.toBytes(ttl));
503     return this;
504   }
505 
506   /**
507    * Subclasses should override this method to add the heap size of their own fields.
508    * @return the heap size to add (will be aligned).
509    */
510   protected long extraHeapSize(){
511     return 0L;
512   }
513 
514   /**
515    * @param row Row to check
516    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
517    * &gt; {@link HConstants#MAX_ROW_LENGTH}
518    * @return <code>row</code>
519    */
520   static byte [] checkRow(final byte [] row) {
521     return checkRow(row, 0, row == null? 0: row.length);
522   }
523 
524   /**
525    * @param row Row to check
526    * @param offset
527    * @param length
528    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
529    * &gt; {@link HConstants#MAX_ROW_LENGTH}
530    * @return <code>row</code>
531    */
532   static byte [] checkRow(final byte [] row, final int offset, final int length) {
533     if (row == null) {
534       throw new IllegalArgumentException("Row buffer is null");
535     }
536     if (length == 0) {
537       throw new IllegalArgumentException("Row length is 0");
538     }
539     if (length > HConstants.MAX_ROW_LENGTH) {
540       throw new IllegalArgumentException("Row length " + length + " is > " +
541         HConstants.MAX_ROW_LENGTH);
542     }
543     return row;
544   }
545 
546   static void checkRow(ByteBuffer row) {
547     if (row == null) {
548       throw new IllegalArgumentException("Row buffer is null");
549     }
550     if (row.remaining() == 0) {
551       throw new IllegalArgumentException("Row length is 0");
552     }
553     if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
554       throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
555           HConstants.MAX_ROW_LENGTH);
556     }
557   }
558 }