View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.client;
20  
21  import java.nio.ByteBuffer;
22  import java.util.ArrayList;
23  import java.util.HashMap;
24  import java.util.List;
25  import java.util.Map;
26  import java.util.NavigableMap;
27  import java.util.TreeMap;
28  import java.util.UUID;
29  
30  import org.apache.hadoop.classification.InterfaceAudience;
31  import org.apache.hadoop.classification.InterfaceStability;
32  import org.apache.hadoop.hbase.Cell;
33  import org.apache.hadoop.hbase.CellScannable;
34  import org.apache.hadoop.hbase.CellScanner;
35  import org.apache.hadoop.hbase.CellUtil;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.KeyValue;
38  import org.apache.hadoop.hbase.KeyValueUtil;
39  import org.apache.hadoop.hbase.io.HeapSize;
40  import org.apache.hadoop.hbase.util.Bytes;
41  import org.apache.hadoop.hbase.util.ClassSize;
42  
43  import com.google.common.collect.Lists;
44  import com.google.common.io.ByteArrayDataInput;
45  import com.google.common.io.ByteArrayDataOutput;
46  import com.google.common.io.ByteStreams;
47  
48  @InterfaceAudience.Public
49  @InterfaceStability.Evolving
50  public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable,
51      HeapSize {
52    public static final long MUTATION_OVERHEAD = ClassSize.align(
53        // This
54        ClassSize.OBJECT +
55        // row + OperationWithAttributes.attributes
56        2 * ClassSize.REFERENCE +
57        // Timestamp
58        1 * Bytes.SIZEOF_LONG +
59        // durability
60        ClassSize.REFERENCE +
61        // familyMap
62        ClassSize.REFERENCE +
63        // familyMap
64        ClassSize.TREEMAP);
65  
66    /**
67     * The attribute for storing the list of clusters that have consumed the change.
68     */
69    private static final String CONSUMED_CLUSTER_IDS = "_cs.id";
70  
71    protected byte [] row = null;
72    protected long ts = HConstants.LATEST_TIMESTAMP;
73    protected Durability durability = Durability.USE_DEFAULT;
74    
75    // A Map sorted by column family.
76    protected NavigableMap<byte [], List<Cell>> familyMap =
77      new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
78  
79    @Override
80    public CellScanner cellScanner() {
81      return CellUtil.createCellScanner(getFamilyCellMap());
82    }
83  
84    /**
85     * Creates an empty list if one doesn't exist for the given column family
86     * or else it returns the associated list of Cell objects.
87     *
88     * @param family column family
89     * @return a list of Cell objects, returns an empty list if one doesn't exist.
90     */
91    List<Cell> getCellList(byte[] family) {
92      List<Cell> list = this.familyMap.get(family);
93      if (list == null) {
94        list = new ArrayList<Cell>();
95      }
96      return list;
97    }
98  
99    /*
100    * Create a nnnnnnnn with this objects row key and the Put identifier.
101    *
102    * @return a KeyValue with this objects row key and the Put identifier.
103    */
104   KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
105     return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
106   }
107 
108   /*
109    * Create a KeyValue with this objects row key and the Put identifier.
110    *
111    * @return a KeyValue with this objects row key and the Put identifier.
112    */
113   KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value) {
114     return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length,
115         family, 0, family == null ? 0 : family.length,
116         qualifier, ts, KeyValue.Type.Put, value);
117   }
118 
119   /**
120    * Compile the column family (i.e. schema) information
121    * into a Map. Useful for parsing and aggregation by debugging,
122    * logging, and administration tools.
123    * @return Map
124    */
125   @Override
126   public Map<String, Object> getFingerprint() {
127     Map<String, Object> map = new HashMap<String, Object>();
128     List<String> families = new ArrayList<String>();
129     // ideally, we would also include table information, but that information
130     // is not stored in each Operation instance.
131     map.put("families", families);
132     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
133       families.add(Bytes.toStringBinary(entry.getKey()));
134     }
135     return map;
136   }
137 
138   /**
139    * Compile the details beyond the scope of getFingerprint (row, columns,
140    * timestamps, etc.) into a Map along with the fingerprinted information.
141    * Useful for debugging, logging, and administration tools.
142    * @param maxCols a limit on the number of columns output prior to truncation
143    * @return Map
144    */
145   @Override
146   public Map<String, Object> toMap(int maxCols) {
147     // we start with the fingerprint map and build on top of it.
148     Map<String, Object> map = getFingerprint();
149     // replace the fingerprint's simple list of families with a
150     // map from column families to lists of qualifiers and kv details
151     Map<String, List<Map<String, Object>>> columns =
152       new HashMap<String, List<Map<String, Object>>>();
153     map.put("families", columns);
154     map.put("row", Bytes.toStringBinary(this.row));
155     int colCount = 0;
156     // iterate through all column families affected
157     for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
158       // map from this family to details for each cell affected within the family
159       List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
160       columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
161       colCount += entry.getValue().size();
162       if (maxCols <= 0) {
163         continue;
164       }
165       // add details for each cell
166       for (Cell cell: entry.getValue()) {
167         if (--maxCols <= 0 ) {
168           continue;
169         }
170         // KeyValue v1 expectation.  Cast for now until we go all Cell all the time.
171         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
172         Map<String, Object> kvMap = kv.toStringMap();
173         // row and family information are already available in the bigger map
174         kvMap.remove("row");
175         kvMap.remove("family");
176         qualifierDetails.add(kvMap);
177       }
178     }
179     map.put("totalColumns", colCount);
180     // add the id if set
181     if (getId() != null) {
182       map.put("id", getId());
183     }
184     return map;
185   }
186 
187   /**
188    * Set the durability for this mutation
189    * @param d
190    */
191   public void setDurability(Durability d) {
192     this.durability = d;
193   }
194 
195   /** Get the current durability */
196   public Durability getDurability() {
197     return this.durability;
198   }
199 
200   /**
201    * Set the durability for this mutation. If this is set to true,
202    * the default durability of the table is set.
203    * @param writeToWal
204    */
205   @Deprecated
206   public void setWriteToWAL(boolean writeToWal) {
207     if(!writeToWal) {
208       setDurability(Durability.SKIP_WAL);
209     } else {
210     // This is required to handle the case where this method is
211     // called twice, first with writeToWal = false,
212     // and then with writeToWal = true
213       setDurability(Durability.USE_DEFAULT);
214     }
215   }
216 
217   /**
218    * Get the durability for this mutation.
219    * @return - true if this mutation is set to write to the WAL either
220    * synchronously, asynchronously or fsync to disk on the file system.
221    * - to get the exact durability, use the {#getDurability} method.
222    */
223   @Deprecated
224   public boolean getWriteToWAL() {
225     return Durability.SKIP_WAL != getDurability();
226   }
227 
228   /*
229    * Method for retrieving the put's familyMap  (family -> KeyValues)
230    * Application should use the getFamilyCellMap and the Cell interface instead of KeyValue.
231    *
232    * @return familyMap
233    */
234   @Deprecated
235   public Map<byte[], List<KeyValue>> getFamilyMap() {
236     Map<byte[], List<KeyValue>> fm = new TreeMap<byte[], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
237     for (Map.Entry<byte[], List<Cell>> e : this.familyMap.entrySet()) {
238       byte[] family = e.getKey();
239       List<Cell> cells = e.getValue();
240       List<KeyValue> kvs = new ArrayList<KeyValue>(cells.size());
241       for (Cell c : cells) {
242          KeyValue kv = KeyValueUtil.ensureKeyValue(c);
243          kvs.add(kv);
244        }
245        fm.put(family, kvs);
246      }
247      return fm;
248    }
249   
250   /**
251    * Method for retrieving the put's familyMap
252    * @return familyMap
253    */
254   public NavigableMap<byte [], List<Cell>> getFamilyCellMap() {
255     return this.familyMap;
256   }
257 
258   /**
259    * Method for setting the put's familyMap
260    */
261   public void setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) {
262     // TODO: Shut this down or move it up to be a Constructor.  Get new object rather than change
263     // this internal data member.
264     this.familyMap = map;
265   }
266 
267   /**
268    * Method for setting the put's familyMap that is deprecated and inefficient.
269    * @deprecated use {@link #setFamilyCellMap(NavigableMap)} instead.
270    */
271   @Deprecated
272   public void setFamilyMap(NavigableMap<byte [], List<KeyValue>> map) {
273     TreeMap<byte[], List<Cell>> fm = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR);
274     for (Map.Entry<byte[], List<KeyValue>> e : map.entrySet()) {
275       fm.put(e.getKey(), Lists.<Cell>newArrayList(e.getValue()));
276     }
277     this.familyMap = fm;
278   }
279 
280   /**
281    * Method to check if the familyMap is empty
282    * @return true if empty, false otherwise
283    */
284   public boolean isEmpty() {
285     return familyMap.isEmpty();
286   }
287 
288   /**
289    * Method for retrieving the delete's row
290    * @return row
291    */
292   @Override
293   public byte [] getRow() {
294     return this.row;
295   }
296 
297   public int compareTo(final Row d) {
298     return Bytes.compareTo(this.getRow(), d.getRow());
299   }
300 
301   /**
302    * Method for retrieving the timestamp
303    * @return timestamp
304    */
305   public long getTimeStamp() {
306     return this.ts;
307   }
308 
309   /**
310    * Marks that the clusters with the given clusterIds have consumed the mutation
311    * @param clusterIds of the clusters that have consumed the mutation
312    */
313   public void setClusterIds(List<UUID> clusterIds) {
314     ByteArrayDataOutput out = ByteStreams.newDataOutput();
315     out.writeInt(clusterIds.size());
316     for (UUID clusterId : clusterIds) {
317       out.writeLong(clusterId.getMostSignificantBits());
318       out.writeLong(clusterId.getLeastSignificantBits());
319     }
320     setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray());
321   }
322 
323   /**
324    * @return the set of clusterIds that have consumed the mutation
325    */
326   public List<UUID> getClusterIds() {
327     List<UUID> clusterIds = new ArrayList<UUID>();
328     byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS);
329     if(bytes != null) {
330       ByteArrayDataInput in = ByteStreams.newDataInput(bytes);
331       int numClusters = in.readInt();
332       for(int i=0; i<numClusters; i++){
333         clusterIds.add(new UUID(in.readLong(), in.readLong()));
334       }
335     }
336     return clusterIds;
337   }
338 
339   /**
340    * Number of KeyValues carried by this Mutation.
341    * @return the total number of KeyValues
342    */
343   public int size() {
344     int size = 0;
345     for (List<Cell> cells : this.familyMap.values()) {
346       size += cells.size();
347     }
348     return size;
349   }
350 
351   /**
352    * @return the number of different families
353    */
354   public int numFamilies() {
355     return familyMap.size();
356   }
357 
358   /**
359    * @return Calculate what Mutation adds to class heap size.
360    */
361   @Override
362   public long heapSize() {
363     long heapsize = MUTATION_OVERHEAD;
364     // Adding row
365     heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
366 
367     // Adding map overhead
368     heapsize +=
369       ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
370     for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) {
371       //Adding key overhead
372       heapsize +=
373         ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
374 
375       //This part is kinds tricky since the JVM can reuse references if you
376       //store the same value, but have a good match with SizeOf at the moment
377       //Adding value overhead
378       heapsize += ClassSize.align(ClassSize.ARRAYLIST);
379       int size = entry.getValue().size();
380       heapsize += ClassSize.align(ClassSize.ARRAY +
381           size * ClassSize.REFERENCE);
382 
383       for(Cell cell : entry.getValue()) {
384         KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
385         heapsize += kv.heapSize();
386       }
387     }
388     heapsize += getAttributeSize();
389     heapsize += extraHeapSize();
390     return ClassSize.align(heapsize);
391   }
392 
393   /**
394    * Subclasses should override this method to add the heap size of their own fields.
395    * @return the heap size to add (will be aligned).
396    */
397   protected long extraHeapSize(){
398     return 0L;
399   }
400 
401 
402   /**
403    * @param row Row to check
404    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
405    * &gt; {@link HConstants#MAX_ROW_LENGTH}
406    * @return <code>row</code>
407    */
408   static byte [] checkRow(final byte [] row) {
409     return checkRow(row, 0, row == null? 0: row.length);
410   }
411 
412   /**
413    * @param row Row to check
414    * @param offset
415    * @param length
416    * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or
417    * &gt; {@link HConstants#MAX_ROW_LENGTH}
418    * @return <code>row</code>
419    */
420   static byte [] checkRow(final byte [] row, final int offset, final int length) {
421     if (row == null) {
422       throw new IllegalArgumentException("Row buffer is null");
423     }
424     if (length == 0) {
425       throw new IllegalArgumentException("Row length is 0");
426     }
427     if (length > HConstants.MAX_ROW_LENGTH) {
428       throw new IllegalArgumentException("Row length " + length + " is > " +
429         HConstants.MAX_ROW_LENGTH);
430     }
431     return row;
432   }
433 
434   static void checkRow(ByteBuffer row) {
435     if (row == null) {
436       throw new IllegalArgumentException("Row buffer is null");
437     }
438     if (row.remaining() == 0) {
439       throw new IllegalArgumentException("Row length is 0");
440     }
441     if (row.remaining() > HConstants.MAX_ROW_LENGTH) {
442       throw new IllegalArgumentException("Row length " + row.remaining() + " is > " +
443           HConstants.MAX_ROW_LENGTH);
444     }
445   }
446 }