View Javadoc

1   /*
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.client;
22  
23  import org.apache.hadoop.hbase.HConstants;
24  import org.apache.hadoop.hbase.filter.Filter;
25  import org.apache.hadoop.hbase.filter.IncompatibleFilterException;
26  import org.apache.hadoop.hbase.io.TimeRange;
27  import org.apache.hadoop.hbase.util.Bytes;
28  import org.apache.hadoop.hbase.util.Classes;
29  import org.apache.hadoop.io.Writable;
30  
31  import java.io.DataInput;
32  import java.io.DataOutput;
33  import java.io.IOException;
34  import java.util.ArrayList;
35  import java.util.HashMap;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.NavigableSet;
39  import java.util.TreeMap;
40  import java.util.TreeSet;
41  
42  /**
43   * Used to perform Scan operations.
44   * <p>
45   * All operations are identical to {@link Get} with the exception of
46   * instantiation.  Rather than specifying a single row, an optional startRow
47   * and stopRow may be defined.  If rows are not specified, the Scanner will
48   * iterate over all rows.
49   * <p>
50   * To scan everything for each row, instantiate a Scan object.
51   * <p>
52   * To modify scanner caching for just this scan, use {@link #setCaching(int) setCaching}.
53   * If caching is NOT set, we will use the caching value of the hosting
54   * {@link HTable}.  See {@link HTable#setScannerCaching(int)}.
55   * <p>
56   * To further define the scope of what to get when scanning, perform additional
57   * methods as outlined below.
58   * <p>
59   * To get all columns from specific families, execute {@link #addFamily(byte[]) addFamily}
60   * for each family to retrieve.
61   * <p>
62   * To get specific columns, execute {@link #addColumn(byte[], byte[]) addColumn}
63   * for each column to retrieve.
64   * <p>
65   * To only retrieve columns within a specific range of version timestamps,
66   * execute {@link #setTimeRange(long, long) setTimeRange}.
67   * <p>
68   * To only retrieve columns with a specific timestamp, execute
69   * {@link #setTimeStamp(long) setTimestamp}.
70   * <p>
71   * To limit the number of versions of each column to be returned, execute
72   * {@link #setMaxVersions(int) setMaxVersions}.
73   * <p>
74   * To limit the maximum number of values returned for each call to next(),
75   * execute {@link #setBatch(int) setBatch}.
76   * <p>
77   * To add a filter, execute {@link #setFilter(org.apache.hadoop.hbase.filter.Filter) setFilter}.
78   * <p>
79   * Expert: To explicitly disable server-side block caching for this scan,
80   * execute {@link #setCacheBlocks(boolean)}.
81   */
82  public class Scan extends OperationWithAttributes implements Writable {
83    private static final String RAW_ATTR = "_raw_";
84    private static final String ONDEMAND_ATTR = "_ondemand_";
85    private static final String ISOLATION_LEVEL = "_isolationlevel_";
86  
87    private static final byte SCAN_VERSION = (byte)2;
88    private byte [] startRow = HConstants.EMPTY_START_ROW;
89    private byte [] stopRow  = HConstants.EMPTY_END_ROW;
90    private int maxVersions = 1;
91    private int batch = -1;
92    // If application wants to collect scan metrics, it needs to
93    // call scan.setAttribute(SCAN_ATTRIBUTES_ENABLE, Bytes.toBytes(Boolean.TRUE))
94    static public String SCAN_ATTRIBUTES_METRICS_ENABLE = "scan.attributes.metrics.enable";
95    static public String SCAN_ATTRIBUTES_METRICS_DATA = "scan.attributes.metrics.data";
96    
97    // If an application wants to use multiple scans over different tables each scan must
98    // define this attribute with the appropriate table name by calling
99    // scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName))
100   static public final String SCAN_ATTRIBUTES_TABLE_NAME = "scan.attributes.table.name";
101 
102   /*
103    * -1 means no caching
104    */
105   private int caching = -1;
106   private boolean cacheBlocks = true;
107   private Filter filter = null;
108   private TimeRange tr = new TimeRange();
109   private Map<byte [], NavigableSet<byte []>> familyMap =
110     new TreeMap<byte [], NavigableSet<byte []>>(Bytes.BYTES_COMPARATOR);
111 
112   /**
113    * Create a Scan operation across all rows.
114    */
115   public Scan() {}
116 
117   public Scan(byte [] startRow, Filter filter) {
118     this(startRow);
119     this.filter = filter;
120   }
121 
122   /**
123    * Create a Scan operation starting at the specified row.
124    * <p>
125    * If the specified row does not exist, the Scanner will start from the
126    * next closest row after the specified row.
127    * @param startRow row to start scanner at or after
128    */
129   public Scan(byte [] startRow) {
130     this.startRow = startRow;
131   }
132 
133   /**
134    * Create a Scan operation for the range of rows specified.
135    * @param startRow row to start scanner at or after (inclusive)
136    * @param stopRow row to stop scanner before (exclusive)
137    */
138   public Scan(byte [] startRow, byte [] stopRow) {
139     this.startRow = startRow;
140     this.stopRow = stopRow;
141   }
142 
143   /**
144    * Creates a new instance of this class while copying all values.
145    *
146    * @param scan  The scan instance to copy from.
147    * @throws IOException When copying the values fails.
148    */
149   public Scan(Scan scan) throws IOException {
150     startRow = scan.getStartRow();
151     stopRow  = scan.getStopRow();
152     maxVersions = scan.getMaxVersions();
153     batch = scan.getBatch();
154     caching = scan.getCaching();
155     cacheBlocks = scan.getCacheBlocks();
156     filter = scan.getFilter(); // clone?
157     TimeRange ctr = scan.getTimeRange();
158     tr = new TimeRange(ctr.getMin(), ctr.getMax());
159     Map<byte[], NavigableSet<byte[]>> fams = scan.getFamilyMap();
160     for (Map.Entry<byte[],NavigableSet<byte[]>> entry : fams.entrySet()) {
161       byte [] fam = entry.getKey();
162       NavigableSet<byte[]> cols = entry.getValue();
163       if (cols != null && cols.size() > 0) {
164         for (byte[] col : cols) {
165           addColumn(fam, col);
166         }
167       } else {
168         addFamily(fam);
169       }
170     }
171     for (Map.Entry<String, byte[]> attr : scan.getAttributesMap().entrySet()) {
172       setAttribute(attr.getKey(), attr.getValue());
173     }
174   }
175 
176   /**
177    * Builds a scan object with the same specs as get.
178    * @param get get to model scan after
179    */
180   public Scan(Get get) {
181     this.startRow = get.getRow();
182     this.stopRow = get.getRow();
183     this.filter = get.getFilter();
184     this.cacheBlocks = get.getCacheBlocks();
185     this.maxVersions = get.getMaxVersions();
186     this.tr = get.getTimeRange();
187     this.familyMap = get.getFamilyMap();
188   }
189 
190   public boolean isGetScan() {
191     return this.startRow != null && this.startRow.length > 0 &&
192       Bytes.equals(this.startRow, this.stopRow);
193   }
194 
195   /**
196    * Get all columns from the specified family.
197    * <p>
198    * Overrides previous calls to addColumn for this family.
199    * @param family family name
200    * @return this
201    */
202   public Scan addFamily(byte [] family) {
203     familyMap.remove(family);
204     familyMap.put(family, null);
205     return this;
206   }
207 
208   /**
209    * Get the column from the specified family with the specified qualifier.
210    * <p>
211    * Overrides previous calls to addFamily for this family.
212    * @param family family name
213    * @param qualifier column qualifier
214    * @return this
215    */
216   public Scan addColumn(byte [] family, byte [] qualifier) {
217     NavigableSet<byte []> set = familyMap.get(family);
218     if(set == null) {
219       set = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
220     }
221     if (qualifier == null) {
222       qualifier = HConstants.EMPTY_BYTE_ARRAY;
223     }
224     set.add(qualifier);
225     familyMap.put(family, set);
226 
227     return this;
228   }
229 
230   /**
231    * Get versions of columns only within the specified timestamp range,
232    * [minStamp, maxStamp).  Note, default maximum versions to return is 1.  If
233    * your time range spans more than one version and you want all versions
234    * returned, up the number of versions beyond the defaut.
235    * @param minStamp minimum timestamp value, inclusive
236    * @param maxStamp maximum timestamp value, exclusive
237    * @throws IOException if invalid time range
238    * @see #setMaxVersions()
239    * @see #setMaxVersions(int)
240    * @return this
241    */
242   public Scan setTimeRange(long minStamp, long maxStamp)
243   throws IOException {
244     tr = new TimeRange(minStamp, maxStamp);
245     return this;
246   }
247 
248   /**
249    * Get versions of columns with the specified timestamp. Note, default maximum
250    * versions to return is 1.  If your time range spans more than one version
251    * and you want all versions returned, up the number of versions beyond the
252    * defaut.
253    * @param timestamp version timestamp
254    * @see #setMaxVersions()
255    * @see #setMaxVersions(int)
256    * @return this
257    */
258   public Scan setTimeStamp(long timestamp) {
259     try {
260       tr = new TimeRange(timestamp, timestamp+1);
261     } catch(IOException e) {
262       // Will never happen
263     }
264     return this;
265   }
266 
267   /**
268    * Set the start row of the scan.
269    * @param startRow row to start scan on (inclusive)
270    * Note: In order to make startRow exclusive add a trailing 0 byte
271    * @return this
272    */
273   public Scan setStartRow(byte [] startRow) {
274     this.startRow = startRow;
275     return this;
276   }
277 
278   /**
279    * Set the stop row.
280    * @param stopRow row to end at (exclusive)
281    * Note: In order to make stopRow inclusive add a trailing 0 byte
282    * @return this
283    */
284   public Scan setStopRow(byte [] stopRow) {
285     this.stopRow = stopRow;
286     return this;
287   }
288 
289   /**
290    * Get all available versions.
291    * @return this
292    */
293   public Scan setMaxVersions() {
294     this.maxVersions = Integer.MAX_VALUE;
295     return this;
296   }
297 
298   /**
299    * Get up to the specified number of versions of each column.
300    * @param maxVersions maximum versions for each column
301    * @return this
302    */
303   public Scan setMaxVersions(int maxVersions) {
304     this.maxVersions = maxVersions;
305     return this;
306   }
307 
308   /**
309    * Set the maximum number of values to return for each call to next()
310    * @param batch the maximum number of values
311    */
312   public void setBatch(int batch) {
313     if (this.hasFilter() && this.filter.hasFilterRow()) {
314       throw new IncompatibleFilterException(
315         "Cannot set batch on a scan using a filter" +
316         " that returns true for filter.hasFilterRow");
317     }
318     this.batch = batch;
319   }
320 
321   /**
322    * Set the number of rows for caching that will be passed to scanners.
323    * If not set, the default setting from {@link HTable#getScannerCaching()} will apply.
324    * Higher caching values will enable faster scanners but will use more memory.
325    * @param caching the number of rows for caching
326    */
327   public void setCaching(int caching) {
328     this.caching = caching;
329   }
330 
331   /**
332    * Apply the specified server-side filter when performing the Scan.
333    * @param filter filter to run on the server
334    * @return this
335    */
336   public Scan setFilter(Filter filter) {
337     this.filter = filter;
338     return this;
339   }
340 
341   /**
342    * Setting the familyMap
343    * @param familyMap map of family to qualifier
344    * @return this
345    */
346   public Scan setFamilyMap(Map<byte [], NavigableSet<byte []>> familyMap) {
347     this.familyMap = familyMap;
348     return this;
349   }
350 
351   /**
352    * Getting the familyMap
353    * @return familyMap
354    */
355   public Map<byte [], NavigableSet<byte []>> getFamilyMap() {
356     return this.familyMap;
357   }
358 
359   /**
360    * @return the number of families in familyMap
361    */
362   public int numFamilies() {
363     if(hasFamilies()) {
364       return this.familyMap.size();
365     }
366     return 0;
367   }
368 
369   /**
370    * @return true if familyMap is non empty, false otherwise
371    */
372   public boolean hasFamilies() {
373     return !this.familyMap.isEmpty();
374   }
375 
376   /**
377    * @return the keys of the familyMap
378    */
379   public byte[][] getFamilies() {
380     if(hasFamilies()) {
381       return this.familyMap.keySet().toArray(new byte[0][0]);
382     }
383     return null;
384   }
385 
386   /**
387    * @return the startrow
388    */
389   public byte [] getStartRow() {
390     return this.startRow;
391   }
392 
393   /**
394    * @return the stoprow
395    */
396   public byte [] getStopRow() {
397     return this.stopRow;
398   }
399 
400   /**
401    * @return the max number of versions to fetch
402    */
403   public int getMaxVersions() {
404     return this.maxVersions;
405   }
406 
407   /**
408    * @return maximum number of values to return for a single call to next()
409    */
410   public int getBatch() {
411     return this.batch;
412   }
413 
414   /**
415    * @return caching the number of rows fetched when calling next on a scanner
416    */
417   public int getCaching() {
418     return this.caching;
419   }
420 
421   /**
422    * @return TimeRange
423    */
424   public TimeRange getTimeRange() {
425     return this.tr;
426   }
427 
428   /**
429    * @return RowFilter
430    */
431   public Filter getFilter() {
432     return filter;
433   }
434 
435   /**
436    * @return true is a filter has been specified, false if not
437    */
438   public boolean hasFilter() {
439     return filter != null;
440   }
441 
442   /**
443    * Set whether blocks should be cached for this Scan.
444    * <p>
445    * This is true by default.  When true, default settings of the table and
446    * family are used (this will never override caching blocks if the block
447    * cache is disabled for that family or entirely).
448    *
449    * @param cacheBlocks if false, default settings are overridden and blocks
450    * will not be cached
451    */
452   public void setCacheBlocks(boolean cacheBlocks) {
453     this.cacheBlocks = cacheBlocks;
454   }
455 
456   /**
457    * Get whether blocks should be cached for this Scan.
458    * @return true if default caching should be used, false if blocks should not
459    * be cached
460    */
461   public boolean getCacheBlocks() {
462     return cacheBlocks;
463   }
464 
465   /**
466    * Set the value indicating whether loading CFs on demand should be allowed (cluster
467    * default is false). On-demand CF loading doesn't load column families until necessary, e.g.
468    * if you filter on one column, the other column family data will be loaded only for the rows
469    * that are included in result, not all rows like in normal case.
470    * With column-specific filters, like SingleColumnValueFilter w/filterIfMissing == true,
471    * this can deliver huge perf gains when there's a cf with lots of data; however, it can
472    * also lead to some inconsistent results, as follows:
473    * - if someone does a concurrent update to both column families in question you may get a row
474    *   that never existed, e.g. for { rowKey = 5, { cat_videos => 1 }, { video => "my cat" } }
475    *   someone puts rowKey 5 with { cat_videos => 0 }, { video => "my dog" }, concurrent scan
476    *   filtering on "cat_videos == 1" can get { rowKey = 5, { cat_videos => 1 },
477    *   { video => "my dog" } }.
478    * - if there's a concurrent split and you have more than 2 column families, some rows may be
479    *   missing some column families.
480    */
481   public void setLoadColumnFamiliesOnDemand(boolean value) {
482     setAttribute(ONDEMAND_ATTR, Bytes.toBytes(value));
483   }
484 
485   /**
486    * Get the logical value indicating whether on-demand CF loading should be allowed.
487    */
488   public boolean doLoadColumnFamiliesOnDemand() {
489     byte[] attr = getAttribute(ONDEMAND_ATTR);
490     return attr == null ? false : Bytes.toBoolean(attr);
491   }
492 
493   /**
494    * Compile the table and column family (i.e. schema) information
495    * into a String. Useful for parsing and aggregation by debugging,
496    * logging, and administration tools.
497    * @return Map
498    */
499   @Override
500   public Map<String, Object> getFingerprint() {
501     Map<String, Object> map = new HashMap<String, Object>();
502     List<String> families = new ArrayList<String>();
503     if(this.familyMap.size() == 0) {
504       map.put("families", "ALL");
505       return map;
506     } else {
507       map.put("families", families);
508     }
509     for (Map.Entry<byte [], NavigableSet<byte[]>> entry :
510         this.familyMap.entrySet()) {
511       families.add(Bytes.toStringBinary(entry.getKey()));
512     }
513     return map;
514   }
515 
516   /**
517    * Compile the details beyond the scope of getFingerprint (row, columns,
518    * timestamps, etc.) into a Map along with the fingerprinted information.
519    * Useful for debugging, logging, and administration tools.
520    * @param maxCols a limit on the number of columns output prior to truncation
521    * @return Map
522    */
523   @Override
524   public Map<String, Object> toMap(int maxCols) {
525     // start with the fingerpring map and build on top of it
526     Map<String, Object> map = getFingerprint();
527     // map from families to column list replaces fingerprint's list of families
528     Map<String, List<String>> familyColumns =
529       new HashMap<String, List<String>>();
530     map.put("families", familyColumns);
531     // add scalar information first
532     map.put("startRow", Bytes.toStringBinary(this.startRow));
533     map.put("stopRow", Bytes.toStringBinary(this.stopRow));
534     map.put("maxVersions", this.maxVersions);
535     map.put("batch", this.batch);
536     map.put("caching", this.caching);
537     map.put("cacheBlocks", this.cacheBlocks);
538     List<Long> timeRange = new ArrayList<Long>();
539     timeRange.add(this.tr.getMin());
540     timeRange.add(this.tr.getMax());
541     map.put("timeRange", timeRange);
542     int colCount = 0;
543     // iterate through affected families and list out up to maxCols columns
544     for (Map.Entry<byte [], NavigableSet<byte[]>> entry :
545       this.familyMap.entrySet()) {
546       List<String> columns = new ArrayList<String>();
547       familyColumns.put(Bytes.toStringBinary(entry.getKey()), columns);
548       if(entry.getValue() == null) {
549         colCount++;
550         --maxCols;
551         columns.add("ALL");
552       } else {
553         colCount += entry.getValue().size();
554         if (maxCols <= 0) {
555           continue;
556         } 
557         for (byte [] column : entry.getValue()) {
558           if (--maxCols <= 0) {
559             continue;
560           }
561           columns.add(Bytes.toStringBinary(column));
562         }
563       } 
564     }       
565     map.put("totalColumns", colCount);
566     if (this.filter != null) {
567       map.put("filter", this.filter.toString());
568     }
569     // add the id if set
570     if (getId() != null) {
571       map.put("id", getId());
572     }
573     return map;
574   }
575 
576   //Writable
577   public void readFields(final DataInput in)
578   throws IOException {
579     int version = in.readByte();
580     if (version > (int)SCAN_VERSION) {
581       throw new IOException("version not supported");
582     }
583     this.startRow = Bytes.readByteArray(in);
584     this.stopRow = Bytes.readByteArray(in);
585     this.maxVersions = in.readInt();
586     this.batch = in.readInt();
587     this.caching = in.readInt();
588     this.cacheBlocks = in.readBoolean();
589     if(in.readBoolean()) {
590       this.filter = Classes.createWritableForName(
591         Bytes.toString(Bytes.readByteArray(in)));
592       this.filter.readFields(in);
593     }
594     this.tr = new TimeRange();
595     tr.readFields(in);
596     int numFamilies = in.readInt();
597     this.familyMap =
598       new TreeMap<byte [], NavigableSet<byte []>>(Bytes.BYTES_COMPARATOR);
599     for(int i=0; i<numFamilies; i++) {
600       byte [] family = Bytes.readByteArray(in);
601       int numColumns = in.readInt();
602       TreeSet<byte []> set = new TreeSet<byte []>(Bytes.BYTES_COMPARATOR);
603       for(int j=0; j<numColumns; j++) {
604         byte [] qualifier = Bytes.readByteArray(in);
605         set.add(qualifier);
606       }
607       this.familyMap.put(family, set);
608     }
609 
610     if (version > 1) {
611       readAttributes(in);
612     }
613   }
614 
615   public void write(final DataOutput out)
616   throws IOException {
617     out.writeByte(SCAN_VERSION);
618     Bytes.writeByteArray(out, this.startRow);
619     Bytes.writeByteArray(out, this.stopRow);
620     out.writeInt(this.maxVersions);
621     out.writeInt(this.batch);
622     out.writeInt(this.caching);
623     out.writeBoolean(this.cacheBlocks);
624     if(this.filter == null) {
625       out.writeBoolean(false);
626     } else {
627       out.writeBoolean(true);
628       Bytes.writeByteArray(out, Bytes.toBytes(filter.getClass().getName()));
629       filter.write(out);
630     }
631     tr.write(out);
632     out.writeInt(familyMap.size());
633     for(Map.Entry<byte [], NavigableSet<byte []>> entry : familyMap.entrySet()) {
634       Bytes.writeByteArray(out, entry.getKey());
635       NavigableSet<byte []> columnSet = entry.getValue();
636       if(columnSet != null){
637         out.writeInt(columnSet.size());
638         for(byte [] qualifier : columnSet) {
639           Bytes.writeByteArray(out, qualifier);
640         }
641       } else {
642         out.writeInt(0);
643       }
644     }
645     writeAttributes(out);
646   }
647 
648   /**
649    * Enable/disable "raw" mode for this scan.
650    * If "raw" is enabled the scan will return all
651    * delete marker and deleted rows that have not
652    * been collected, yet.
653    * This is mostly useful for Scan on column families
654    * that have KEEP_DELETED_ROWS enabled.
655    * It is an error to specify any column when "raw" is set.
656    * @param raw True/False to enable/disable "raw" mode.
657    */
658   public void setRaw(boolean raw) {
659     setAttribute(RAW_ATTR, Bytes.toBytes(raw));
660   }
661 
662   /**
663    * @return True if this Scan is in "raw" mode.
664    */
665   public boolean isRaw() {
666     byte[] attr = getAttribute(RAW_ATTR);
667     return attr == null ? false : Bytes.toBoolean(attr);
668   }
669 
670   /*
671    * Set the isolation level for this scan. If the
672    * isolation level is set to READ_UNCOMMITTED, then
673    * this scan will return data from committed and
674    * uncommitted transactions. If the isolation level 
675    * is set to READ_COMMITTED, then this scan will return 
676    * data from committed transactions only. If a isolation
677    * level is not explicitly set on a Scan, then it 
678    * is assumed to be READ_COMMITTED.
679    * @param level IsolationLevel for this scan
680    */
681   public void setIsolationLevel(IsolationLevel level) {
682     setAttribute(ISOLATION_LEVEL, level.toBytes());
683   }
684   /*
685    * @return The isolation level of this scan.
686    * If no isolation level was set for this scan object, 
687    * then it returns READ_COMMITTED.
688    * @return The IsolationLevel for this scan
689    */
690   public IsolationLevel getIsolationLevel() {
691     byte[] attr = getAttribute(ISOLATION_LEVEL);
692     return attr == null ? IsolationLevel.READ_COMMITTED :
693                           IsolationLevel.fromBytes(attr);
694   }
695 }