View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.security.visibility;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.hbase.Cell;
29  import org.apache.hadoop.hbase.KeyValue;
30  import org.apache.hadoop.hbase.KeyValue.Type;
31  import org.apache.hadoop.hbase.Tag;
32  import org.apache.hadoop.hbase.regionserver.ScanDeleteTracker;
33  import org.apache.hadoop.hbase.util.Bytes;
34  import org.apache.hadoop.hbase.util.Pair;
35  import org.apache.hadoop.hbase.util.Triple;
36  
37  /**
38   * Similar to ScanDeletTracker but tracks the visibility expression also before
39   * deciding if a Cell can be considered deleted
40   */
41  @InterfaceAudience.Private
42  public class VisibilityScanDeleteTracker extends ScanDeleteTracker {
43  
44    private static final Log LOG = LogFactory.getLog(VisibilityScanDeleteTracker.class);
45  
46    // Its better to track the visibility tags in delete based on each type.  Create individual
47    // data structures for tracking each of them.  This would ensure that there is no tracking based
48    // on time and also would handle all cases where deletefamily or deletecolumns is specified with
49    // Latest_timestamp.  In such cases the ts in the delete marker and the masking
50    // put will not be same. So going with individual data structures for different delete
51    // type would solve this problem and also ensure that the combination of different type
52    // of deletes with diff ts would also work fine
53    // Track per TS
54    private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamily =
55        new ArrayList<Triple<List<Tag>, Byte, Long>>();
56    // Delete family version with different ts and different visibility expression could come.
57    // Need to track it per ts.
58    private List<Triple<List<Tag>, Byte, Long>> visibilityTagsDeleteFamilyVersion =
59        new ArrayList<Triple<List<Tag>, Byte, Long>>();
60    private List<Pair<List<Tag>, Byte>> visibilityTagsDeleteColumns;
61    // Tracking as List<List> is to handle same ts cell but different visibility tag. 
62    // TODO : Need to handle puts with same ts but different vis tags.
63    private List<Pair<List<Tag>, Byte>> visiblityTagsDeleteColumnVersion =
64        new ArrayList<Pair<List<Tag>, Byte>>();
65  
66    public VisibilityScanDeleteTracker() {
67      super();
68    }
69  
70    @Override
71    public void add(Cell delCell) {
72      //Cannot call super.add because need to find if the delete needs to be considered
73      long timestamp = delCell.getTimestamp();
74      int qualifierOffset = delCell.getQualifierOffset();
75      int qualifierLength = delCell.getQualifierLength();
76      byte type = delCell.getTypeByte();
77      if (type == KeyValue.Type.DeleteFamily.getCode()) {
78        hasFamilyStamp = true;
79        boolean hasVisTag = extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
80        if (!hasVisTag && timestamp > familyStamp) {
81          familyStamp = timestamp;
82        }
83        return;
84      } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
85        familyVersionStamps.add(timestamp);
86        extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamilyVersion);
87        return;
88      }
89      // new column, or more general delete type
90      if (deleteBuffer != null) {
91        if (Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength, delCell.getQualifierArray(),
92            qualifierOffset, qualifierLength) != 0) {
93          // A case where there are deletes for a column qualifier but there are
94          // no corresponding puts for them. Rare case.
95          visibilityTagsDeleteColumns = null;
96          visiblityTagsDeleteColumnVersion = null;
97        } else if (type == KeyValue.Type.Delete.getCode() && (deleteTimestamp != timestamp)) {
98          // there is a timestamp change which means we could clear the list
99          // when ts is same and the vis tags are different we need to collect
100         // them all. Interesting part is that in the normal case of puts if
101         // there are 2 cells with same ts and diff vis tags only one of them is
102         // returned. Handling with a single List<Tag> would mean that only one
103         // of the cell would be considered. Doing this as a precaution.
104         // Rare cases.
105         visiblityTagsDeleteColumnVersion = null;
106       }
107     }
108     deleteBuffer = delCell.getQualifierArray();
109     deleteOffset = qualifierOffset;
110     deleteLength = qualifierLength;
111     deleteType = type;
112     deleteTimestamp = timestamp;
113     extractDeleteCellVisTags(delCell, KeyValue.Type.codeToType(type));
114   }
115 
116   private boolean extractDeleteCellVisTags(Cell delCell, Type type) {
117     // If tag is present in the delete
118     boolean hasVisTag = false;
119     if (delCell.getTagsLengthUnsigned() > 0) {
120       switch (type) {
121         case DeleteFamily:
122           List<Tag> delTags = new ArrayList<Tag>();
123           if (visibilityTagsDeleteFamily != null) {
124             Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
125             if (!delTags.isEmpty()) {
126               visibilityTagsDeleteFamily.add(new Triple<List<Tag>, Byte, Long>(
127                   delTags, deleteCellVisTagsFormat, delCell.getTimestamp()));
128               hasVisTag = true;
129             }
130           }
131           break;
132         case DeleteFamilyVersion:
133           delTags = new ArrayList<Tag>();
134           Byte deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
135           if (!delTags.isEmpty()) {
136             visibilityTagsDeleteFamilyVersion.add(new Triple<List<Tag>, Byte, Long>(delTags,
137                 deleteCellVisTagsFormat, delCell.getTimestamp()));
138             hasVisTag = true;
139           }
140           break;
141         case DeleteColumn:
142           if (visibilityTagsDeleteColumns == null) {
143             visibilityTagsDeleteColumns = new ArrayList<Pair<List<Tag>, Byte>>();
144           }
145           delTags = new ArrayList<Tag>();
146           deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
147           if (!delTags.isEmpty()) {
148             visibilityTagsDeleteColumns.add(new Pair<List<Tag>, Byte>(delTags,
149                 deleteCellVisTagsFormat));
150             hasVisTag = true;
151           }
152           break;
153         case Delete:
154           if (visiblityTagsDeleteColumnVersion == null) {
155             visiblityTagsDeleteColumnVersion = new ArrayList<Pair<List<Tag>, Byte>>();
156           }
157           delTags = new ArrayList<Tag>();
158           deleteCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(delCell, delTags);
159           if (!delTags.isEmpty()) {
160             visiblityTagsDeleteColumnVersion.add(new Pair<List<Tag>, Byte>(delTags,
161                 deleteCellVisTagsFormat));
162             hasVisTag = true;
163           }
164           break;
165         default:
166           throw new IllegalArgumentException("Invalid delete type");
167       }
168     } else {
169       switch (type) {
170         case DeleteFamily:
171           visibilityTagsDeleteFamily = null;
172           break;
173         case DeleteFamilyVersion:
174           visibilityTagsDeleteFamilyVersion = null;
175           break;
176         case DeleteColumn:
177           visibilityTagsDeleteColumns = null;
178           break;
179         case Delete:
180           visiblityTagsDeleteColumnVersion = null;
181           break;
182         default:
183           throw new IllegalArgumentException("Invalid delete type");
184       }
185     }
186     return hasVisTag;
187   }
188 
189   @Override
190   public DeleteResult isDeleted(Cell cell) {
191     long timestamp = cell.getTimestamp();
192     int qualifierOffset = cell.getQualifierOffset();
193     int qualifierLength = cell.getQualifierLength();
194     try {
195       if (hasFamilyStamp) {
196         if (visibilityTagsDeleteFamily != null) {
197           for (int i = 0; i < visibilityTagsDeleteFamily.size(); i++) {
198             // visibilityTagsDeleteFamily is ArrayList
199             Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamily.get(i);
200             if (timestamp <= triple.getThird()) {
201               List<Tag> putVisTags = new ArrayList<Tag>();
202               Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
203               boolean matchFound = VisibilityLabelServiceManager
204                   .getInstance().getVisibilityLabelService()
205                   .matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
206                       triple.getSecond());
207               if (matchFound) {
208                 // A return type of FAMILY_DELETED will cause skip for all remaining cells from this
209                 // family. We would like to match visibility expression on every put cells after
210                 // this and only remove those matching with the family delete visibility. So we are
211                 // returning FAMILY_VERSION_DELETED from here.
212                 return DeleteResult.FAMILY_VERSION_DELETED;
213               }
214             }
215           }
216         } else {
217           if (!VisibilityUtils.isVisibilityTagsPresent(cell) && timestamp<=familyStamp) {
218             // No tags
219             return DeleteResult.FAMILY_VERSION_DELETED;
220           }
221         }
222       }
223       if (familyVersionStamps.contains(Long.valueOf(timestamp))) {
224         if (visibilityTagsDeleteFamilyVersion != null) {
225           for (int i = 0; i < visibilityTagsDeleteFamilyVersion.size(); i++) {
226             // visibilityTagsDeleteFamilyVersion is ArrayList
227             Triple<List<Tag>, Byte, Long> triple = visibilityTagsDeleteFamilyVersion.get(i);
228             if (timestamp == triple.getThird()) {
229               List<Tag> putVisTags = new ArrayList<Tag>();
230               Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
231               boolean matchFound = VisibilityLabelServiceManager
232                   .getInstance()
233                   .getVisibilityLabelService()
234                   .matchVisibility(putVisTags, putCellVisTagsFormat, triple.getFirst(),
235                       triple.getSecond());
236               if (matchFound) {
237                 return DeleteResult.FAMILY_VERSION_DELETED;
238               }
239             }
240           }
241         } else {
242           if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
243             // No tags
244             return DeleteResult.FAMILY_VERSION_DELETED;
245           }
246         }
247       }
248       if (deleteBuffer != null) {
249         int ret = Bytes.compareTo(deleteBuffer, deleteOffset, deleteLength,
250             cell.getQualifierArray(), qualifierOffset, qualifierLength);
251 
252         if (ret == 0) {
253           if (deleteType == KeyValue.Type.DeleteColumn.getCode()) {
254             if (visibilityTagsDeleteColumns != null) {
255               for (Pair<List<Tag>, Byte> tags : visibilityTagsDeleteColumns) {
256                 List<Tag> putVisTags = new ArrayList<Tag>();
257                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
258                 boolean matchFound = VisibilityLabelServiceManager
259                     .getInstance()
260                     .getVisibilityLabelService()
261                     .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
262                         tags.getSecond());
263                 if (matchFound) {
264                   return DeleteResult.VERSION_DELETED;
265                 }
266               }
267             } else {
268               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
269                 // No tags
270                 return DeleteResult.VERSION_DELETED;
271               }
272             }
273           }
274           // Delete (aka DeleteVersion)
275           // If the timestamp is the same, keep this one
276           if (timestamp == deleteTimestamp) {
277             if (visiblityTagsDeleteColumnVersion != null) {
278               for (Pair<List<Tag>, Byte> tags : visiblityTagsDeleteColumnVersion) {
279                 List<Tag> putVisTags = new ArrayList<Tag>();
280                 Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
281                 boolean matchFound = VisibilityLabelServiceManager
282                     .getInstance()
283                     .getVisibilityLabelService()
284                     .matchVisibility(putVisTags, putCellVisTagsFormat, tags.getFirst(),
285                         tags.getSecond());
286                 if (matchFound) {
287                   return DeleteResult.VERSION_DELETED;
288                 }
289               }
290             } else {
291               if (!VisibilityUtils.isVisibilityTagsPresent(cell)) {
292                 // No tags
293                 return DeleteResult.VERSION_DELETED;
294               }
295             }
296           }
297         } else if (ret < 0) {
298           // Next column case.
299           deleteBuffer = null;
300           visibilityTagsDeleteColumns = null;
301           visiblityTagsDeleteColumnVersion = null;
302         } else {
303           throw new IllegalStateException("isDeleted failed: deleteBuffer="
304               + Bytes.toStringBinary(deleteBuffer, deleteOffset, deleteLength) + ", qualifier="
305               + Bytes.toStringBinary(cell.getQualifierArray(), qualifierOffset, qualifierLength)
306               + ", timestamp=" + timestamp + ", comparison result: " + ret);
307         }
308       }
309     } catch (IOException e) {
310       LOG.error("Error in isDeleted() check! Will treat cell as not deleted", e);
311     }
312     return DeleteResult.NOT_DELETED;
313   }
314 
315   @Override
316   public void reset() {
317     super.reset();
318     visibilityTagsDeleteColumns = null;
319     visibilityTagsDeleteFamily = new ArrayList<Triple<List<Tag>, Byte, Long>>();
320     visibilityTagsDeleteFamilyVersion = new ArrayList<Triple<List<Tag>, Byte, Long>>();
321     visiblityTagsDeleteColumnVersion = null;
322   }
323 }