View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   * http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.security.visibility;
19  
20  import static org.apache.hadoop.hbase.TagType.VISIBILITY_TAG_TYPE;
21  
22  import java.io.ByteArrayOutputStream;
23  import java.io.DataOutputStream;
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.HashMap;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Map.Entry;
32  import java.util.Set;
33  
34  import org.apache.commons.lang.StringUtils;
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.hbase.classification.InterfaceAudience;
38  import org.apache.hadoop.conf.Configuration;
39  import org.apache.hadoop.hbase.Cell;
40  import org.apache.hadoop.hbase.CellUtil;
41  import org.apache.hadoop.hbase.HColumnDescriptor;
42  import org.apache.hadoop.hbase.Tag;
43  import org.apache.hadoop.hbase.TagType;
44  import org.apache.hadoop.hbase.exceptions.DeserializationException;
45  import org.apache.hadoop.hbase.filter.Filter;
46  import org.apache.hadoop.hbase.io.util.StreamUtils;
47  import org.apache.hadoop.hbase.ipc.RequestContext;
48  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
49  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.MultiUserAuthorizations;
50  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.UserAuthorizations;
51  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel;
52  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest;
53  import org.apache.hadoop.hbase.regionserver.HRegion;
54  import org.apache.hadoop.hbase.security.AccessDeniedException;
55  import org.apache.hadoop.hbase.security.User;
56  import org.apache.hadoop.hbase.security.visibility.expression.ExpressionNode;
57  import org.apache.hadoop.hbase.security.visibility.expression.LeafExpressionNode;
58  import org.apache.hadoop.hbase.security.visibility.expression.NonLeafExpressionNode;
59  import org.apache.hadoop.hbase.security.visibility.expression.Operator;
60  import org.apache.hadoop.hbase.util.ByteRange;
61  import org.apache.hadoop.hbase.util.ByteStringer;
62  import org.apache.hadoop.hbase.util.Bytes;
63  import org.apache.hadoop.hbase.util.SimpleByteRange;
64  import org.apache.hadoop.util.ReflectionUtils;
65  
66  import com.google.protobuf.InvalidProtocolBufferException;
67  
68  /**
69   * Utility method to support visibility
70   */
71  @InterfaceAudience.Private
72  public class VisibilityUtils {
73  
74    private static final Log LOG = LogFactory.getLog(VisibilityUtils.class);
75  
76    public static final String VISIBILITY_LABEL_GENERATOR_CLASS =
77        "hbase.regionserver.scan.visibility.label.generator.class";
78    public static final String SYSTEM_LABEL = "system";
79    public static final Tag SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG = new Tag(
80        TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE,
81        VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG_VAL);
82    private static final String COMMA = ",";
83  
84    private static final ExpressionParser EXP_PARSER = new ExpressionParser();
85    private static final ExpressionExpander EXP_EXPANDER = new ExpressionExpander();
86  
87    /**
88     * Creates the labels data to be written to zookeeper.
89     * @param existingLabels
90     * @return Bytes form of labels and their ordinal details to be written to zookeeper.
91     */
92    public static byte[] getDataToWriteToZooKeeper(Map<String, Integer> existingLabels) {
93      VisibilityLabelsRequest.Builder visReqBuilder = VisibilityLabelsRequest.newBuilder();
94      for (Entry<String, Integer> entry : existingLabels.entrySet()) {
95        VisibilityLabel.Builder visLabBuilder = VisibilityLabel.newBuilder();
96        visLabBuilder.setLabel(ByteStringer.wrap(Bytes.toBytes(entry.getKey())));
97        visLabBuilder.setOrdinal(entry.getValue());
98        visReqBuilder.addVisLabel(visLabBuilder.build());
99      }
100     return ProtobufUtil.prependPBMagic(visReqBuilder.build().toByteArray());
101   }
102 
103   /**
104    * Creates the user auth data to be written to zookeeper.
105    * @param userAuths
106    * @return Bytes form of user auths details to be written to zookeeper.
107    */
108   public static byte[] getUserAuthsDataToWriteToZooKeeper(Map<String, List<Integer>> userAuths) {
109     MultiUserAuthorizations.Builder builder = MultiUserAuthorizations.newBuilder();
110     for (Entry<String, List<Integer>> entry : userAuths.entrySet()) {
111       UserAuthorizations.Builder userAuthsBuilder = UserAuthorizations.newBuilder();
112       userAuthsBuilder.setUser(ByteStringer.wrap(Bytes.toBytes(entry.getKey())));
113       for (Integer label : entry.getValue()) {
114         userAuthsBuilder.addAuth(label);
115       }
116       builder.addUserAuths(userAuthsBuilder.build());
117     }
118     return ProtobufUtil.prependPBMagic(builder.build().toByteArray());
119   }
120 
121   /**
122    * Reads back from the zookeeper. The data read here is of the form written by
123    * writeToZooKeeper(Map<byte[], Integer> entries).
124    * 
125    * @param data
126    * @return Labels and their ordinal details
127    * @throws DeserializationException
128    */
129   public static List<VisibilityLabel> readLabelsFromZKData(byte[] data)
130       throws DeserializationException {
131     if (ProtobufUtil.isPBMagicPrefix(data)) {
132       int pblen = ProtobufUtil.lengthOfPBMagic();
133       try {
134         VisibilityLabelsRequest request = VisibilityLabelsRequest.newBuilder()
135             .mergeFrom(data, pblen, data.length - pblen).build();
136         return request.getVisLabelList();
137       } catch (InvalidProtocolBufferException e) {
138         throw new DeserializationException(e);
139       }
140     }
141     return null;
142   }
143 
144   /**
145    * Reads back User auth data written to zookeeper.
146    * @param data
147    * @return User auth details
148    * @throws DeserializationException
149    */
150   public static MultiUserAuthorizations readUserAuthsFromZKData(byte[] data) 
151       throws DeserializationException {
152     if (ProtobufUtil.isPBMagicPrefix(data)) {
153       int pblen = ProtobufUtil.lengthOfPBMagic();
154       try {
155         MultiUserAuthorizations multiUserAuths = MultiUserAuthorizations.newBuilder()
156             .mergeFrom(data, pblen, data.length - pblen).build();
157         return multiUserAuths;
158       } catch (InvalidProtocolBufferException e) {
159         throw new DeserializationException(e);
160       }
161     }
162     return null;
163   }
164 
165   /**
166    * @param conf The configuration to use
167    * @return Stack of ScanLabelGenerator instances. ScanLabelGenerator classes can be specified in
168    *         Configuration as comma separated list using key
169    *         "hbase.regionserver.scan.visibility.label.generator.class"
170    * @throws IllegalArgumentException
171    *           when any of the specified ScanLabelGenerator class can not be loaded.
172    */
173   public static List<ScanLabelGenerator> getScanLabelGenerators(Configuration conf) {
174     // There can be n SLG specified as comma separated in conf
175     String slgClassesCommaSeparated = conf.get(VISIBILITY_LABEL_GENERATOR_CLASS);
176     // We have only System level SLGs now. The order of execution will be same as the order in the
177     // comma separated config value
178     List<ScanLabelGenerator> slgs = new ArrayList<ScanLabelGenerator>();
179     if (StringUtils.isNotEmpty(slgClassesCommaSeparated)) {
180       String[] slgClasses = slgClassesCommaSeparated.split(COMMA);
181       for (String slgClass : slgClasses) {
182         Class<? extends ScanLabelGenerator> slgKlass;
183         try {
184           slgKlass = (Class<? extends ScanLabelGenerator>) conf.getClassByName(slgClass.trim());
185           slgs.add(ReflectionUtils.newInstance(slgKlass, conf));
186         } catch (ClassNotFoundException e) {
187           throw new IllegalArgumentException("Unable to find " + slgClass, e);
188         }
189       }
190     }
191     // If no SLG is specified in conf, by default we'll add two SLGs
192     // 1. FeedUserAuthScanLabelGenerator
193     // 2. DefinedSetFilterScanLabelGenerator
194     // This stacking will achieve the following default behavior:
195     // 1. If there is no Auths in the scan, we will obtain the global defined set for the user
196     //    from the labels table.
197     // 2. If there is Auths in the scan, we will examine the passed in Auths and filter out the
198     //    labels that the user is not entitled to. Then use the resulting label set.
199     if (slgs.isEmpty()) {
200       slgs.add(ReflectionUtils.newInstance(FeedUserAuthScanLabelGenerator.class, conf));
201       slgs.add(ReflectionUtils.newInstance(DefinedSetFilterScanLabelGenerator.class, conf));
202     }
203     return slgs;
204   }
205 
206   /**
207    * Extract the visibility tags of the given Cell into the given List
208    * @param cell - the cell
209    * @param tags - the array that will be populated if visibility tags are present
210    * @return The visibility tags serialization format
211    */
212   public static Byte extractVisibilityTags(Cell cell, List<Tag> tags) {
213     Byte serializationFormat = null;
214     if (cell.getTagsLengthUnsigned() > 0) {
215       Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
216           cell.getTagsLengthUnsigned());
217       while (tagsIterator.hasNext()) {
218         Tag tag = tagsIterator.next();
219         if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) {
220           serializationFormat = tag.getBuffer()[tag.getTagOffset()];
221         } else if (tag.getType() == TagType.VISIBILITY_TAG_TYPE) {
222           tags.add(tag);
223         }
224       }
225     }
226     return serializationFormat;
227   }
228 
229   /**
230    * Extracts and partitions the visibility tags and nonVisibility Tags
231    *
232    * @param cell - the cell for which we would extract and partition the
233    * visibility and non visibility tags
234    * @param visTags
235    *          - all the visibilty tags of type TagType.VISIBILITY_TAG_TYPE would
236    *          be added to this list
237    * @param nonVisTags - all the non visibility tags would be added to this list
238    * @return - the serailization format of the tag. Can be null if no tags are found or
239    * if there is no visibility tag found
240    */
241   public static Byte extractAndPartitionTags(Cell cell, List<Tag> visTags,
242       List<Tag> nonVisTags) {
243     Byte serializationFormat = null;
244     if (cell.getTagsLength() > 0) {
245       Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
246           cell.getTagsLength());
247       while (tagsIterator.hasNext()) {
248         Tag tag = tagsIterator.next();
249         if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) {
250           serializationFormat = tag.getBuffer()[tag.getTagOffset()];
251         } else if (tag.getType() == VISIBILITY_TAG_TYPE) {
252           visTags.add(tag);
253         } else {
254           // ignore string encoded visibility expressions, will be added in replication handling
255           nonVisTags.add(tag);
256         }
257       }
258     }
259     return serializationFormat;
260   }
261 
262   public static boolean isVisibilityTagsPresent(Cell cell) {
263     if (cell.getTagsLengthUnsigned() == 0) {
264       return false;
265     }
266     Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
267         cell.getTagsLengthUnsigned());
268     while (tagsIterator.hasNext()) {
269       Tag tag = tagsIterator.next();
270       if (tag.getType() == TagType.VISIBILITY_TAG_TYPE) {
271         return true;
272       }
273     }
274     return false;
275   }
276 
277   public static Filter createVisibilityLabelFilter(HRegion region, Authorizations authorizations)
278       throws IOException {
279     Map<ByteRange, Integer> cfVsMaxVersions = new HashMap<ByteRange, Integer>();
280     for (HColumnDescriptor hcd : region.getTableDesc().getFamilies()) {
281       cfVsMaxVersions.put(new SimpleByteRange(hcd.getName()), hcd.getMaxVersions());
282     }
283     VisibilityLabelService vls = VisibilityLabelServiceManager.getInstance()
284         .getVisibilityLabelService();
285     Filter visibilityLabelFilter = new VisibilityLabelFilter(
286         vls.getVisibilityExpEvaluator(authorizations), cfVsMaxVersions);
287     return visibilityLabelFilter;
288   }
289 
290   /**
291    * @return User who called RPC method. For non-RPC handling, falls back to system user
292    * @throws IOException When there is IOE in getting the system user (During non-RPC handling).
293    */
294   public static User getActiveUser() throws IOException {
295     User user = RequestContext.getRequestUser();
296     if (!RequestContext.isInRequestContext()) {
297       user = User.getCurrent();
298     }
299     if (LOG.isTraceEnabled()) {
300       LOG.trace("Current active user name is " + user.getShortName());
301     }
302     return user;
303   }
304 
305   public static List<Tag> createVisibilityExpTags(String visExpression,
306       boolean withSerializationFormat, boolean checkAuths, Set<Integer> auths,
307       VisibilityLabelOrdinalProvider ordinalProvider) throws IOException {
308     ExpressionNode node = null;
309     try {
310       node = EXP_PARSER.parse(visExpression);
311     } catch (ParseException e) {
312       throw new IOException(e);
313     }
314     node = EXP_EXPANDER.expand(node);
315     List<Tag> tags = new ArrayList<Tag>();
316     ByteArrayOutputStream baos = new ByteArrayOutputStream();
317     DataOutputStream dos = new DataOutputStream(baos);
318     List<Integer> labelOrdinals = new ArrayList<Integer>();
319     // We will be adding this tag before the visibility tags and the presence of this
320     // tag indicates we are supporting deletes with cell visibility
321     if (withSerializationFormat) {
322       tags.add(VisibilityUtils.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG);
323     }
324     if (node.isSingleNode()) {
325       getLabelOrdinals(node, labelOrdinals, auths, checkAuths, ordinalProvider);
326       writeLabelOrdinalsToStream(labelOrdinals, dos);
327       tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
328       baos.reset();
329     } else {
330       NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node;
331       if (nlNode.getOperator() == Operator.OR) {
332         for (ExpressionNode child : nlNode.getChildExps()) {
333           getLabelOrdinals(child, labelOrdinals, auths, checkAuths, ordinalProvider);
334           writeLabelOrdinalsToStream(labelOrdinals, dos);
335           tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
336           baos.reset();
337           labelOrdinals.clear();
338         }
339       } else {
340         getLabelOrdinals(nlNode, labelOrdinals, auths, checkAuths, ordinalProvider);
341         writeLabelOrdinalsToStream(labelOrdinals, dos);
342         tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
343         baos.reset();
344       }
345     }
346     return tags;
347   }
348 
349   private static void getLabelOrdinals(ExpressionNode node, List<Integer> labelOrdinals,
350       Set<Integer> auths, boolean checkAuths, VisibilityLabelOrdinalProvider ordinalProvider)
351       throws IOException, InvalidLabelException {
352     if (node.isSingleNode()) {
353       String identifier = null;
354       int labelOrdinal = 0;
355       if (node instanceof LeafExpressionNode) {
356         identifier = ((LeafExpressionNode) node).getIdentifier();
357         if (LOG.isTraceEnabled()) {
358           LOG.trace("The identifier is " + identifier);
359         }
360         labelOrdinal = ordinalProvider.getLabelOrdinal(identifier);
361         checkAuths(auths, labelOrdinal, identifier, checkAuths);
362       } else {
363         // This is a NOT node.
364         LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node)
365             .getChildExps().get(0);
366         identifier = lNode.getIdentifier();
367         labelOrdinal = ordinalProvider.getLabelOrdinal(identifier);
368         checkAuths(auths, labelOrdinal, identifier, checkAuths);
369         labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal.
370       }
371       if (labelOrdinal == 0) {
372         throw new InvalidLabelException("Invalid visibility label " + identifier);
373       }
374       labelOrdinals.add(labelOrdinal);
375     } else {
376       List<ExpressionNode> childExps = ((NonLeafExpressionNode) node).getChildExps();
377       for (ExpressionNode child : childExps) {
378         getLabelOrdinals(child, labelOrdinals, auths, checkAuths, ordinalProvider);
379       }
380     }
381   }
382 
383   /**
384    * This will sort the passed labels in ascending oder and then will write one after the other to
385    * the passed stream.
386    * 
387    * @param labelOrdinals
388    *          Unsorted label ordinals
389    * @param dos
390    *          Stream where to write the labels.
391    * @throws IOException
392    *           When IOE during writes to Stream.
393    */
394   private static void writeLabelOrdinalsToStream(List<Integer> labelOrdinals, DataOutputStream dos)
395       throws IOException {
396     Collections.sort(labelOrdinals);
397     for (Integer labelOrdinal : labelOrdinals) {
398       StreamUtils.writeRawVInt32(dos, labelOrdinal);
399     }
400   }
401 
402   private static void checkAuths(Set<Integer> auths, int labelOrdinal, String identifier,
403       boolean checkAuths) throws IOException {
404     if (checkAuths) {
405       if (auths == null || (!auths.contains(labelOrdinal))) {
406         throw new AccessDeniedException("Visibility label " + identifier
407             + " not authorized for the user " + VisibilityUtils.getActiveUser().getShortName());
408       }
409     }
410   }
411 }