View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   * http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.security.visibility;
19  
20  import static org.apache.hadoop.hbase.TagType.VISIBILITY_TAG_TYPE;
21  
22  import java.io.ByteArrayOutputStream;
23  import java.io.DataOutputStream;
24  import java.io.IOException;
25  import java.util.ArrayList;
26  import java.util.Collections;
27  import java.util.HashMap;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.Map.Entry;
32  import java.util.Set;
33  
34  import org.apache.commons.lang.StringUtils;
35  import org.apache.commons.logging.Log;
36  import org.apache.commons.logging.LogFactory;
37  import org.apache.hadoop.classification.InterfaceAudience;
38  import org.apache.hadoop.conf.Configuration;
39  import org.apache.hadoop.hbase.Cell;
40  import org.apache.hadoop.hbase.CellUtil;
41  import org.apache.hadoop.hbase.HColumnDescriptor;
42  import org.apache.hadoop.hbase.Tag;
43  import org.apache.hadoop.hbase.TagType;
44  import org.apache.hadoop.hbase.exceptions.DeserializationException;
45  import org.apache.hadoop.hbase.filter.Filter;
46  import org.apache.hadoop.hbase.io.util.StreamUtils;
47  import org.apache.hadoop.hbase.ipc.RequestContext;
48  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
49  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.MultiUserAuthorizations;
50  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.UserAuthorizations;
51  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel;
52  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest;
53  import org.apache.hadoop.hbase.regionserver.HRegion;
54  import org.apache.hadoop.hbase.security.AccessDeniedException;
55  import org.apache.hadoop.hbase.security.User;
56  import org.apache.hadoop.hbase.security.visibility.expression.ExpressionNode;
57  import org.apache.hadoop.hbase.security.visibility.expression.LeafExpressionNode;
58  import org.apache.hadoop.hbase.security.visibility.expression.NonLeafExpressionNode;
59  import org.apache.hadoop.hbase.security.visibility.expression.Operator;
60  import org.apache.hadoop.hbase.util.ByteRange;
61  import org.apache.hadoop.hbase.util.ByteStringer;
62  import org.apache.hadoop.hbase.util.Bytes;
63  import org.apache.hadoop.hbase.util.SimpleByteRange;
64  import org.apache.hadoop.util.ReflectionUtils;
65  
66  import com.google.protobuf.InvalidProtocolBufferException;
67  
68  /**
69   * Utility method to support visibility
70   */
71  @InterfaceAudience.Private
72  public class VisibilityUtils {
73  
74    private static final Log LOG = LogFactory.getLog(VisibilityUtils.class);
75  
76    public static final String VISIBILITY_LABEL_GENERATOR_CLASS =
77        "hbase.regionserver.scan.visibility.label.generator.class";
78    public static final String SYSTEM_LABEL = "system";
79    public static final Tag SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG = new Tag(
80        TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE,
81        VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG_VAL);
82    private static final String COMMA = ",";
83  
84    private static final ExpressionParser EXP_PARSER = new ExpressionParser();
85    private static final ExpressionExpander EXP_EXPANDER = new ExpressionExpander();
86  
87    /**
88     * Creates the labels data to be written to zookeeper.
89     * @param existingLabels
90     * @return Bytes form of labels and their ordinal details to be written to zookeeper.
91     */
92    public static byte[] getDataToWriteToZooKeeper(Map<String, Integer> existingLabels) {
93      VisibilityLabelsRequest.Builder visReqBuilder = VisibilityLabelsRequest.newBuilder();
94      for (Entry<String, Integer> entry : existingLabels.entrySet()) {
95        VisibilityLabel.Builder visLabBuilder = VisibilityLabel.newBuilder();
96        visLabBuilder.setLabel(ByteStringer.wrap(Bytes.toBytes(entry.getKey())));
97        visLabBuilder.setOrdinal(entry.getValue());
98        visReqBuilder.addVisLabel(visLabBuilder.build());
99      }
100     return ProtobufUtil.prependPBMagic(visReqBuilder.build().toByteArray());
101   }
102 
103   /**
104    * Creates the user auth data to be written to zookeeper.
105    * @param userAuths
106    * @return Bytes form of user auths details to be written to zookeeper.
107    */
108   public static byte[] getUserAuthsDataToWriteToZooKeeper(Map<String, List<Integer>> userAuths) {
109     MultiUserAuthorizations.Builder builder = MultiUserAuthorizations.newBuilder();
110     for (Entry<String, List<Integer>> entry : userAuths.entrySet()) {
111       UserAuthorizations.Builder userAuthsBuilder = UserAuthorizations.newBuilder();
112       userAuthsBuilder.setUser(ByteStringer.wrap(Bytes.toBytes(entry.getKey())));
113       for (Integer label : entry.getValue()) {
114         userAuthsBuilder.addAuth(label);
115       }
116       builder.addUserAuths(userAuthsBuilder.build());
117     }
118     return ProtobufUtil.prependPBMagic(builder.build().toByteArray());
119   }
120 
121   /**
122    * Reads back from the zookeeper. The data read here is of the form written by
123    * writeToZooKeeper(Map<byte[], Integer> entries).
124    * 
125    * @param data
126    * @return Labels and their ordinal details
127    * @throws DeserializationException
128    */
129   public static List<VisibilityLabel> readLabelsFromZKData(byte[] data)
130       throws DeserializationException {
131     if (ProtobufUtil.isPBMagicPrefix(data)) {
132       int pblen = ProtobufUtil.lengthOfPBMagic();
133       try {
134         VisibilityLabelsRequest request = VisibilityLabelsRequest.newBuilder()
135             .mergeFrom(data, pblen, data.length - pblen).build();
136         return request.getVisLabelList();
137       } catch (InvalidProtocolBufferException e) {
138         throw new DeserializationException(e);
139       }
140     }
141     return null;
142   }
143 
144   /**
145    * Reads back User auth data written to zookeeper.
146    * @param data
147    * @return User auth details
148    * @throws DeserializationException
149    */
150   public static MultiUserAuthorizations readUserAuthsFromZKData(byte[] data) 
151       throws DeserializationException {
152     if (ProtobufUtil.isPBMagicPrefix(data)) {
153       int pblen = ProtobufUtil.lengthOfPBMagic();
154       try {
155         MultiUserAuthorizations multiUserAuths = MultiUserAuthorizations.newBuilder()
156             .mergeFrom(data, pblen, data.length - pblen).build();
157         return multiUserAuths;
158       } catch (InvalidProtocolBufferException e) {
159         throw new DeserializationException(e);
160       }
161     }
162     return null;
163   }
164 
165   /**
166    * @param conf The configuration to use
167    * @return Stack of ScanLabelGenerator instances. ScanLabelGenerator classes can be specified in
168    *         Configuration as comma separated list using key
169    *         "hbase.regionserver.scan.visibility.label.generator.class"
170    * @throws IllegalArgumentException
171    *           when any of the specified ScanLabelGenerator class can not be loaded.
172    */
173   public static List<ScanLabelGenerator> getScanLabelGenerators(Configuration conf) {
174     // There can be n SLG specified as comma separated in conf
175     String slgClassesCommaSeparated = conf.get(VISIBILITY_LABEL_GENERATOR_CLASS);
176     // We have only System level SLGs now. The order of execution will be same as the order in the
177     // comma separated config value
178     List<ScanLabelGenerator> slgs = new ArrayList<ScanLabelGenerator>();
179     if (StringUtils.isNotEmpty(slgClassesCommaSeparated)) {
180       String[] slgClasses = slgClassesCommaSeparated.split(COMMA);
181       for (String slgClass : slgClasses) {
182         Class<? extends ScanLabelGenerator> slgKlass;
183         try {
184           slgKlass = (Class<? extends ScanLabelGenerator>) conf.getClassByName(slgClass.trim());
185           slgs.add(ReflectionUtils.newInstance(slgKlass, conf));
186         } catch (ClassNotFoundException e) {
187           throw new IllegalArgumentException("Unable to find " + slgClass, e);
188         }
189       }
190     }
191     // If the conf is not configured by default we need to have one SLG to be used
192     // ie. DefaultScanLabelGenerator
193     if (slgs.isEmpty()) {
194       slgs.add(ReflectionUtils.newInstance(DefaultScanLabelGenerator.class, conf));
195     }
196     return slgs;
197   }
198 
199   /**
200    * Extract the visibility tags of the given Cell into the given List
201    * @param cell - the cell
202    * @param tags - the array that will be populated if visibility tags are present
203    * @return The visibility tags serialization format
204    */
205   public static Byte extractVisibilityTags(Cell cell, List<Tag> tags) {
206     Byte serializationFormat = null;
207     if (cell.getTagsLengthUnsigned() > 0) {
208       Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
209           cell.getTagsLengthUnsigned());
210       while (tagsIterator.hasNext()) {
211         Tag tag = tagsIterator.next();
212         if (tag.getType() == TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) {
213           serializationFormat = tag.getBuffer()[tag.getTagOffset()];
214         } else if (tag.getType() == TagType.VISIBILITY_TAG_TYPE) {
215           tags.add(tag);
216         }
217       }
218     }
219     return serializationFormat;
220   }
221 
222   public static boolean isVisibilityTagsPresent(Cell cell) {
223     if (cell.getTagsLengthUnsigned() == 0) {
224       return false;
225     }
226     Iterator<Tag> tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
227         cell.getTagsLengthUnsigned());
228     while (tagsIterator.hasNext()) {
229       Tag tag = tagsIterator.next();
230       if (tag.getType() == TagType.VISIBILITY_TAG_TYPE) {
231         return true;
232       }
233     }
234     return false;
235   }
236 
237   public static Filter createVisibilityLabelFilter(HRegion region, Authorizations authorizations)
238       throws IOException {
239     Map<ByteRange, Integer> cfVsMaxVersions = new HashMap<ByteRange, Integer>();
240     for (HColumnDescriptor hcd : region.getTableDesc().getFamilies()) {
241       cfVsMaxVersions.put(new SimpleByteRange(hcd.getName()), hcd.getMaxVersions());
242     }
243     VisibilityLabelService vls = VisibilityLabelServiceManager.getInstance()
244         .getVisibilityLabelService();
245     Filter visibilityLabelFilter = new VisibilityLabelFilter(
246         vls.getVisibilityExpEvaluator(authorizations), cfVsMaxVersions);
247     return visibilityLabelFilter;
248   }
249 
250   /**
251    * @return User who called RPC method. For non-RPC handling, falls back to system user
252    * @throws IOException When there is IOE in getting the system user (During non-RPC handling).
253    */
254   public static User getActiveUser() throws IOException {
255     User user = RequestContext.getRequestUser();
256     if (!RequestContext.isInRequestContext()) {
257       user = User.getCurrent();
258     }
259     if (LOG.isTraceEnabled()) {
260       LOG.trace("Current active user name is " + user.getShortName());
261     }
262     return user;
263   }
264 
265   public static List<Tag> createVisibilityExpTags(String visExpression,
266       boolean withSerializationFormat, boolean checkAuths, Set<Integer> auths,
267       VisibilityLabelOrdinalProvider ordinalProvider) throws IOException {
268     ExpressionNode node = null;
269     try {
270       node = EXP_PARSER.parse(visExpression);
271     } catch (ParseException e) {
272       throw new IOException(e);
273     }
274     node = EXP_EXPANDER.expand(node);
275     List<Tag> tags = new ArrayList<Tag>();
276     ByteArrayOutputStream baos = new ByteArrayOutputStream();
277     DataOutputStream dos = new DataOutputStream(baos);
278     List<Integer> labelOrdinals = new ArrayList<Integer>();
279     // We will be adding this tag before the visibility tags and the presence of this
280     // tag indicates we are supporting deletes with cell visibility
281     if (withSerializationFormat) {
282       tags.add(VisibilityUtils.SORTED_ORDINAL_SERIALIZATION_FORMAT_TAG);
283     }
284     if (node.isSingleNode()) {
285       getLabelOrdinals(node, labelOrdinals, auths, checkAuths, ordinalProvider);
286       writeLabelOrdinalsToStream(labelOrdinals, dos);
287       tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
288       baos.reset();
289     } else {
290       NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node;
291       if (nlNode.getOperator() == Operator.OR) {
292         for (ExpressionNode child : nlNode.getChildExps()) {
293           getLabelOrdinals(child, labelOrdinals, auths, checkAuths, ordinalProvider);
294           writeLabelOrdinalsToStream(labelOrdinals, dos);
295           tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
296           baos.reset();
297           labelOrdinals.clear();
298         }
299       } else {
300         getLabelOrdinals(nlNode, labelOrdinals, auths, checkAuths, ordinalProvider);
301         writeLabelOrdinalsToStream(labelOrdinals, dos);
302         tags.add(new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray()));
303         baos.reset();
304       }
305     }
306     return tags;
307   }
308 
309   private static void getLabelOrdinals(ExpressionNode node, List<Integer> labelOrdinals,
310       Set<Integer> auths, boolean checkAuths, VisibilityLabelOrdinalProvider ordinalProvider)
311       throws IOException, InvalidLabelException {
312     if (node.isSingleNode()) {
313       String identifier = null;
314       int labelOrdinal = 0;
315       if (node instanceof LeafExpressionNode) {
316         identifier = ((LeafExpressionNode) node).getIdentifier();
317         if (LOG.isTraceEnabled()) {
318           LOG.trace("The identifier is " + identifier);
319         }
320         labelOrdinal = ordinalProvider.getLabelOrdinal(identifier);
321         checkAuths(auths, labelOrdinal, identifier, checkAuths);
322       } else {
323         // This is a NOT node.
324         LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node)
325             .getChildExps().get(0);
326         identifier = lNode.getIdentifier();
327         labelOrdinal = ordinalProvider.getLabelOrdinal(identifier);
328         checkAuths(auths, labelOrdinal, identifier, checkAuths);
329         labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal.
330       }
331       if (labelOrdinal == 0) {
332         throw new InvalidLabelException("Invalid visibility label " + identifier);
333       }
334       labelOrdinals.add(labelOrdinal);
335     } else {
336       List<ExpressionNode> childExps = ((NonLeafExpressionNode) node).getChildExps();
337       for (ExpressionNode child : childExps) {
338         getLabelOrdinals(child, labelOrdinals, auths, checkAuths, ordinalProvider);
339       }
340     }
341   }
342 
343   /**
344    * This will sort the passed labels in ascending oder and then will write one after the other to
345    * the passed stream.
346    * 
347    * @param labelOrdinals
348    *          Unsorted label ordinals
349    * @param dos
350    *          Stream where to write the labels.
351    * @throws IOException
352    *           When IOE during writes to Stream.
353    */
354   private static void writeLabelOrdinalsToStream(List<Integer> labelOrdinals, DataOutputStream dos)
355       throws IOException {
356     Collections.sort(labelOrdinals);
357     for (Integer labelOrdinal : labelOrdinals) {
358       StreamUtils.writeRawVInt32(dos, labelOrdinal);
359     }
360   }
361 
362   private static void checkAuths(Set<Integer> auths, int labelOrdinal, String identifier,
363       boolean checkAuths) throws IOException {
364     if (checkAuths) {
365       if (auths == null || (!auths.contains(labelOrdinal))) {
366         throw new AccessDeniedException("Visibility label " + identifier
367             + " not authorized for the user " + VisibilityUtils.getActiveUser().getShortName());
368       }
369     }
370   }
371 }