View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   * http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.security.visibility;
20  
21  import static org.apache.hadoop.hbase.HConstants.OperationStatusCode.SANITY_CHECK_FAILURE;
22  import static org.apache.hadoop.hbase.HConstants.OperationStatusCode.SUCCESS;
23  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
24  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.HashMap;
29  import java.util.Iterator;
30  import java.util.List;
31  import java.util.Map;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.hbase.classification.InterfaceAudience;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.Cell;
38  import org.apache.hadoop.hbase.CellScanner;
39  import org.apache.hadoop.hbase.CellUtil;
40  import org.apache.hadoop.hbase.CoprocessorEnvironment;
41  import org.apache.hadoop.hbase.DoNotRetryIOException;
42  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HTableDescriptor;
46  import org.apache.hadoop.hbase.KeyValue;
47  import org.apache.hadoop.hbase.KeyValue.Type;
48  import org.apache.hadoop.hbase.KeyValueUtil;
49  import org.apache.hadoop.hbase.TableName;
50  import org.apache.hadoop.hbase.Tag;
51  import org.apache.hadoop.hbase.TagType;
52  import org.apache.hadoop.hbase.catalog.MetaReader;
53  import org.apache.hadoop.hbase.client.Append;
54  import org.apache.hadoop.hbase.client.Delete;
55  import org.apache.hadoop.hbase.client.Get;
56  import org.apache.hadoop.hbase.client.Increment;
57  import org.apache.hadoop.hbase.client.Mutation;
58  import org.apache.hadoop.hbase.client.Put;
59  import org.apache.hadoop.hbase.client.Result;
60  import org.apache.hadoop.hbase.client.Scan;
61  import org.apache.hadoop.hbase.constraint.ConstraintException;
62  import org.apache.hadoop.hbase.coprocessor.BaseMasterAndRegionObserver;
63  import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
64  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
65  import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
66  import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
67  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
68  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
69  import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
70  import org.apache.hadoop.hbase.exceptions.DeserializationException;
71  import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
72  import org.apache.hadoop.hbase.filter.Filter;
73  import org.apache.hadoop.hbase.filter.FilterBase;
74  import org.apache.hadoop.hbase.filter.FilterList;
75  import org.apache.hadoop.hbase.io.hfile.HFile;
76  import org.apache.hadoop.hbase.ipc.RequestContext;
77  import org.apache.hadoop.hbase.master.MasterServices;
78  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
79  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
80  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos;
81  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsRequest;
82  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
83  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.SetAuthsRequest;
84  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel;
85  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest;
86  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
87  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsService;
88  import org.apache.hadoop.hbase.regionserver.BloomType;
89  import org.apache.hadoop.hbase.regionserver.DeleteTracker;
90  import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
91  import org.apache.hadoop.hbase.regionserver.HRegion;
92  import org.apache.hadoop.hbase.regionserver.InternalScanner;
93  import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
94  import org.apache.hadoop.hbase.regionserver.OperationStatus;
95  import org.apache.hadoop.hbase.regionserver.RegionScanner;
96  import org.apache.hadoop.hbase.security.AccessDeniedException;
97  import org.apache.hadoop.hbase.security.User;
98  import org.apache.hadoop.hbase.security.access.AccessControlLists;
99  import org.apache.hadoop.hbase.security.access.AccessController;
100 import org.apache.hadoop.hbase.util.ByteStringer;
101 import org.apache.hadoop.hbase.util.Bytes;
102 
103 import com.google.common.collect.Lists;
104 import com.google.common.collect.MapMaker;
105 import com.google.protobuf.ByteString;
106 import com.google.protobuf.RpcCallback;
107 import com.google.protobuf.RpcController;
108 import com.google.protobuf.Service;
109 
110 /**
111  * Coprocessor that has both the MasterObserver and RegionObserver implemented that supports in
112  * visibility labels
113  */
114 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
115 public class VisibilityController extends BaseMasterAndRegionObserver implements
116     VisibilityLabelsService.Interface, CoprocessorService {
117 
118   private static final Log LOG = LogFactory.getLog(VisibilityController.class);
119   // flags if we are running on a region of the 'labels' table
120   private boolean labelsRegion = false;
121   // Flag denoting whether AcessController is available or not.
122   private boolean acOn = false;
123   private Configuration conf;
124   private volatile boolean initialized = false;
125   private boolean checkAuths = false;
126   /** Mapping of scanner instances to the user who created them */
127   private Map<InternalScanner,String> scannerOwners =
128       new MapMaker().weakKeys().makeMap();
129 
130   List<String> superUsers;
131   private VisibilityLabelService visibilityLabelService;
132 
133   // Add to this list if there are any reserved tag types
134   private static ArrayList<Byte> RESERVED_VIS_TAG_TYPES = new ArrayList<Byte>();
135   static {
136     RESERVED_VIS_TAG_TYPES.add(TagType.VISIBILITY_TAG_TYPE);
137     RESERVED_VIS_TAG_TYPES.add(TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE);
138   }
139 
140   @Override
141   public void start(CoprocessorEnvironment env) throws IOException {
142     this.conf = env.getConfiguration();
143     if (HFile.getFormatVersion(conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
144       throw new RuntimeException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS
145         + " is required to persist visibility labels. Consider setting " + HFile.FORMAT_VERSION_KEY
146         + " accordingly.");
147     }
148     if (env instanceof RegionServerCoprocessorEnvironment) {
149       throw new RuntimeException(
150           "Visibility controller should not be configured as " +
151           "'hbase.coprocessor.regionserver.classes'.");
152     }
153 
154     if (env instanceof RegionCoprocessorEnvironment) {
155       // VisibilityLabelService to be instantiated only with Region Observer.
156       visibilityLabelService = VisibilityLabelServiceManager.getInstance()
157           .getVisibilityLabelService(this.conf);
158     }
159     this.superUsers = getSystemAndSuperUsers();
160   }
161 
162   @Override
163   public void stop(CoprocessorEnvironment env) throws IOException {
164 
165   }
166 
167   /********************************* Master related hooks **********************************/
168 
169   @Override
170   public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
171     // Need to create the new system table for labels here
172     MasterServices master = ctx.getEnvironment().getMasterServices();
173     if (!MetaReader.tableExists(master.getCatalogTracker(), LABELS_TABLE_NAME)) {
174       HTableDescriptor labelsTable = new HTableDescriptor(LABELS_TABLE_NAME);
175       HColumnDescriptor labelsColumn = new HColumnDescriptor(LABELS_TABLE_FAMILY);
176       labelsColumn.setBloomFilterType(BloomType.NONE);
177       labelsColumn.setBlockCacheEnabled(false); // We will cache all the labels. No need of normal
178                                                  // table block cache.
179       labelsTable.addFamily(labelsColumn);
180       // Let the "labels" table having only one region always. We are not expecting too many labels in
181       // the system.
182       labelsTable.setValue(HTableDescriptor.SPLIT_POLICY,
183           DisabledRegionSplitPolicy.class.getName());
184       labelsTable.setValue(Bytes.toBytes(HConstants.DISALLOW_WRITES_IN_RECOVERING),
185           Bytes.toBytes(true));
186       master.createTable(labelsTable, null);
187     }
188   }
189 
190   @Override
191   public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
192       TableName tableName, HTableDescriptor htd) throws IOException {
193     if (LABELS_TABLE_NAME.equals(tableName)) {
194       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
195     }
196   }
197 
198   @Override
199   public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
200       HColumnDescriptor column) throws IOException {
201     if (LABELS_TABLE_NAME.equals(tableName)) {
202       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
203     }
204   }
205 
206   @Override
207   public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
208       TableName tableName, HColumnDescriptor descriptor) throws IOException {
209     if (LABELS_TABLE_NAME.equals(tableName)) {
210       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
211     }
212   }
213 
214   @Override
215   public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
216       TableName tableName, byte[] c) throws IOException {
217     if (LABELS_TABLE_NAME.equals(tableName)) {
218       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
219     }
220   }
221 
222   @Override
223   public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
224       throws IOException {
225     if (LABELS_TABLE_NAME.equals(tableName)) {
226       throw new ConstraintException("Cannot disable " + LABELS_TABLE_NAME);
227     }
228   }
229 
230   /****************************** Region related hooks ******************************/
231 
232   @Override
233   public void postOpen(ObserverContext<RegionCoprocessorEnvironment> e) {
234     // Read the entire labels table and populate the zk
235     if (e.getEnvironment().getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) {
236       this.labelsRegion = true;
237       this.acOn = CoprocessorHost.getLoadedCoprocessors().contains(AccessController.class.getName());
238       // Defer the init of VisibilityLabelService on labels region until it is in recovering state.
239       if (!e.getEnvironment().getRegion().isRecovering()) {
240         initVisibilityLabelService(e.getEnvironment());
241       }
242     } else {
243       checkAuths = e.getEnvironment().getConfiguration()
244           .getBoolean(VisibilityConstants.CHECK_AUTHS_FOR_MUTATION, false);
245       initVisibilityLabelService(e.getEnvironment());
246     }
247   }
248 
249   @Override
250   public void postLogReplay(ObserverContext<RegionCoprocessorEnvironment> e) {
251     if (this.labelsRegion) {
252       initVisibilityLabelService(e.getEnvironment());
253       LOG.debug("post labels region log replay");
254     }
255   }
256 
257   private void initVisibilityLabelService(RegionCoprocessorEnvironment env) {
258     try {
259       this.visibilityLabelService.init(env);
260       this.initialized = true;
261     } catch (IOException ioe) {
262       LOG.error("Error while initializing VisibilityLabelService..", ioe);
263       throw new RuntimeException(ioe);
264     }
265   }
266 
267   @Override
268   public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
269       MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
270     if (c.getEnvironment().getRegion().getRegionInfo().getTable().isSystemTable()) {
271       return;
272     }
273     // TODO this can be made as a global LRU cache at HRS level?
274     Map<String, List<Tag>> labelCache = new HashMap<String, List<Tag>>();
275     for (int i = 0; i < miniBatchOp.size(); i++) {
276       Mutation m = miniBatchOp.getOperation(i);
277       CellVisibility cellVisibility = null;
278       try {
279         cellVisibility = m.getCellVisibility();
280       } catch (DeserializationException de) {
281         miniBatchOp.setOperationStatus(i,
282             new OperationStatus(SANITY_CHECK_FAILURE, de.getMessage()));
283         continue;
284       }
285       boolean sanityFailure = false;
286       for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) {
287         if (!checkForReservedVisibilityTagPresence(cellScanner.current())) {
288           miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE,
289               "Mutation contains cell with reserved type tag"));
290           sanityFailure = true;
291           break;
292         }
293       }
294       if (!sanityFailure) {
295         if (cellVisibility != null) {
296           String labelsExp = cellVisibility.getExpression();
297           List<Tag> visibilityTags = labelCache.get(labelsExp);
298           if (visibilityTags == null) {
299             // Don't check user auths for labels with Mutations when the user is super user
300             boolean authCheck = this.checkAuths && !(isSystemOrSuperUser());
301             try {
302               visibilityTags = this.visibilityLabelService.createVisibilityExpTags(labelsExp, true,
303                   authCheck);
304             } catch (InvalidLabelException e) {
305               miniBatchOp.setOperationStatus(i,
306                   new OperationStatus(SANITY_CHECK_FAILURE, e.getMessage()));
307             }
308             if (visibilityTags != null) {
309               labelCache.put(labelsExp, visibilityTags);
310             }
311           }
312           if (visibilityTags != null) {
313             List<Cell> updatedCells = new ArrayList<Cell>();
314             for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) {
315               Cell cell = cellScanner.current();
316               List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
317                   cell.getTagsLengthUnsigned());
318               tags.addAll(visibilityTags);
319               Cell updatedCell = new KeyValue(cell.getRowArray(), cell.getRowOffset(),
320                   cell.getRowLength(), cell.getFamilyArray(), cell.getFamilyOffset(),
321                   cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(),
322                   cell.getQualifierLength(), cell.getTimestamp(), Type.codeToType(cell
323                       .getTypeByte()), cell.getValueArray(), cell.getValueOffset(),
324                   cell.getValueLength(), tags);
325               updatedCells.add(updatedCell);
326             }
327             m.getFamilyCellMap().clear();
328             // Clear and add new Cells to the Mutation.
329             for (Cell cell : updatedCells) {
330               if (m instanceof Put) {
331                 Put p = (Put) m;
332                 p.add(cell);
333               } else if (m instanceof Delete) {
334                 // TODO : Cells without visibility tags would be handled in follow up issue
335                 Delete d = (Delete) m;
336                 d.addDeleteMarker(cell);
337               }
338             }
339           }
340         }
341       }
342     }
343   }
344 
345   @Override
346   public void prePrepareTimeStampForDeleteVersion(
347       ObserverContext<RegionCoprocessorEnvironment> ctx, Mutation delete, Cell cell,
348       byte[] byteNow, Get get) throws IOException {
349     KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
350     CellVisibility cellVisibility = null;
351     try {
352       cellVisibility = delete.getCellVisibility();
353     } catch (DeserializationException de) {
354       throw new IOException("Invalid cell visibility specified " + delete, de);
355     }
356     // The check for checkForReservedVisibilityTagPresence happens in preBatchMutate happens.
357     // It happens for every mutation and that would be enough.
358     List<Tag> visibilityTags = new ArrayList<Tag>();
359     if (cellVisibility != null) {
360       String labelsExp = cellVisibility.getExpression();
361       try {
362         visibilityTags = this.visibilityLabelService.createVisibilityExpTags(labelsExp, false,
363             false);
364       } catch (InvalidLabelException e) {
365         throw new IOException("Invalid cell visibility specified " + labelsExp, e);
366       }
367     }
368     get.setFilter(new DeleteVersionVisibilityExpressionFilter(visibilityTags,
369         VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT));
370     List<Cell> result = ctx.getEnvironment().getRegion().get(get, false);
371 
372     if (result.size() < get.getMaxVersions()) {
373       // Nothing to delete
374       kv.updateLatestStamp(Bytes.toBytes(Long.MIN_VALUE));
375       return;
376     }
377     if (result.size() > get.getMaxVersions()) {
378       throw new RuntimeException("Unexpected size: " + result.size()
379           + ". Results more than the max versions obtained.");
380     }
381     KeyValue getkv = KeyValueUtil.ensureKeyValue(result.get(get.getMaxVersions() - 1));
382     Bytes.putBytes(kv.getBuffer(), kv.getTimestampOffset(), getkv.getBuffer(),
383         getkv.getTimestampOffset(), Bytes.SIZEOF_LONG);
384     // We are bypassing here because in the HRegion.updateDeleteLatestVersionTimeStamp we would
385     // update with the current timestamp after again doing a get. As the hook as already determined
386     // the needed timestamp we need to bypass here.
387     // TODO : See if HRegion.updateDeleteLatestVersionTimeStamp() could be
388     // called only if the hook is not called.
389     ctx.bypass();
390   }
391 
392   // Checks whether cell contains any tag with type as VISIBILITY_TAG_TYPE.
393   // This tag type is reserved and should not be explicitly set by user.
394   private boolean checkForReservedVisibilityTagPresence(Cell cell) throws IOException {
395     // Bypass this check when the operation is done by a system/super user.
396     // This is done because, while Replication, the Cells coming to the peer cluster with reserved
397     // typed tags and this is fine and should get added to the peer cluster table
398     if (isSystemOrSuperUser()) {
399       return true;
400     }
401     if (cell.getTagsLengthUnsigned() > 0) {
402       Iterator<Tag> tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
403           cell.getTagsLengthUnsigned());
404       while (tagsItr.hasNext()) {
405         if (RESERVED_VIS_TAG_TYPES.contains(tagsItr.next().getType())) {
406           return false;
407         }
408       }
409     }
410     return true;
411   }
412 
413   @Override
414   public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
415       RegionScanner s) throws IOException {
416     if (!initialized) {
417       throw new VisibilityControllerNotReadyException("VisibilityController not yet initialized!");
418     }
419     HRegion region = e.getEnvironment().getRegion();
420     Authorizations authorizations = null;
421     try {
422       authorizations = scan.getAuthorizations();
423     } catch (DeserializationException de) {
424       throw new IOException(de);
425     }
426     if (authorizations == null) {
427       // No Authorizations present for this scan/Get!
428       // In case of system tables other than "labels" just scan with out visibility check and
429       // filtering. Checking visibility labels for META and NAMESPACE table is not needed.
430       TableName table = region.getRegionInfo().getTable();
431       if (table.isSystemTable() && !table.equals(LABELS_TABLE_NAME)) {
432         return s;
433       }
434     }
435 
436     Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(region,
437         authorizations);
438     if (visibilityLabelFilter != null) {
439       Filter filter = scan.getFilter();
440       if (filter != null) {
441         scan.setFilter(new FilterList(filter, visibilityLabelFilter));
442       } else {
443         scan.setFilter(visibilityLabelFilter);
444       }
445     }
446     return s;
447   }
448 
449   @Override
450   public DeleteTracker postInstantiateDeleteTracker(
451       ObserverContext<RegionCoprocessorEnvironment> ctx, DeleteTracker delTracker)
452       throws IOException {
453     HRegion region = ctx.getEnvironment().getRegion();
454     TableName table = region.getRegionInfo().getTable();
455     if (table.isSystemTable()) {
456       return delTracker;
457     }
458     // We are creating a new type of delete tracker here which is able to track
459     // the timestamps and also the
460     // visibility tags per cell. The covering cells are determined not only
461     // based on the delete type and ts
462     // but also on the visibility expression matching.
463     return new VisibilityScanDeleteTracker();
464   }
465 
466   @Override
467   public RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
468       final Scan scan, final RegionScanner s) throws IOException {
469     User user = VisibilityUtils.getActiveUser();
470     if (user != null && user.getShortName() != null) {
471       scannerOwners.put(s, user.getShortName());
472     }
473     return s;
474   }
475 
476   @Override
477   public boolean preScannerNext(final ObserverContext<RegionCoprocessorEnvironment> c,
478       final InternalScanner s, final List<Result> result, final int limit, final boolean hasNext)
479       throws IOException {
480     requireScannerOwner(s);
481     return hasNext;
482   }
483 
484   @Override
485   public void preScannerClose(final ObserverContext<RegionCoprocessorEnvironment> c,
486       final InternalScanner s) throws IOException {
487     requireScannerOwner(s);
488   }
489 
490   @Override
491   public void postScannerClose(final ObserverContext<RegionCoprocessorEnvironment> c,
492       final InternalScanner s) throws IOException {
493     // clean up any associated owner mapping
494     scannerOwners.remove(s);
495   }
496 
497   /**
498    * Verify, when servicing an RPC, that the caller is the scanner owner. If so, we assume that
499    * access control is correctly enforced based on the checks performed in preScannerOpen()
500    */
501   private void requireScannerOwner(InternalScanner s) throws AccessDeniedException {
502     if (RequestContext.isInRequestContext()) {
503       String requestUName = RequestContext.getRequestUserName();
504       String owner = scannerOwners.get(s);
505       if (owner != null && !owner.equals(requestUName)) {
506         throw new AccessDeniedException("User '" + requestUName + "' is not the scanner owner!");
507       }
508     }
509   }
510 
511   @Override
512   public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
513       throws IOException {
514     if (!initialized) {
515       throw new VisibilityControllerNotReadyException("VisibilityController not yet initialized!");
516     }
517     HRegion region = e.getEnvironment().getRegion();
518     Authorizations authorizations = null;
519     try {
520       authorizations = get.getAuthorizations();
521     } catch (DeserializationException de) {
522       throw new IOException(de);
523     }
524     if (authorizations == null) {
525       // No Authorizations present for this scan/Get!
526       // In case of system tables other than "labels" just scan with out visibility check and
527       // filtering. Checking visibility labels for META and NAMESPACE table is not needed.
528       TableName table = region.getRegionInfo().getTable();
529       if (table.isSystemTable() && !table.equals(LABELS_TABLE_NAME)) {
530         return;
531       }
532     }
533     Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(e.getEnvironment()
534         .getRegion(), authorizations);
535     if (visibilityLabelFilter != null) {
536       Filter filter = get.getFilter();
537       if (filter != null) {
538         get.setFilter(new FilterList(filter, visibilityLabelFilter));
539       } else {
540         get.setFilter(visibilityLabelFilter);
541       }
542     }
543   }
544 
545   private List<String> getSystemAndSuperUsers() throws IOException {
546     User user = User.getCurrent();
547     if (user == null) {
548       throw new IOException("Unable to obtain the current user, "
549           + "authorization checks for internal operations will not work correctly!");
550     }
551     if (LOG.isTraceEnabled()) {
552       LOG.trace("Current user name is "+user.getShortName());
553     }
554     String currentUser = user.getShortName();
555     List<String> superUsers = Lists.asList(currentUser,
556         this.conf.getStrings(AccessControlLists.SUPERUSER_CONF_KEY, new String[0]));
557     return superUsers;
558   }
559 
560   private boolean isSystemOrSuperUser() throws IOException {
561     User activeUser = VisibilityUtils.getActiveUser();
562     return this.superUsers.contains(activeUser.getShortName());
563   }
564 
565   @Override
566   public Result preAppend(ObserverContext<RegionCoprocessorEnvironment> e, Append append)
567       throws IOException {
568     for (CellScanner cellScanner = append.cellScanner(); cellScanner.advance();) {
569       if (!checkForReservedVisibilityTagPresence(cellScanner.current())) {
570         throw new FailedSanityCheckException("Append contains cell with reserved type tag");
571       }
572     }
573     return null;
574   }
575 
576   @Override
577   public Result preIncrement(ObserverContext<RegionCoprocessorEnvironment> e, Increment increment)
578       throws IOException {
579     for (CellScanner cellScanner = increment.cellScanner(); cellScanner.advance();) {
580       if (!checkForReservedVisibilityTagPresence(cellScanner.current())) {
581         throw new FailedSanityCheckException("Increment contains cell with reserved type tag");
582       }
583     }
584     return null;
585   }
586 
587   @Override
588   public Cell postMutationBeforeWAL(ObserverContext<RegionCoprocessorEnvironment> ctx,
589       MutationType opType, Mutation mutation, Cell oldCell, Cell newCell) throws IOException {
590     List<Tag> tags = Lists.newArrayList();
591     CellVisibility cellVisibility = null;
592     try {
593       cellVisibility = mutation.getCellVisibility();
594     } catch (DeserializationException e) {
595       throw new IOException(e);
596     }
597     if (cellVisibility == null) {
598       return newCell;
599     }
600     // Prepend new visibility tags to a new list of tags for the cell
601     // Don't check user auths for labels with Mutations when the user is super user
602     boolean authCheck = this.checkAuths && !(isSystemOrSuperUser());
603     tags.addAll(this.visibilityLabelService.createVisibilityExpTags(cellVisibility.getExpression(),
604         true, authCheck));
605     // Save an object allocation where we can
606     if (newCell.getTagsLengthUnsigned() > 0) {
607       // Carry forward all other tags
608       Iterator<Tag> tagsItr = CellUtil.tagsIterator(newCell.getTagsArray(),
609           newCell.getTagsOffset(), newCell.getTagsLengthUnsigned());
610       while (tagsItr.hasNext()) {
611         Tag tag = tagsItr.next();
612         if (tag.getType() != TagType.VISIBILITY_TAG_TYPE
613             && tag.getType() != TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) {
614           tags.add(tag);
615         }
616       }
617     }
618 
619     // We need to create another KV, unfortunately, because the current new KV
620     // has no space for tags
621     KeyValue rewriteKv = new KeyValue(newCell.getRowArray(), newCell.getRowOffset(),
622         newCell.getRowLength(), newCell.getFamilyArray(), newCell.getFamilyOffset(),
623         newCell.getFamilyLength(), newCell.getQualifierArray(), newCell.getQualifierOffset(),
624         newCell.getQualifierLength(), newCell.getTimestamp(), KeyValue.Type.codeToType(newCell
625             .getTypeByte()), newCell.getValueArray(), newCell.getValueOffset(),
626         newCell.getValueLength(), tags);
627     // Preserve mvcc data
628     rewriteKv.setMvccVersion(newCell.getMvccVersion());
629     return rewriteKv;
630   }
631 
632   @Override
633   public Service getService() {
634     return VisibilityLabelsProtos.VisibilityLabelsService.newReflectiveService(this);
635   }
636 
637   /****************************** VisibilityEndpoint service related methods ******************************/
638   @Override
639   public synchronized void addLabels(RpcController controller, VisibilityLabelsRequest request,
640       RpcCallback<VisibilityLabelsResponse> done) {
641     VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder();
642     List<VisibilityLabel> visLabels = request.getVisLabelList();
643     if (!initialized) {
644       setExceptionResults(visLabels.size(),
645         new VisibilityControllerNotReadyException("VisibilityController not yet initialized!"),
646         response);
647     } else {
648       try {
649         checkCallingUserAuth();
650         List<byte[]> labels = new ArrayList<byte[]>(visLabels.size());
651         RegionActionResult successResult = RegionActionResult.newBuilder().build();
652         for (VisibilityLabel visLabel : visLabels) {
653           byte[] label = visLabel.getLabel().toByteArray();
654           labels.add(label);
655           response.addResult(successResult); // Just mark as success. Later it will get reset
656                                              // based on the result from
657                                              // visibilityLabelService.addLabels ()
658         }
659         if (!labels.isEmpty()) {
660           OperationStatus[] opStatus = this.visibilityLabelService.addLabels(labels);
661           int i = 0;
662           for (OperationStatus status : opStatus) {
663             while (response.getResult(i) != successResult)
664               i++;
665             if (status.getOperationStatusCode() != SUCCESS) {
666               RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
667               failureResultBuilder.setException(ResponseConverter
668                   .buildException(new DoNotRetryIOException(status.getExceptionMsg())));
669               response.setResult(i, failureResultBuilder.build());
670             }
671             i++;
672           }
673         }
674       } catch (IOException e) {
675         LOG.error(e);
676         setExceptionResults(visLabels.size(), e, response);
677       }
678     }
679     done.run(response.build());
680   }
681 
682   private void setExceptionResults(int size, IOException e,
683       VisibilityLabelsResponse.Builder response) {
684     RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
685     failureResultBuilder.setException(ResponseConverter.buildException(e));
686     RegionActionResult failureResult = failureResultBuilder.build();
687     for (int i = 0; i < size; i++) {
688       response.addResult(i, failureResult);
689     }
690   }
691 
692   @Override
693   public synchronized void setAuths(RpcController controller, SetAuthsRequest request,
694       RpcCallback<VisibilityLabelsResponse> done) {
695     VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder();
696     List<ByteString> auths = request.getAuthList();
697     if (!initialized) {
698       setExceptionResults(auths.size(),
699         new VisibilityControllerNotReadyException("VisibilityController not yet initialized!"),
700         response);
701     } else {
702       try {
703         checkCallingUserAuth();
704         List<byte[]> labelAuths = new ArrayList<byte[]>(auths.size());
705         for (ByteString authBS : auths) {
706           labelAuths.add(authBS.toByteArray());
707         }
708         OperationStatus[] opStatus = this.visibilityLabelService.setAuths(request.getUser()
709             .toByteArray(), labelAuths);
710         RegionActionResult successResult = RegionActionResult.newBuilder().build();
711         for (OperationStatus status : opStatus) {
712           if (status.getOperationStatusCode() == SUCCESS) {
713             response.addResult(successResult);
714           } else {
715             RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
716             failureResultBuilder.setException(ResponseConverter
717                 .buildException(new DoNotRetryIOException(status.getExceptionMsg())));
718             response.addResult(failureResultBuilder.build());
719           }
720         }
721       } catch (IOException e) {
722         LOG.error(e);
723         setExceptionResults(auths.size(), e, response);
724       }
725     }
726     done.run(response.build());
727   }
728 
729   @Override
730   public synchronized void getAuths(RpcController controller, GetAuthsRequest request,
731       RpcCallback<GetAuthsResponse> done) {
732     GetAuthsResponse.Builder response = GetAuthsResponse.newBuilder();
733     if (!initialized) {
734       controller.setFailed("VisibilityController not yet initialized");
735     } else {
736       byte[] user = request.getUser().toByteArray();
737       List<String> labels = null;
738       try {
739         // We do ACL check here as we create scanner directly on region. It will not make calls to
740         // AccessController CP methods.
741         if (this.acOn && !isSystemOrSuperUser()) {
742           User requestingUser = VisibilityUtils.getActiveUser();
743           throw new AccessDeniedException("User '"
744               + (requestingUser != null ? requestingUser.getShortName() : "null")
745               + " is not authorized to perform this action.");
746         }
747         labels = this.visibilityLabelService.getAuths(user, false);
748       } catch (IOException e) {
749         ResponseConverter.setControllerException(controller, e);
750       }
751       response.setUser(request.getUser());
752       if (labels != null) {
753         for (String label : labels) {
754           response.addAuth(ByteStringer.wrap(Bytes.toBytes(label)));
755         }
756       }
757     }
758     done.run(response.build());
759   }
760 
761   @Override
762   public synchronized void clearAuths(RpcController controller, SetAuthsRequest request,
763       RpcCallback<VisibilityLabelsResponse> done) {
764     VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder();
765     List<ByteString> auths = request.getAuthList();
766     if (!initialized) {
767       setExceptionResults(auths.size(), new CoprocessorException(
768           "VisibilityController not yet initialized"), response);
769     } else {
770       try {
771         // When AC is ON, do AC based user auth check
772         if (this.acOn && !isSystemOrSuperUser()) {
773           User user = VisibilityUtils.getActiveUser();
774           throw new AccessDeniedException("User '" + (user != null ? user.getShortName() : "null")
775               + " is not authorized to perform this action.");
776         }
777         checkCallingUserAuth(); // When AC is not in place the calling user should have SYSTEM_LABEL
778                                 // auth to do this action.
779         List<byte[]> labelAuths = new ArrayList<byte[]>(auths.size());
780         for (ByteString authBS : auths) {
781           labelAuths.add(authBS.toByteArray());
782         }
783         OperationStatus[] opStatus = this.visibilityLabelService.clearAuths(request.getUser()
784             .toByteArray(), labelAuths);
785         RegionActionResult successResult = RegionActionResult.newBuilder().build();
786         for (OperationStatus status : opStatus) {
787           if (status.getOperationStatusCode() == SUCCESS) {
788             response.addResult(successResult);
789           } else {
790             RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
791             failureResultBuilder.setException(ResponseConverter
792                 .buildException(new DoNotRetryIOException(status.getExceptionMsg())));
793             response.addResult(failureResultBuilder.build());
794           }
795         }
796       } catch (IOException e) {
797         LOG.error(e);
798         setExceptionResults(auths.size(), e, response);
799       }
800     }
801     done.run(response.build());
802   }
803 
804   private void checkCallingUserAuth() throws IOException {
805     if (!this.acOn) {
806       User user = VisibilityUtils.getActiveUser();
807       if (user == null) {
808         throw new IOException("Unable to retrieve calling user");
809       }
810       if (!(this.visibilityLabelService.havingSystemAuth(Bytes.toBytes(user.getShortName())))) {
811         throw new AccessDeniedException("User '" + user.getShortName()
812             + "' is not authorized to perform this action.");
813       }
814     }
815   }
816 
817   private static class DeleteVersionVisibilityExpressionFilter extends FilterBase {
818     private List<Tag> deleteCellVisTags;
819     private Byte deleteCellVisTagsFormat;
820 
821     public DeleteVersionVisibilityExpressionFilter(List<Tag> deleteCellVisTags,
822         Byte deleteCellVisTagsFormat) {
823       this.deleteCellVisTags = deleteCellVisTags;
824       this.deleteCellVisTagsFormat = deleteCellVisTagsFormat;
825     }
826 
827     @Override
828     public ReturnCode filterKeyValue(Cell cell) throws IOException {
829       List<Tag> putVisTags = new ArrayList<Tag>();
830       Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
831       boolean matchFound = VisibilityLabelServiceManager
832           .getInstance().getVisibilityLabelService()
833           .matchVisibility(putVisTags, putCellVisTagsFormat, deleteCellVisTags,
834               deleteCellVisTagsFormat);
835       return matchFound ? ReturnCode.INCLUDE : ReturnCode.SKIP;
836     }
837   }
838 }