View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   * http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.security.visibility;
20  
21  import static org.apache.hadoop.hbase.HConstants.OperationStatusCode.SANITY_CHECK_FAILURE;
22  import static org.apache.hadoop.hbase.HConstants.OperationStatusCode.SUCCESS;
23  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
24  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
25  
26  import java.io.IOException;
27  import java.util.ArrayList;
28  import java.util.HashMap;
29  import java.util.Iterator;
30  import java.util.List;
31  import java.util.Map;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.classification.InterfaceAudience;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.Cell;
38  import org.apache.hadoop.hbase.CellScanner;
39  import org.apache.hadoop.hbase.CellUtil;
40  import org.apache.hadoop.hbase.CoprocessorEnvironment;
41  import org.apache.hadoop.hbase.DoNotRetryIOException;
42  import org.apache.hadoop.hbase.HColumnDescriptor;
43  import org.apache.hadoop.hbase.HConstants;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.KeyValue;
46  import org.apache.hadoop.hbase.KeyValue.Type;
47  import org.apache.hadoop.hbase.KeyValueUtil;
48  import org.apache.hadoop.hbase.TableName;
49  import org.apache.hadoop.hbase.Tag;
50  import org.apache.hadoop.hbase.TagType;
51  import org.apache.hadoop.hbase.catalog.MetaReader;
52  import org.apache.hadoop.hbase.client.Append;
53  import org.apache.hadoop.hbase.client.Delete;
54  import org.apache.hadoop.hbase.client.Get;
55  import org.apache.hadoop.hbase.client.Increment;
56  import org.apache.hadoop.hbase.client.Mutation;
57  import org.apache.hadoop.hbase.client.Put;
58  import org.apache.hadoop.hbase.client.Result;
59  import org.apache.hadoop.hbase.client.Scan;
60  import org.apache.hadoop.hbase.constraint.ConstraintException;
61  import org.apache.hadoop.hbase.coprocessor.BaseMasterAndRegionObserver;
62  import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
63  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
64  import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
65  import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
66  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
67  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
68  import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
69  import org.apache.hadoop.hbase.exceptions.DeserializationException;
70  import org.apache.hadoop.hbase.exceptions.FailedSanityCheckException;
71  import org.apache.hadoop.hbase.filter.Filter;
72  import org.apache.hadoop.hbase.filter.FilterBase;
73  import org.apache.hadoop.hbase.filter.FilterList;
74  import org.apache.hadoop.hbase.io.hfile.HFile;
75  import org.apache.hadoop.hbase.ipc.RequestContext;
76  import org.apache.hadoop.hbase.master.MasterServices;
77  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
78  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
79  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos;
80  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsRequest;
81  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
82  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.SetAuthsRequest;
83  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel;
84  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest;
85  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
86  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsService;
87  import org.apache.hadoop.hbase.regionserver.BloomType;
88  import org.apache.hadoop.hbase.regionserver.DeleteTracker;
89  import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
90  import org.apache.hadoop.hbase.regionserver.HRegion;
91  import org.apache.hadoop.hbase.regionserver.InternalScanner;
92  import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
93  import org.apache.hadoop.hbase.regionserver.OperationStatus;
94  import org.apache.hadoop.hbase.regionserver.RegionScanner;
95  import org.apache.hadoop.hbase.security.AccessDeniedException;
96  import org.apache.hadoop.hbase.security.User;
97  import org.apache.hadoop.hbase.security.access.AccessControlLists;
98  import org.apache.hadoop.hbase.security.access.AccessController;
99  import org.apache.hadoop.hbase.util.ByteStringer;
100 import org.apache.hadoop.hbase.util.Bytes;
101 
102 import com.google.common.collect.Lists;
103 import com.google.common.collect.MapMaker;
104 import com.google.protobuf.ByteString;
105 import com.google.protobuf.RpcCallback;
106 import com.google.protobuf.RpcController;
107 import com.google.protobuf.Service;
108 
109 /**
110  * Coprocessor that has both the MasterObserver and RegionObserver implemented that supports in
111  * visibility labels
112  */
113 @InterfaceAudience.Private
114 public class VisibilityController extends BaseMasterAndRegionObserver implements
115     VisibilityLabelsService.Interface, CoprocessorService {
116 
117   private static final Log LOG = LogFactory.getLog(VisibilityController.class);
118   // flags if we are running on a region of the 'labels' table
119   private boolean labelsRegion = false;
120   // Flag denoting whether AcessController is available or not.
121   private boolean acOn = false;
122   private Configuration conf;
123   private volatile boolean initialized = false;
124   private boolean checkAuths = false;
125   /** Mapping of scanner instances to the user who created them */
126   private Map<InternalScanner,String> scannerOwners =
127       new MapMaker().weakKeys().makeMap();
128 
129   List<String> superUsers;
130   private VisibilityLabelService visibilityLabelService;
131 
132   // Add to this list if there are any reserved tag types
133   private static ArrayList<Byte> RESERVED_VIS_TAG_TYPES = new ArrayList<Byte>();
134   static {
135     RESERVED_VIS_TAG_TYPES.add(TagType.VISIBILITY_TAG_TYPE);
136     RESERVED_VIS_TAG_TYPES.add(TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE);
137   }
138 
139   @Override
140   public void start(CoprocessorEnvironment env) throws IOException {
141     this.conf = env.getConfiguration();
142     if (HFile.getFormatVersion(conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
143       throw new RuntimeException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS
144         + " is required to persist visibility labels. Consider setting " + HFile.FORMAT_VERSION_KEY
145         + " accordingly.");
146     }
147     if (env instanceof RegionServerCoprocessorEnvironment) {
148       throw new RuntimeException(
149           "Visibility controller should not be configured as " +
150           "'hbase.coprocessor.regionserver.classes'.");
151     }
152 
153     if (env instanceof RegionCoprocessorEnvironment) {
154       // VisibilityLabelService to be instantiated only with Region Observer.
155       visibilityLabelService = VisibilityLabelServiceManager.getInstance()
156           .getVisibilityLabelService(this.conf);
157     }
158     this.superUsers = getSystemAndSuperUsers();
159   }
160 
161   @Override
162   public void stop(CoprocessorEnvironment env) throws IOException {
163 
164   }
165 
166   /********************************* Master related hooks **********************************/
167 
168   @Override
169   public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
170     // Need to create the new system table for labels here
171     MasterServices master = ctx.getEnvironment().getMasterServices();
172     if (!MetaReader.tableExists(master.getCatalogTracker(), LABELS_TABLE_NAME)) {
173       HTableDescriptor labelsTable = new HTableDescriptor(LABELS_TABLE_NAME);
174       HColumnDescriptor labelsColumn = new HColumnDescriptor(LABELS_TABLE_FAMILY);
175       labelsColumn.setBloomFilterType(BloomType.NONE);
176       labelsColumn.setBlockCacheEnabled(false); // We will cache all the labels. No need of normal
177                                                  // table block cache.
178       labelsTable.addFamily(labelsColumn);
179       // Let the "labels" table having only one region always. We are not expecting too many labels in
180       // the system.
181       labelsTable.setValue(HTableDescriptor.SPLIT_POLICY,
182           DisabledRegionSplitPolicy.class.getName());
183       labelsTable.setValue(Bytes.toBytes(HConstants.DISALLOW_WRITES_IN_RECOVERING),
184           Bytes.toBytes(true));
185       master.createTable(labelsTable, null);
186     }
187   }
188 
189   @Override
190   public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
191       TableName tableName, HTableDescriptor htd) throws IOException {
192     if (LABELS_TABLE_NAME.equals(tableName)) {
193       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
194     }
195   }
196 
197   @Override
198   public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
199       HColumnDescriptor column) throws IOException {
200     if (LABELS_TABLE_NAME.equals(tableName)) {
201       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
202     }
203   }
204 
205   @Override
206   public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
207       TableName tableName, HColumnDescriptor descriptor) throws IOException {
208     if (LABELS_TABLE_NAME.equals(tableName)) {
209       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
210     }
211   }
212 
213   @Override
214   public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
215       TableName tableName, byte[] c) throws IOException {
216     if (LABELS_TABLE_NAME.equals(tableName)) {
217       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
218     }
219   }
220 
221   @Override
222   public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
223       throws IOException {
224     if (LABELS_TABLE_NAME.equals(tableName)) {
225       throw new ConstraintException("Cannot disable " + LABELS_TABLE_NAME);
226     }
227   }
228 
229   /****************************** Region related hooks ******************************/
230 
231   @Override
232   public void postOpen(ObserverContext<RegionCoprocessorEnvironment> e) {
233     // Read the entire labels table and populate the zk
234     if (e.getEnvironment().getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) {
235       this.labelsRegion = true;
236       this.acOn = CoprocessorHost.getLoadedCoprocessors().contains(AccessController.class.getName());
237       // Defer the init of VisibilityLabelService on labels region until it is in recovering state.
238       if (!e.getEnvironment().getRegion().isRecovering()) {
239         initVisibilityLabelService(e.getEnvironment());
240       }
241     } else {
242       checkAuths = e.getEnvironment().getConfiguration()
243           .getBoolean(VisibilityConstants.CHECK_AUTHS_FOR_MUTATION, false);
244       initVisibilityLabelService(e.getEnvironment());
245     }
246   }
247 
248   @Override
249   public void postLogReplay(ObserverContext<RegionCoprocessorEnvironment> e) {
250     if (this.labelsRegion) {
251       initVisibilityLabelService(e.getEnvironment());
252       LOG.debug("post labels region log replay");
253     }
254   }
255 
256   private void initVisibilityLabelService(RegionCoprocessorEnvironment env) {
257     try {
258       this.visibilityLabelService.init(env);
259       this.initialized = true;
260     } catch (IOException ioe) {
261       LOG.error("Error while initializing VisibilityLabelService..", ioe);
262       throw new RuntimeException(ioe);
263     }
264   }
265 
266   @Override
267   public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
268       MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
269     if (c.getEnvironment().getRegion().getRegionInfo().getTable().isSystemTable()) {
270       return;
271     }
272     // TODO this can be made as a global LRU cache at HRS level?
273     Map<String, List<Tag>> labelCache = new HashMap<String, List<Tag>>();
274     for (int i = 0; i < miniBatchOp.size(); i++) {
275       Mutation m = miniBatchOp.getOperation(i);
276       CellVisibility cellVisibility = null;
277       try {
278         cellVisibility = m.getCellVisibility();
279       } catch (DeserializationException de) {
280         miniBatchOp.setOperationStatus(i,
281             new OperationStatus(SANITY_CHECK_FAILURE, de.getMessage()));
282         continue;
283       }
284       boolean sanityFailure = false;
285       for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) {
286         if (!checkForReservedVisibilityTagPresence(cellScanner.current())) {
287           miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE,
288               "Mutation contains cell with reserved type tag"));
289           sanityFailure = true;
290           break;
291         }
292       }
293       if (!sanityFailure) {
294         if (cellVisibility != null) {
295           String labelsExp = cellVisibility.getExpression();
296           List<Tag> visibilityTags = labelCache.get(labelsExp);
297           if (visibilityTags == null) {
298             // Don't check user auths for labels with Mutations when the user is super user
299             boolean authCheck = this.checkAuths && !(isSystemOrSuperUser());
300             try {
301               visibilityTags = this.visibilityLabelService.createVisibilityExpTags(labelsExp, true,
302                   authCheck);
303             } catch (InvalidLabelException e) {
304               miniBatchOp.setOperationStatus(i,
305                   new OperationStatus(SANITY_CHECK_FAILURE, e.getMessage()));
306             }
307             if (visibilityTags != null) {
308               labelCache.put(labelsExp, visibilityTags);
309             }
310           }
311           if (visibilityTags != null) {
312             List<Cell> updatedCells = new ArrayList<Cell>();
313             for (CellScanner cellScanner = m.cellScanner(); cellScanner.advance();) {
314               Cell cell = cellScanner.current();
315               List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
316                   cell.getTagsLengthUnsigned());
317               tags.addAll(visibilityTags);
318               Cell updatedCell = new KeyValue(cell.getRowArray(), cell.getRowOffset(),
319                   cell.getRowLength(), cell.getFamilyArray(), cell.getFamilyOffset(),
320                   cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(),
321                   cell.getQualifierLength(), cell.getTimestamp(), Type.codeToType(cell
322                       .getTypeByte()), cell.getValueArray(), cell.getValueOffset(),
323                   cell.getValueLength(), tags);
324               updatedCells.add(updatedCell);
325             }
326             m.getFamilyCellMap().clear();
327             // Clear and add new Cells to the Mutation.
328             for (Cell cell : updatedCells) {
329               if (m instanceof Put) {
330                 Put p = (Put) m;
331                 p.add(cell);
332               } else if (m instanceof Delete) {
333                 // TODO : Cells without visibility tags would be handled in follow up issue
334                 Delete d = (Delete) m;
335                 d.addDeleteMarker(cell);
336               }
337             }
338           }
339         }
340       }
341     }
342   }
343 
344   @Override
345   public void prePrepareTimeStampForDeleteVersion(
346       ObserverContext<RegionCoprocessorEnvironment> ctx, Mutation delete, Cell cell,
347       byte[] byteNow, Get get) throws IOException {
348     KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
349     CellVisibility cellVisibility = null;
350     try {
351       cellVisibility = delete.getCellVisibility();
352     } catch (DeserializationException de) {
353       throw new IOException("Invalid cell visibility specified " + delete, de);
354     }
355     // The check for checkForReservedVisibilityTagPresence happens in preBatchMutate happens.
356     // It happens for every mutation and that would be enough.
357     List<Tag> visibilityTags = new ArrayList<Tag>();
358     if (cellVisibility != null) {
359       String labelsExp = cellVisibility.getExpression();
360       try {
361         visibilityTags = this.visibilityLabelService.createVisibilityExpTags(labelsExp, false,
362             false);
363       } catch (InvalidLabelException e) {
364         throw new IOException("Invalid cell visibility specified " + labelsExp, e);
365       }
366     }
367     get.setFilter(new DeleteVersionVisibilityExpressionFilter(visibilityTags,
368         VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT));
369     List<Cell> result = ctx.getEnvironment().getRegion().get(get, false);
370 
371     if (result.size() < get.getMaxVersions()) {
372       // Nothing to delete
373       kv.updateLatestStamp(Bytes.toBytes(Long.MIN_VALUE));
374       return;
375     }
376     if (result.size() > get.getMaxVersions()) {
377       throw new RuntimeException("Unexpected size: " + result.size()
378           + ". Results more than the max versions obtained.");
379     }
380     KeyValue getkv = KeyValueUtil.ensureKeyValue(result.get(get.getMaxVersions() - 1));
381     Bytes.putBytes(kv.getBuffer(), kv.getTimestampOffset(), getkv.getBuffer(),
382         getkv.getTimestampOffset(), Bytes.SIZEOF_LONG);
383     // We are bypassing here because in the HRegion.updateDeleteLatestVersionTimeStamp we would
384     // update with the current timestamp after again doing a get. As the hook as already determined
385     // the needed timestamp we need to bypass here.
386     // TODO : See if HRegion.updateDeleteLatestVersionTimeStamp() could be
387     // called only if the hook is not called.
388     ctx.bypass();
389   }
390 
391   // Checks whether cell contains any tag with type as VISIBILITY_TAG_TYPE.
392   // This tag type is reserved and should not be explicitly set by user.
393   private boolean checkForReservedVisibilityTagPresence(Cell cell) throws IOException {
394     // Bypass this check when the operation is done by a system/super user.
395     // This is done because, while Replication, the Cells coming to the peer cluster with reserved
396     // typed tags and this is fine and should get added to the peer cluster table
397     if (isSystemOrSuperUser()) {
398       return true;
399     }
400     if (cell.getTagsLengthUnsigned() > 0) {
401       Iterator<Tag> tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
402           cell.getTagsLengthUnsigned());
403       while (tagsItr.hasNext()) {
404         if (RESERVED_VIS_TAG_TYPES.contains(tagsItr.next().getType())) {
405           return false;
406         }
407       }
408     }
409     return true;
410   }
411 
412   @Override
413   public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
414       RegionScanner s) throws IOException {
415     if (!initialized) {
416       throw new VisibilityControllerNotReadyException("VisibilityController not yet initialized!");
417     }
418     HRegion region = e.getEnvironment().getRegion();
419     Authorizations authorizations = null;
420     try {
421       authorizations = scan.getAuthorizations();
422     } catch (DeserializationException de) {
423       throw new IOException(de);
424     }
425     if (authorizations == null) {
426       // No Authorizations present for this scan/Get!
427       // In case of system tables other than "labels" just scan with out visibility check and
428       // filtering. Checking visibility labels for META and NAMESPACE table is not needed.
429       TableName table = region.getRegionInfo().getTable();
430       if (table.isSystemTable() && !table.equals(LABELS_TABLE_NAME)) {
431         return s;
432       }
433     }
434 
435     Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(region,
436         authorizations);
437     if (visibilityLabelFilter != null) {
438       Filter filter = scan.getFilter();
439       if (filter != null) {
440         scan.setFilter(new FilterList(filter, visibilityLabelFilter));
441       } else {
442         scan.setFilter(visibilityLabelFilter);
443       }
444     }
445     return s;
446   }
447 
448   @Override
449   public DeleteTracker postInstantiateDeleteTracker(
450       ObserverContext<RegionCoprocessorEnvironment> ctx, DeleteTracker delTracker)
451       throws IOException {
452     HRegion region = ctx.getEnvironment().getRegion();
453     TableName table = region.getRegionInfo().getTable();
454     if (table.isSystemTable()) {
455       return delTracker;
456     }
457     // We are creating a new type of delete tracker here which is able to track
458     // the timestamps and also the
459     // visibility tags per cell. The covering cells are determined not only
460     // based on the delete type and ts
461     // but also on the visibility expression matching.
462     return new VisibilityScanDeleteTracker();
463   }
464 
465   @Override
466   public RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
467       final Scan scan, final RegionScanner s) throws IOException {
468     User user = VisibilityUtils.getActiveUser();
469     if (user != null && user.getShortName() != null) {
470       scannerOwners.put(s, user.getShortName());
471     }
472     return s;
473   }
474 
475   @Override
476   public boolean preScannerNext(final ObserverContext<RegionCoprocessorEnvironment> c,
477       final InternalScanner s, final List<Result> result, final int limit, final boolean hasNext)
478       throws IOException {
479     requireScannerOwner(s);
480     return hasNext;
481   }
482 
483   @Override
484   public void preScannerClose(final ObserverContext<RegionCoprocessorEnvironment> c,
485       final InternalScanner s) throws IOException {
486     requireScannerOwner(s);
487   }
488 
489   @Override
490   public void postScannerClose(final ObserverContext<RegionCoprocessorEnvironment> c,
491       final InternalScanner s) throws IOException {
492     // clean up any associated owner mapping
493     scannerOwners.remove(s);
494   }
495 
496   /**
497    * Verify, when servicing an RPC, that the caller is the scanner owner. If so, we assume that
498    * access control is correctly enforced based on the checks performed in preScannerOpen()
499    */
500   private void requireScannerOwner(InternalScanner s) throws AccessDeniedException {
501     if (RequestContext.isInRequestContext()) {
502       String requestUName = RequestContext.getRequestUserName();
503       String owner = scannerOwners.get(s);
504       if (owner != null && !owner.equals(requestUName)) {
505         throw new AccessDeniedException("User '" + requestUName + "' is not the scanner owner!");
506       }
507     }
508   }
509 
510   @Override
511   public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
512       throws IOException {
513     if (!initialized) {
514       throw new VisibilityControllerNotReadyException("VisibilityController not yet initialized!");
515     }
516     HRegion region = e.getEnvironment().getRegion();
517     Authorizations authorizations = null;
518     try {
519       authorizations = get.getAuthorizations();
520     } catch (DeserializationException de) {
521       throw new IOException(de);
522     }
523     if (authorizations == null) {
524       // No Authorizations present for this scan/Get!
525       // In case of system tables other than "labels" just scan with out visibility check and
526       // filtering. Checking visibility labels for META and NAMESPACE table is not needed.
527       TableName table = region.getRegionInfo().getTable();
528       if (table.isSystemTable() && !table.equals(LABELS_TABLE_NAME)) {
529         return;
530       }
531     }
532     Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(e.getEnvironment()
533         .getRegion(), authorizations);
534     if (visibilityLabelFilter != null) {
535       Filter filter = get.getFilter();
536       if (filter != null) {
537         get.setFilter(new FilterList(filter, visibilityLabelFilter));
538       } else {
539         get.setFilter(visibilityLabelFilter);
540       }
541     }
542   }
543 
544   private List<String> getSystemAndSuperUsers() throws IOException {
545     User user = User.getCurrent();
546     if (user == null) {
547       throw new IOException("Unable to obtain the current user, "
548           + "authorization checks for internal operations will not work correctly!");
549     }
550     if (LOG.isTraceEnabled()) {
551       LOG.trace("Current user name is "+user.getShortName());
552     }
553     String currentUser = user.getShortName();
554     List<String> superUsers = Lists.asList(currentUser,
555         this.conf.getStrings(AccessControlLists.SUPERUSER_CONF_KEY, new String[0]));
556     return superUsers;
557   }
558 
559   private boolean isSystemOrSuperUser() throws IOException {
560     User activeUser = VisibilityUtils.getActiveUser();
561     return this.superUsers.contains(activeUser.getShortName());
562   }
563 
564   @Override
565   public Result preAppend(ObserverContext<RegionCoprocessorEnvironment> e, Append append)
566       throws IOException {
567     for (CellScanner cellScanner = append.cellScanner(); cellScanner.advance();) {
568       if (!checkForReservedVisibilityTagPresence(cellScanner.current())) {
569         throw new FailedSanityCheckException("Append contains cell with reserved type tag");
570       }
571     }
572     return null;
573   }
574 
575   @Override
576   public Result preIncrement(ObserverContext<RegionCoprocessorEnvironment> e, Increment increment)
577       throws IOException {
578     for (CellScanner cellScanner = increment.cellScanner(); cellScanner.advance();) {
579       if (!checkForReservedVisibilityTagPresence(cellScanner.current())) {
580         throw new FailedSanityCheckException("Increment contains cell with reserved type tag");
581       }
582     }
583     return null;
584   }
585 
586   @Override
587   public Cell postMutationBeforeWAL(ObserverContext<RegionCoprocessorEnvironment> ctx,
588       MutationType opType, Mutation mutation, Cell oldCell, Cell newCell) throws IOException {
589     List<Tag> tags = Lists.newArrayList();
590     CellVisibility cellVisibility = null;
591     try {
592       cellVisibility = mutation.getCellVisibility();
593     } catch (DeserializationException e) {
594       throw new IOException(e);
595     }
596     if (cellVisibility == null) {
597       return newCell;
598     }
599     // Prepend new visibility tags to a new list of tags for the cell
600     // Don't check user auths for labels with Mutations when the user is super user
601     boolean authCheck = this.checkAuths && !(isSystemOrSuperUser());
602     tags.addAll(this.visibilityLabelService.createVisibilityExpTags(cellVisibility.getExpression(),
603         true, authCheck));
604     // Save an object allocation where we can
605     if (newCell.getTagsLengthUnsigned() > 0) {
606       // Carry forward all other tags
607       Iterator<Tag> tagsItr = CellUtil.tagsIterator(newCell.getTagsArray(),
608           newCell.getTagsOffset(), newCell.getTagsLengthUnsigned());
609       while (tagsItr.hasNext()) {
610         Tag tag = tagsItr.next();
611         if (tag.getType() != TagType.VISIBILITY_TAG_TYPE
612             && tag.getType() != TagType.VISIBILITY_EXP_SERIALIZATION_FORMAT_TAG_TYPE) {
613           tags.add(tag);
614         }
615       }
616     }
617 
618     // We need to create another KV, unfortunately, because the current new KV
619     // has no space for tags
620     KeyValue rewriteKv = new KeyValue(newCell.getRowArray(), newCell.getRowOffset(),
621         newCell.getRowLength(), newCell.getFamilyArray(), newCell.getFamilyOffset(),
622         newCell.getFamilyLength(), newCell.getQualifierArray(), newCell.getQualifierOffset(),
623         newCell.getQualifierLength(), newCell.getTimestamp(), KeyValue.Type.codeToType(newCell
624             .getTypeByte()), newCell.getValueArray(), newCell.getValueOffset(),
625         newCell.getValueLength(), tags);
626     // Preserve mvcc data
627     rewriteKv.setMvccVersion(newCell.getMvccVersion());
628     return rewriteKv;
629   }
630 
631   @Override
632   public Service getService() {
633     return VisibilityLabelsProtos.VisibilityLabelsService.newReflectiveService(this);
634   }
635 
636   /****************************** VisibilityEndpoint service related methods ******************************/
637   @Override
638   public synchronized void addLabels(RpcController controller, VisibilityLabelsRequest request,
639       RpcCallback<VisibilityLabelsResponse> done) {
640     VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder();
641     List<VisibilityLabel> visLabels = request.getVisLabelList();
642     if (!initialized) {
643       setExceptionResults(visLabels.size(),
644         new VisibilityControllerNotReadyException("VisibilityController not yet initialized!"),
645         response);
646     } else {
647       try {
648         checkCallingUserAuth();
649         List<byte[]> labels = new ArrayList<byte[]>(visLabels.size());
650         RegionActionResult successResult = RegionActionResult.newBuilder().build();
651         for (VisibilityLabel visLabel : visLabels) {
652           byte[] label = visLabel.getLabel().toByteArray();
653           labels.add(label);
654           response.addResult(successResult); // Just mark as success. Later it will get reset
655                                              // based on the result from
656                                              // visibilityLabelService.addLabels ()
657         }
658         if (!labels.isEmpty()) {
659           OperationStatus[] opStatus = this.visibilityLabelService.addLabels(labels);
660           int i = 0;
661           for (OperationStatus status : opStatus) {
662             while (response.getResult(i) != successResult)
663               i++;
664             if (status.getOperationStatusCode() != SUCCESS) {
665               RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
666               failureResultBuilder.setException(ResponseConverter
667                   .buildException(new DoNotRetryIOException(status.getExceptionMsg())));
668               response.setResult(i, failureResultBuilder.build());
669             }
670             i++;
671           }
672         }
673       } catch (IOException e) {
674         LOG.error(e);
675         setExceptionResults(visLabels.size(), e, response);
676       }
677     }
678     done.run(response.build());
679   }
680 
681   private void setExceptionResults(int size, IOException e,
682       VisibilityLabelsResponse.Builder response) {
683     RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
684     failureResultBuilder.setException(ResponseConverter.buildException(e));
685     RegionActionResult failureResult = failureResultBuilder.build();
686     for (int i = 0; i < size; i++) {
687       response.addResult(i, failureResult);
688     }
689   }
690 
691   @Override
692   public synchronized void setAuths(RpcController controller, SetAuthsRequest request,
693       RpcCallback<VisibilityLabelsResponse> done) {
694     VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder();
695     List<ByteString> auths = request.getAuthList();
696     if (!initialized) {
697       setExceptionResults(auths.size(),
698         new VisibilityControllerNotReadyException("VisibilityController not yet initialized!"),
699         response);
700     } else {
701       try {
702         checkCallingUserAuth();
703         List<byte[]> labelAuths = new ArrayList<byte[]>(auths.size());
704         for (ByteString authBS : auths) {
705           labelAuths.add(authBS.toByteArray());
706         }
707         OperationStatus[] opStatus = this.visibilityLabelService.setAuths(request.getUser()
708             .toByteArray(), labelAuths);
709         RegionActionResult successResult = RegionActionResult.newBuilder().build();
710         for (OperationStatus status : opStatus) {
711           if (status.getOperationStatusCode() == SUCCESS) {
712             response.addResult(successResult);
713           } else {
714             RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
715             failureResultBuilder.setException(ResponseConverter
716                 .buildException(new DoNotRetryIOException(status.getExceptionMsg())));
717             response.addResult(failureResultBuilder.build());
718           }
719         }
720       } catch (IOException e) {
721         LOG.error(e);
722         setExceptionResults(auths.size(), e, response);
723       }
724     }
725     done.run(response.build());
726   }
727 
728   @Override
729   public synchronized void getAuths(RpcController controller, GetAuthsRequest request,
730       RpcCallback<GetAuthsResponse> done) {
731     GetAuthsResponse.Builder response = GetAuthsResponse.newBuilder();
732     if (!initialized) {
733       controller.setFailed("VisibilityController not yet initialized");
734     } else {
735       byte[] user = request.getUser().toByteArray();
736       List<String> labels = null;
737       try {
738         // We do ACL check here as we create scanner directly on region. It will not make calls to
739         // AccessController CP methods.
740         if (this.acOn && !isSystemOrSuperUser()) {
741           User requestingUser = VisibilityUtils.getActiveUser();
742           throw new AccessDeniedException("User '"
743               + (requestingUser != null ? requestingUser.getShortName() : "null")
744               + " is not authorized to perform this action.");
745         }
746         labels = this.visibilityLabelService.getAuths(user, false);
747       } catch (IOException e) {
748         ResponseConverter.setControllerException(controller, e);
749       }
750       response.setUser(request.getUser());
751       if (labels != null) {
752         for (String label : labels) {
753           response.addAuth(ByteStringer.wrap(Bytes.toBytes(label)));
754         }
755       }
756     }
757     done.run(response.build());
758   }
759 
760   @Override
761   public synchronized void clearAuths(RpcController controller, SetAuthsRequest request,
762       RpcCallback<VisibilityLabelsResponse> done) {
763     VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder();
764     List<ByteString> auths = request.getAuthList();
765     if (!initialized) {
766       setExceptionResults(auths.size(), new CoprocessorException(
767           "VisibilityController not yet initialized"), response);
768     } else {
769       try {
770         // When AC is ON, do AC based user auth check
771         if (this.acOn && !isSystemOrSuperUser()) {
772           User user = VisibilityUtils.getActiveUser();
773           throw new AccessDeniedException("User '" + (user != null ? user.getShortName() : "null")
774               + " is not authorized to perform this action.");
775         }
776         checkCallingUserAuth(); // When AC is not in place the calling user should have SYSTEM_LABEL
777                                 // auth to do this action.
778         List<byte[]> labelAuths = new ArrayList<byte[]>(auths.size());
779         for (ByteString authBS : auths) {
780           labelAuths.add(authBS.toByteArray());
781         }
782         OperationStatus[] opStatus = this.visibilityLabelService.clearAuths(request.getUser()
783             .toByteArray(), labelAuths);
784         RegionActionResult successResult = RegionActionResult.newBuilder().build();
785         for (OperationStatus status : opStatus) {
786           if (status.getOperationStatusCode() == SUCCESS) {
787             response.addResult(successResult);
788           } else {
789             RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
790             failureResultBuilder.setException(ResponseConverter
791                 .buildException(new DoNotRetryIOException(status.getExceptionMsg())));
792             response.addResult(failureResultBuilder.build());
793           }
794         }
795       } catch (IOException e) {
796         LOG.error(e);
797         setExceptionResults(auths.size(), e, response);
798       }
799     }
800     done.run(response.build());
801   }
802 
803   private void checkCallingUserAuth() throws IOException {
804     if (!this.acOn) {
805       User user = VisibilityUtils.getActiveUser();
806       if (user == null) {
807         throw new IOException("Unable to retrieve calling user");
808       }
809       if (!(this.visibilityLabelService.havingSystemAuth(Bytes.toBytes(user.getShortName())))) {
810         throw new AccessDeniedException("User '" + user.getShortName()
811             + "' is not authorized to perform this action.");
812       }
813     }
814   }
815 
816   private static class DeleteVersionVisibilityExpressionFilter extends FilterBase {
817     private List<Tag> deleteCellVisTags;
818     private Byte deleteCellVisTagsFormat;
819 
820     public DeleteVersionVisibilityExpressionFilter(List<Tag> deleteCellVisTags,
821         Byte deleteCellVisTagsFormat) {
822       this.deleteCellVisTags = deleteCellVisTags;
823       this.deleteCellVisTagsFormat = deleteCellVisTagsFormat;
824     }
825 
826     @Override
827     public ReturnCode filterKeyValue(Cell cell) throws IOException {
828       List<Tag> putVisTags = new ArrayList<Tag>();
829       Byte putCellVisTagsFormat = VisibilityUtils.extractVisibilityTags(cell, putVisTags);
830       boolean matchFound = VisibilityLabelServiceManager
831           .getInstance().getVisibilityLabelService()
832           .matchVisibility(putVisTags, putCellVisTagsFormat, deleteCellVisTags,
833               deleteCellVisTagsFormat);
834       return matchFound ? ReturnCode.INCLUDE : ReturnCode.SKIP;
835     }
836   }
837 }