View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   * http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.security.visibility;
20  
21  import static org.apache.hadoop.hbase.HConstants.OperationStatusCode.SANITY_CHECK_FAILURE;
22  import static org.apache.hadoop.hbase.HConstants.OperationStatusCode.SUCCESS;
23  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
24  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
25  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
26  import static org.apache.hadoop.hbase.security.visibility.VisibilityUtils.SYSTEM_LABEL;
27  
28  import java.io.ByteArrayOutputStream;
29  import java.io.DataOutputStream;
30  import java.io.IOException;
31  import java.util.ArrayList;
32  import java.util.BitSet;
33  import java.util.HashMap;
34  import java.util.Iterator;
35  import java.util.List;
36  import java.util.Map;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.hadoop.classification.InterfaceAudience;
41  import org.apache.hadoop.conf.Configuration;
42  import org.apache.hadoop.hbase.Cell;
43  import org.apache.hadoop.hbase.CellScanner;
44  import org.apache.hadoop.hbase.CellUtil;
45  import org.apache.hadoop.hbase.CoprocessorEnvironment;
46  import org.apache.hadoop.hbase.DoNotRetryIOException;
47  import org.apache.hadoop.hbase.HColumnDescriptor;
48  import org.apache.hadoop.hbase.HConstants;
49  import org.apache.hadoop.hbase.HRegionInfo;
50  import org.apache.hadoop.hbase.HTableDescriptor;
51  import org.apache.hadoop.hbase.KeyValue;
52  import org.apache.hadoop.hbase.KeyValue.Type;
53  import org.apache.hadoop.hbase.KeyValueUtil;
54  import org.apache.hadoop.hbase.NamespaceDescriptor;
55  import org.apache.hadoop.hbase.ServerName;
56  import org.apache.hadoop.hbase.TableName;
57  import org.apache.hadoop.hbase.Tag;
58  import org.apache.hadoop.hbase.catalog.MetaReader;
59  import org.apache.hadoop.hbase.client.Delete;
60  import org.apache.hadoop.hbase.client.Get;
61  import org.apache.hadoop.hbase.client.Mutation;
62  import org.apache.hadoop.hbase.client.Put;
63  import org.apache.hadoop.hbase.client.Result;
64  import org.apache.hadoop.hbase.client.Scan;
65  import org.apache.hadoop.hbase.constraint.ConstraintException;
66  import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
67  import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
68  import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
69  import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
70  import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment;
71  import org.apache.hadoop.hbase.coprocessor.MasterObserver;
72  import org.apache.hadoop.hbase.coprocessor.ObserverContext;
73  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
74  import org.apache.hadoop.hbase.coprocessor.RegionObserver;
75  import org.apache.hadoop.hbase.coprocessor.RegionServerCoprocessorEnvironment;
76  import org.apache.hadoop.hbase.exceptions.DeserializationException;
77  import org.apache.hadoop.hbase.filter.Filter;
78  import org.apache.hadoop.hbase.filter.FilterList;
79  import org.apache.hadoop.hbase.io.hfile.HFile;
80  import org.apache.hadoop.hbase.io.util.StreamUtils;
81  import org.apache.hadoop.hbase.ipc.RequestContext;
82  import org.apache.hadoop.hbase.master.MasterServices;
83  import org.apache.hadoop.hbase.master.RegionPlan;
84  import org.apache.hadoop.hbase.protobuf.ResponseConverter;
85  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
86  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
87  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos;
88  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsRequest;
89  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
90  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.SetAuthsRequest;
91  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabel;
92  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsRequest;
93  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
94  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsService;
95  import org.apache.hadoop.hbase.regionserver.BloomType;
96  import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
97  import org.apache.hadoop.hbase.regionserver.HRegion;
98  import org.apache.hadoop.hbase.regionserver.InternalScanner;
99  import org.apache.hadoop.hbase.regionserver.MiniBatchOperationInProgress;
100 import org.apache.hadoop.hbase.regionserver.OperationStatus;
101 import org.apache.hadoop.hbase.regionserver.RegionScanner;
102 import org.apache.hadoop.hbase.security.AccessDeniedException;
103 import org.apache.hadoop.hbase.security.User;
104 import org.apache.hadoop.hbase.security.access.AccessControlLists;
105 import org.apache.hadoop.hbase.security.access.AccessController;
106 import org.apache.hadoop.hbase.security.visibility.expression.ExpressionNode;
107 import org.apache.hadoop.hbase.security.visibility.expression.LeafExpressionNode;
108 import org.apache.hadoop.hbase.security.visibility.expression.NonLeafExpressionNode;
109 import org.apache.hadoop.hbase.security.visibility.expression.Operator;
110 import org.apache.hadoop.hbase.util.ByteRange;
111 import org.apache.hadoop.hbase.util.Bytes;
112 import org.apache.hadoop.hbase.util.Pair;
113 import org.apache.hadoop.hbase.util.SimpleByteRange;
114 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
115 
116 import com.google.common.collect.Lists;
117 import com.google.common.collect.MapMaker;
118 import com.google.protobuf.ByteString;
119 import com.google.protobuf.HBaseZeroCopyByteString;
120 import com.google.protobuf.RpcCallback;
121 import com.google.protobuf.RpcController;
122 import com.google.protobuf.Service;
123 
124 /**
125  * Coprocessor that has both the MasterObserver and RegionObserver implemented that supports in
126  * visibility labels
127  */
128 @InterfaceAudience.Private
129 public class VisibilityController extends BaseRegionObserver implements MasterObserver,
130     RegionObserver, VisibilityLabelsService.Interface, CoprocessorService {
131 
132   private static final Log LOG = LogFactory.getLog(VisibilityController.class);
133   private static final byte[] DUMMY_VALUE = new byte[0];
134   // "system" label is having an ordinal value 1.
135   private static final int SYSTEM_LABEL_ORDINAL = 1;
136   private static final Tag[] LABELS_TABLE_TAGS = new Tag[1];
137 
138   private final ExpressionParser expressionParser = new ExpressionParser();
139   private final ExpressionExpander expressionExpander = new ExpressionExpander();
140   private VisibilityLabelsManager visibilityManager;
141   // defined only for Endpoint implementation, so it can have way to access region services.
142   private RegionCoprocessorEnvironment regionEnv;
143   private List<ScanLabelGenerator> scanLabelGenerators;
144   
145   private volatile int ordinalCounter = -1;
146   // flags if we are running on a region of the 'labels' table
147   private boolean labelsRegion = false;
148   // Flag denoting whether AcessController is available or not.
149   private boolean acOn = false;
150   private Configuration conf;
151   private volatile boolean initialized = false;
152 
153   /** Mapping of scanner instances to the user who created them */
154   private Map<InternalScanner,String> scannerOwners =
155       new MapMaker().weakKeys().makeMap();
156 
157   static {
158     ByteArrayOutputStream baos = new ByteArrayOutputStream();
159     DataOutputStream dos = new DataOutputStream(baos);
160     try {
161       StreamUtils.writeRawVInt32(dos, SYSTEM_LABEL_ORDINAL);
162     } catch (IOException e) {
163       // We write to a byte array. No Exception can happen.
164     }
165     LABELS_TABLE_TAGS[0] = new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray());
166   }
167 
168   @Override
169   public void start(CoprocessorEnvironment env) throws IOException {
170     this.conf = env.getConfiguration();
171     if (HFile.getFormatVersion(conf) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
172       throw new RuntimeException("A minimum HFile version of " + HFile.MIN_FORMAT_VERSION_WITH_TAGS
173         + " is required to persist visibility labels. Consider setting " + HFile.FORMAT_VERSION_KEY
174         + " accordingly.");
175     }
176     ZooKeeperWatcher zk = null;
177     if (env instanceof MasterCoprocessorEnvironment) {
178       // if running on HMaster
179       MasterCoprocessorEnvironment mEnv = (MasterCoprocessorEnvironment) env;
180       zk = mEnv.getMasterServices().getZooKeeper();
181     } else if (env instanceof RegionCoprocessorEnvironment) {
182       // if running at region
183       regionEnv = (RegionCoprocessorEnvironment) env;
184       zk = regionEnv.getRegionServerServices().getZooKeeper();
185     } else if (env instanceof RegionServerCoprocessorEnvironment) {
186       throw new RuntimeException(
187           "Visibility controller should not be configured as " +
188           "'hbase.coprocessor.regionserver.classes'.");
189     }
190 
191     // If zk is null or IOException while obtaining auth manager,
192     // throw RuntimeException so that the coprocessor is unloaded.
193     if (zk == null) {
194       throw new RuntimeException("Error obtaining VisibilityLabelsManager, zk found null.");
195     }
196     try {
197       this.visibilityManager = VisibilityLabelsManager.get(zk, this.conf);
198     } catch (IOException ioe) {
199       throw new RuntimeException("Error obtaining VisibilityLabelsManager", ioe);
200     }
201     if (env instanceof RegionCoprocessorEnvironment) {
202       // ScanLabelGenerator to be instantiated only with Region Observer.
203       scanLabelGenerators = VisibilityUtils.getScanLabelGenerators(this.conf);
204     }
205   }
206 
207   @Override
208   public void stop(CoprocessorEnvironment env) throws IOException {
209 
210   }
211 
212   /********************************* Master related hooks **********************************/
213 
214   @Override
215   public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
216     // Need to create the new system table for labels here
217     MasterServices master = ctx.getEnvironment().getMasterServices();
218     if (!MetaReader.tableExists(master.getCatalogTracker(), LABELS_TABLE_NAME)) {
219       HTableDescriptor labelsTable = new HTableDescriptor(LABELS_TABLE_NAME);
220       HColumnDescriptor labelsColumn = new HColumnDescriptor(LABELS_TABLE_FAMILY);
221       labelsColumn.setBloomFilterType(BloomType.NONE);
222       labelsColumn.setBlockCacheEnabled(false); // We will cache all the labels. No need of normal
223                                                  // table block cache.
224       labelsTable.addFamily(labelsColumn);
225       // Let the "labels" table having only one region always. We are not expecting too many labels in
226       // the system.
227       labelsTable.setValue(HTableDescriptor.SPLIT_POLICY,
228           DisabledRegionSplitPolicy.class.getName());
229       labelsTable.setValue(Bytes.toBytes(HConstants.DISALLOW_WRITES_IN_RECOVERING),
230           Bytes.toBytes(true));
231       master.createTable(labelsTable, null);
232     }
233   }
234 
235   @Override
236   public void preCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
237       HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
238   }
239 
240   @Override
241   public void postCreateTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
242       HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
243   }
244 
245   @Override
246   public void preCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
247       HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
248   }
249 
250   @Override
251   public void postCreateTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
252       HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
253   }
254 
255   @Override
256   public void preDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
257       throws IOException {
258   }
259 
260   @Override
261   public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
262       throws IOException {
263   }
264 
265   @Override
266   public void preDeleteTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
267       TableName tableName) throws IOException {
268   }
269 
270   @Override
271   public void postDeleteTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
272       TableName tableName) throws IOException {
273   }
274 
275   @Override
276   public void preModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
277       TableName tableName, HTableDescriptor htd) throws IOException {
278     if (LABELS_TABLE_NAME.equals(tableName)) {
279       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
280     }
281   }
282 
283   @Override
284   public void postModifyTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
285       TableName tableName, HTableDescriptor htd) throws IOException {
286   }
287 
288   @Override
289   public void preModifyTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
290       TableName tableName, HTableDescriptor htd) throws IOException {
291   }
292 
293   @Override
294   public void postModifyTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
295       TableName tableName, HTableDescriptor htd) throws IOException {
296   }
297 
298   @Override
299   public void preAddColumn(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
300       HColumnDescriptor column) throws IOException {
301     if (LABELS_TABLE_NAME.equals(tableName)) {
302       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
303     }
304   }
305 
306   @Override
307   public void postAddColumn(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName,
308       HColumnDescriptor column) throws IOException {
309   }
310 
311   @Override
312   public void preAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
313       TableName tableName, HColumnDescriptor column) throws IOException {
314   }
315 
316   @Override
317   public void postAddColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
318       TableName tableName, HColumnDescriptor column) throws IOException {
319   }
320 
321   @Override
322   public void preModifyColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
323       TableName tableName, HColumnDescriptor descriptor) throws IOException {
324     if (LABELS_TABLE_NAME.equals(tableName)) {
325       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
326     }
327   }
328 
329   @Override
330   public void postModifyColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
331       TableName tableName, HColumnDescriptor descriptor) throws IOException {
332   }
333 
334   @Override
335   public void preModifyColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
336       TableName tableName, HColumnDescriptor descriptor) throws IOException {
337   }
338 
339   @Override
340   public void postModifyColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
341       TableName tableName, HColumnDescriptor descriptor) throws IOException {
342   }
343 
344   @Override
345   public void preDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
346       TableName tableName, byte[] c) throws IOException {
347     if (LABELS_TABLE_NAME.equals(tableName)) {
348       throw new ConstraintException("Cannot alter " + LABELS_TABLE_NAME);
349     }
350   }
351 
352   @Override
353   public void postDeleteColumn(ObserverContext<MasterCoprocessorEnvironment> ctx,
354       TableName tableName, byte[] c) throws IOException {
355   }
356 
357   @Override
358   public void preDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
359       TableName tableName, byte[] c) throws IOException {
360   }
361 
362   @Override
363   public void postDeleteColumnHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
364       TableName tableName, byte[] c) throws IOException {
365   }
366 
367   @Override
368   public void preEnableTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
369       throws IOException {
370   }
371 
372   @Override
373   public void postEnableTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
374       throws IOException {
375   }
376 
377   @Override
378   public void preEnableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
379       TableName tableName) throws IOException {
380   }
381 
382   @Override
383   public void postEnableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
384       TableName tableName) throws IOException {
385   }
386 
387   @Override
388   public void preDisableTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName)
389       throws IOException {
390     if (LABELS_TABLE_NAME.equals(tableName)) {
391       throw new ConstraintException("Cannot disable " + LABELS_TABLE_NAME);
392     }
393   }
394 
395   @Override
396   public void postDisableTable(ObserverContext<MasterCoprocessorEnvironment> ctx,
397       TableName tableName) throws IOException {
398   }
399 
400   @Override
401   public void preDisableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
402       TableName tableName) throws IOException {
403   }
404 
405   @Override
406   public void postDisableTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
407       TableName tableName) throws IOException {
408   }
409 
410   @Override
411   public void preMove(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo region,
412       ServerName srcServer, ServerName destServer) throws IOException {
413   }
414 
415   @Override
416   public void postMove(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo region,
417       ServerName srcServer, ServerName destServer) throws IOException {
418   }
419 
420   @Override
421   public void preAssign(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo regionInfo)
422       throws IOException {
423   }
424 
425   @Override
426   public void postAssign(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo regionInfo)
427       throws IOException {
428   }
429 
430   @Override
431   public void preUnassign(ObserverContext<MasterCoprocessorEnvironment> ctx,
432       HRegionInfo regionInfo, boolean force) throws IOException {
433   }
434 
435   @Override
436   public void postUnassign(ObserverContext<MasterCoprocessorEnvironment> ctx,
437       HRegionInfo regionInfo, boolean force) throws IOException {
438   }
439 
440   @Override
441   public void preRegionOffline(ObserverContext<MasterCoprocessorEnvironment> ctx,
442       HRegionInfo regionInfo) throws IOException {
443   }
444 
445   @Override
446   public void postRegionOffline(ObserverContext<MasterCoprocessorEnvironment> ctx,
447       HRegionInfo regionInfo) throws IOException {
448   }
449 
450   @Override
451   public void preBalance(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
452   }
453 
454   @Override
455   public void postBalance(ObserverContext<MasterCoprocessorEnvironment> ctx, List<RegionPlan> plans)
456       throws IOException {
457   }
458 
459   @Override
460   public boolean preBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> ctx,
461       boolean newValue) throws IOException {
462     return false;
463   }
464 
465   @Override
466   public void postBalanceSwitch(ObserverContext<MasterCoprocessorEnvironment> ctx,
467       boolean oldValue, boolean newValue) throws IOException {
468   }
469 
470   @Override
471   public void preShutdown(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
472   }
473 
474   @Override
475   public void preStopMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
476   }
477 
478   @Override
479   public void preSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
480       SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
481   }
482 
483   @Override
484   public void postSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
485       SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
486   }
487 
488   @Override
489   public void preCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
490       SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
491   }
492 
493   @Override
494   public void postCloneSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
495       SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
496   }
497 
498   @Override
499   public void preRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
500       SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
501   }
502 
503   @Override
504   public void postRestoreSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
505       SnapshotDescription snapshot, HTableDescriptor hTableDescriptor) throws IOException {
506   }
507 
508   @Override
509   public void preDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
510       SnapshotDescription snapshot) throws IOException {
511   }
512 
513   @Override
514   public void postDeleteSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
515       SnapshotDescription snapshot) throws IOException {
516   }
517 
518   @Override
519   public void preGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
520       List<TableName> tableNamesList, List<HTableDescriptor> descriptors) throws IOException {
521   }
522 
523   @Override
524   public void postGetTableDescriptors(ObserverContext<MasterCoprocessorEnvironment> ctx,
525       List<HTableDescriptor> descriptors) throws IOException {
526   }
527 
528   @Override
529   public void preCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx,
530       NamespaceDescriptor ns) throws IOException {
531   }
532 
533   @Override
534   public void postCreateNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx,
535       NamespaceDescriptor ns) throws IOException {
536   }
537 
538   @Override
539   public void preDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx, 
540       String namespace) throws IOException {
541   }
542 
543   @Override
544   public void postDeleteNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx,
545       String namespace) throws IOException {
546   }
547 
548   @Override
549   public void preModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx,
550       NamespaceDescriptor ns) throws IOException {
551   }
552 
553   @Override
554   public void postModifyNamespace(ObserverContext<MasterCoprocessorEnvironment> ctx,
555       NamespaceDescriptor ns) throws IOException {
556   }
557 
558   @Override
559   public void preMasterInitialization(ObserverContext<MasterCoprocessorEnvironment> ctx)
560       throws IOException {
561 
562   }
563 
564   /****************************** Region related hooks ******************************/
565 
566   @Override
567   public void postOpen(ObserverContext<RegionCoprocessorEnvironment> e) {
568     // Read the entire labels table and populate the zk
569     if (e.getEnvironment().getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) {
570       this.labelsRegion = true;
571       this.acOn = CoprocessorHost.getLoadedCoprocessors().contains(AccessController.class.getName());
572       if (!e.getEnvironment().getRegion().isRecovering()) {
573         initialize(e);
574       }
575     } else {
576       this.initialized = true;
577     }
578   }
579 
580   @Override
581   public void postLogReplay(ObserverContext<RegionCoprocessorEnvironment> e) {
582     if (this.labelsRegion) {
583       initialize(e);
584     }
585   }
586 
587   private void initialize(ObserverContext<RegionCoprocessorEnvironment> e) {
588     try {
589       Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths = 
590           extractLabelsAndAuths(getExistingLabelsWithAuths());
591       Map<String, Integer> labels = labelsAndUserAuths.getFirst();
592       Map<String, List<Integer>> userAuths = labelsAndUserAuths.getSecond();
593       // Add the "system" label if it is not added into the system yet
594       addSystemLabel(e.getEnvironment().getRegion(), labels, userAuths);
595       int ordinal = 1; // Ordinal 1 is reserved for "system" label.
596       for (Integer i : labels.values()) {
597         if (i > ordinal) {
598           ordinal = i;
599         }
600       }
601       this.ordinalCounter = ordinal + 1;
602       if (labels.size() > 0) {
603         // If there is no data need not write to zk
604         byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(labels);
605         this.visibilityManager.writeToZookeeper(serialized, true);
606       }
607       if (userAuths.size() > 0) {
608         byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths);
609         this.visibilityManager.writeToZookeeper(serialized, false);
610       }
611       initialized = true;
612     } catch (IOException ioe) {
613       LOG.error("Error while updating the zk with the exisiting labels data", ioe);
614     }
615   }
616 
617   private void addSystemLabel(HRegion region, Map<String, Integer> labels,
618       Map<String, List<Integer>> userAuths) throws IOException {
619     if (!labels.containsKey(SYSTEM_LABEL)) {
620       Put p = new Put(Bytes.toBytes(SYSTEM_LABEL_ORDINAL));
621       p.addImmutable(LABELS_TABLE_FAMILY, LABEL_QUALIFIER, Bytes.toBytes(SYSTEM_LABEL));
622       // Set auth for "system" label for all super users.
623       List<String> superUsers = getSystemAndSuperUsers();
624       for (String superUser : superUsers) {
625         p.addImmutable(
626             LABELS_TABLE_FAMILY, Bytes.toBytes(superUser), DUMMY_VALUE, LABELS_TABLE_TAGS);
627       }
628       region.put(p);
629       labels.put(SYSTEM_LABEL, SYSTEM_LABEL_ORDINAL);
630       for (String superUser : superUsers) {
631         List<Integer> auths = userAuths.get(superUser);
632         if (auths == null) {
633           auths = new ArrayList<Integer>(1);
634           userAuths.put(superUser, auths);
635         }
636         auths.add(SYSTEM_LABEL_ORDINAL);
637       }
638     }
639   }
640 
641   @Override
642   public void preBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
643       MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
644     if (c.getEnvironment().getRegion().getRegionInfo().getTable().isSystemTable()) {
645       return;
646     }
647     // TODO this can be made as a global LRU cache at HRS level?
648     Map<String, List<Tag>> labelCache = new HashMap<String, List<Tag>>();
649     for (int i = 0; i < miniBatchOp.size(); i++) {
650       Mutation m = miniBatchOp.getOperation(i);
651       CellVisibility cellVisibility = null;
652       try {
653         cellVisibility = m.getCellVisibility();
654       } catch (DeserializationException de) {
655         miniBatchOp.setOperationStatus(i,
656             new OperationStatus(SANITY_CHECK_FAILURE, de.getMessage()));
657         continue;
658       }
659       if (m instanceof Put) {
660         Put p = (Put) m;
661         boolean sanityFailure = false;
662         for (CellScanner cellScanner = p.cellScanner(); cellScanner.advance();) {
663           if (!checkForReservedVisibilityTagPresence(cellScanner.current())) {
664             miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE,
665                 "Mutation contains cell with reserved type tag"));
666             sanityFailure = true;
667             break;
668           }
669         }
670         if (!sanityFailure) {
671           if (cellVisibility != null) {
672             String labelsExp = cellVisibility.getExpression();
673             List<Tag> visibilityTags = labelCache.get(labelsExp);
674             if (visibilityTags == null) {
675               try {
676                 visibilityTags = createVisibilityTags(labelsExp);
677               } catch (ParseException e) {
678                 miniBatchOp.setOperationStatus(i,
679                     new OperationStatus(SANITY_CHECK_FAILURE, e.getMessage()));
680               } catch (InvalidLabelException e) {
681                 miniBatchOp.setOperationStatus(i,
682                     new OperationStatus(SANITY_CHECK_FAILURE, e.getMessage()));
683               }
684             }
685             if (visibilityTags != null) {
686               labelCache.put(labelsExp, visibilityTags);
687               List<Cell> updatedCells = new ArrayList<Cell>();
688               for (CellScanner cellScanner = p.cellScanner(); cellScanner.advance();) {
689                 Cell cell = cellScanner.current();
690                 List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
691                     cell.getTagsLength());
692                 tags.addAll(visibilityTags);
693                 Cell updatedCell = new KeyValue(cell.getRowArray(), cell.getRowOffset(),
694                     cell.getRowLength(), cell.getFamilyArray(), cell.getFamilyOffset(),
695                     cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(),
696                     cell.getQualifierLength(), cell.getTimestamp(), Type.codeToType(cell
697                         .getTypeByte()), cell.getValueArray(), cell.getValueOffset(),
698                     cell.getValueLength(), tags);
699                 updatedCells.add(updatedCell);
700               }
701               p.getFamilyCellMap().clear();
702               // Clear and add new Cells to the Mutation.
703               for (Cell cell : updatedCells) {
704                 p.add(cell);
705               }
706             }
707           }
708         }
709       } else if (cellVisibility != null) {
710         // CellVisibility in a Delete is not legal! Fail the operation
711         miniBatchOp.setOperationStatus(i, new OperationStatus(SANITY_CHECK_FAILURE,
712             "CellVisibility cannot be set on Delete mutation"));
713       }
714     }
715   }
716 
717   @Override
718   public void postBatchMutate(ObserverContext<RegionCoprocessorEnvironment> c,
719       MiniBatchOperationInProgress<Mutation> miniBatchOp) throws IOException {
720     if (this.labelsRegion) {
721       // We will add to zookeeper here.
722       Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths =
723           extractLabelsAndAuths(getExistingLabelsWithAuths());
724       Map<String, Integer> existingLabels = labelsAndUserAuths.getFirst();
725       Map<String, List<Integer>> userAuths = labelsAndUserAuths.getSecond();
726       boolean isNewLabels = false;
727       boolean isUserAuthsChange = false;
728       for (int i = 0; i < miniBatchOp.size(); i++) {
729         Mutation m = miniBatchOp.getOperation(i);
730         if (miniBatchOp.getOperationStatus(i).getOperationStatusCode() == SUCCESS) {
731           for (List<Cell> cells : m.getFamilyCellMap().values()) {
732             for (Cell cell : cells) {
733               int labelOrdinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset());
734               if (Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
735                       cell.getQualifierLength(), LABEL_QUALIFIER, 0,
736                       LABEL_QUALIFIER.length)) {
737                 if (m instanceof Put) {
738                   existingLabels.put(
739                       Bytes.toString(cell.getValueArray(), cell.getValueOffset(),
740                           cell.getValueLength()), labelOrdinal);
741                   isNewLabels = true;
742                 }
743               } else {
744                 String user = Bytes.toString(cell.getQualifierArray(),
745                     cell.getQualifierOffset(), cell.getQualifierLength());
746                 List<Integer> auths = userAuths.get(user);
747                 if (auths == null) {
748                   auths = new ArrayList<Integer>();
749                   userAuths.put(user, auths);
750                 }
751                 if (m instanceof Delete) {
752                   auths.remove(Integer.valueOf(labelOrdinal));
753                 } else {
754                   auths.add(labelOrdinal);
755                 }
756                 isUserAuthsChange = true;
757               }
758             }
759           }
760         }
761       }
762       if (isNewLabels) {
763         byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(existingLabels);
764         this.visibilityManager.writeToZookeeper(serialized, true);
765       }
766       if (isUserAuthsChange) {
767         byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths);
768         this.visibilityManager.writeToZookeeper(serialized, false);
769       }
770     }
771   }
772 
773   private Pair<Map<String, Integer>, Map<String, List<Integer>>> extractLabelsAndAuths(
774       List<List<Cell>> labelDetails) {
775     Map<String, Integer> labels = new HashMap<String, Integer>();
776     Map<String, List<Integer>> userAuths = new HashMap<String, List<Integer>>();
777     for (List<Cell> cells : labelDetails) {
778       for (Cell cell : cells) {
779         if (Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
780             cell.getQualifierLength(), LABEL_QUALIFIER, 0, LABEL_QUALIFIER.length)) {
781           labels.put(
782               Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()),
783               Bytes.toInt(cell.getRowArray(), cell.getRowOffset()));
784         } else {
785           // These are user cells who has authorization for this label
786           String user = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(),
787               cell.getQualifierLength());
788           List<Integer> auths = userAuths.get(user);
789           if (auths == null) {
790             auths = new ArrayList<Integer>();
791             userAuths.put(user, auths);
792           }
793           auths.add(Bytes.toInt(cell.getRowArray(), cell.getRowOffset()));
794         }
795       }
796     }
797     return new Pair<Map<String, Integer>, Map<String, List<Integer>>>(labels, userAuths);
798   }
799 
800   // Checks whether cell contains any tag with type as VISIBILITY_TAG_TYPE.
801   // This tag type is reserved and should not be explicitly set by user.
802   private boolean checkForReservedVisibilityTagPresence(Cell cell) throws IOException {
803     if (cell.getTagsLength() > 0) {
804       Iterator<Tag> tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
805           cell.getTagsLength());
806       while (tagsItr.hasNext()) {
807         if (tagsItr.next().getType() == VisibilityUtils.VISIBILITY_TAG_TYPE) {
808           return false;
809         }
810       }
811     }
812     return true;
813   }
814 
815   private List<Tag> createVisibilityTags(String visibilityLabelsExp) throws IOException,
816       ParseException, InvalidLabelException {
817     ExpressionNode node = null;
818     node = this.expressionParser.parse(visibilityLabelsExp);
819     node = this.expressionExpander.expand(node);
820     List<Tag> tags = new ArrayList<Tag>();
821     ByteArrayOutputStream baos = new ByteArrayOutputStream();
822     DataOutputStream dos = new DataOutputStream(baos);
823     if (node.isSingleNode()) {
824       writeLabelOrdinalsToStream(node, dos);
825       tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
826       baos.reset();
827     } else {
828       NonLeafExpressionNode nlNode = (NonLeafExpressionNode) node;
829       if (nlNode.getOperator() == Operator.OR) {
830         for (ExpressionNode child : nlNode.getChildExps()) {
831           writeLabelOrdinalsToStream(child, dos);
832           tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
833           baos.reset();
834         }
835       } else {
836         writeLabelOrdinalsToStream(nlNode, dos);
837         tags.add(new Tag(VisibilityUtils.VISIBILITY_TAG_TYPE, baos.toByteArray()));
838         baos.reset();
839       }
840     }
841     return tags;
842   }
843 
844   private void writeLabelOrdinalsToStream(ExpressionNode node, DataOutputStream dos)
845       throws IOException, InvalidLabelException {
846     if (node.isSingleNode()) {
847       String identifier = null;
848       int labelOrdinal = 0;
849       if (node instanceof LeafExpressionNode) {
850         identifier = ((LeafExpressionNode) node)
851             .getIdentifier();
852         if (LOG.isTraceEnabled()) {
853           LOG.trace("The identifier is "+identifier);
854         }
855         labelOrdinal = this.visibilityManager.getLabelOrdinal(identifier);
856       } else {
857         // This is a NOT node.
858         LeafExpressionNode lNode = (LeafExpressionNode) ((NonLeafExpressionNode) node)
859             .getChildExps().get(0);
860         identifier = lNode.getIdentifier();
861         labelOrdinal = this.visibilityManager.getLabelOrdinal(identifier);
862         labelOrdinal = -1 * labelOrdinal; // Store NOT node as -ve ordinal.
863       }
864       if (labelOrdinal == 0) {
865         throw new InvalidLabelException("Invalid visibility label " + identifier);
866       }
867       StreamUtils.writeRawVInt32(dos, labelOrdinal);
868     } else {
869       List<ExpressionNode> childExps = ((NonLeafExpressionNode) node).getChildExps();
870       for (ExpressionNode child : childExps) {
871         writeLabelOrdinalsToStream(child, dos);
872       }
873     }
874   }
875   
876   @Override
877   public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
878       RegionScanner s) throws IOException {
879     HRegion region = e.getEnvironment().getRegion();
880     Authorizations authorizations = null;
881     // If a super user issues a scan, he should be able to scan the cells
882     // irrespective of the Visibility labels
883     if (checkIfScanOrGetFromSuperUser()) {
884       return s;
885     }
886     try {
887       authorizations = scan.getAuthorizations();
888     } catch (DeserializationException de) {
889       throw new IOException(de);
890     }
891     Filter visibilityLabelFilter = createVisibilityLabelFilter(region, authorizations);
892     if (visibilityLabelFilter != null) {
893       Filter filter = scan.getFilter();
894       if (filter != null) {
895         scan.setFilter(new FilterList(filter, visibilityLabelFilter));
896       } else {
897         scan.setFilter(visibilityLabelFilter);
898       }
899     }
900     return s;
901   }
902 
903   private boolean checkIfScanOrGetFromSuperUser() throws IOException {
904     User user = getActiveUser();
905     if (user != null && user.getShortName() != null) {
906       List<String> auths = this.visibilityManager.getAuths(user.getShortName());
907       return (auths.contains(SYSTEM_LABEL));
908     }
909     return false;
910   }
911 
912   @Override
913   public RegionScanner postScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c,
914       final Scan scan, final RegionScanner s) throws IOException {
915     User user = getActiveUser();
916     if (user != null && user.getShortName() != null) {
917       scannerOwners.put(s, user.getShortName());
918     }
919     return s;
920   }
921 
922   @Override
923   public boolean preScannerNext(final ObserverContext<RegionCoprocessorEnvironment> c,
924       final InternalScanner s, final List<Result> result, final int limit, final boolean hasNext)
925       throws IOException {
926     requireScannerOwner(s);
927     return hasNext;
928   }
929 
930   @Override
931   public void preScannerClose(final ObserverContext<RegionCoprocessorEnvironment> c,
932       final InternalScanner s) throws IOException {
933     requireScannerOwner(s);
934   }
935 
936   @Override
937   public void postScannerClose(final ObserverContext<RegionCoprocessorEnvironment> c,
938       final InternalScanner s) throws IOException {
939     // clean up any associated owner mapping
940     scannerOwners.remove(s);
941   }
942 
943   /**
944    * Verify, when servicing an RPC, that the caller is the scanner owner. If so, we assume that
945    * access control is correctly enforced based on the checks performed in preScannerOpen()
946    */
947   private void requireScannerOwner(InternalScanner s) throws AccessDeniedException {
948     if (RequestContext.isInRequestContext()) {
949       String requestUName = RequestContext.getRequestUserName();
950       String owner = scannerOwners.get(s);
951       if (owner != null && !owner.equals(requestUName)) {
952         throw new AccessDeniedException("User '" + requestUName + "' is not the scanner owner!");
953       }
954     }
955   }
956 
957   @Override
958   public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get, List<Cell> results)
959       throws IOException {
960     Authorizations authorizations = null;
961     // If a super user issues a get, he should be able to scan the cells
962     // irrespective of the Visibility labels
963     if (checkIfScanOrGetFromSuperUser()) {
964       return;
965     }
966     try {
967       authorizations = get.getAuthorizations();
968     } catch (DeserializationException de) {
969       throw new IOException(de);
970     }
971     Filter visibilityLabelFilter = createVisibilityLabelFilter(e.getEnvironment().getRegion(),
972         authorizations);
973     if (visibilityLabelFilter != null) {
974       Filter filter = get.getFilter();
975       if (filter != null) {
976         get.setFilter(new FilterList(filter, visibilityLabelFilter));
977       } else {
978         get.setFilter(visibilityLabelFilter);
979       }
980     }
981   }
982 
983   private Filter createVisibilityLabelFilter(HRegion region, Authorizations authorizations)
984       throws IOException {
985     Map<ByteRange, Integer> cfVsMaxVersions = new HashMap<ByteRange, Integer>();
986     for (HColumnDescriptor hcd : region.getTableDesc().getFamilies()) {
987       cfVsMaxVersions.put(new SimpleByteRange(hcd.getName()), hcd.getMaxVersions());
988     }
989     if (authorizations == null) {
990       // No Authorizations present for this scan/Get!
991       // In case of system tables other than "labels" just scan with out visibility check and
992       // filtering. Checking visibility labels for META and NAMESPACE table is not needed.
993       TableName table = region.getRegionInfo().getTable();
994       if (table.isSystemTable() && !table.equals(LABELS_TABLE_NAME)) {
995         return null;
996       }
997     } else {
998       for (String label : authorizations.getLabels()) {
999         if (!VisibilityLabelsValidator.isValidLabel(label)) {
1000           throw new IllegalArgumentException("Invalid authorization label : " + label
1001               + ". Authorizations cannot contain '(', ')' ,'&' ,'|', '!'" + " and cannot be empty");
1002         }
1003       }
1004     }
1005     Filter visibilityLabelFilter = null;
1006     if (this.scanLabelGenerators != null) {
1007       List<String> labels = null;
1008       for (ScanLabelGenerator scanLabelGenerator : this.scanLabelGenerators) {
1009         try {
1010           // null authorizations to be handled inside SLG impl.
1011           labels = scanLabelGenerator.getLabels(getActiveUser(), authorizations);
1012           labels = (labels == null) ? new ArrayList<String>() : labels;
1013           authorizations = new Authorizations(labels);
1014         } catch (Throwable t) {
1015           LOG.error(t);
1016           throw new IOException(t);
1017         }
1018       }
1019       int labelsCount = this.visibilityManager.getLabelsCount();
1020       BitSet bs = new BitSet(labelsCount + 1); // ordinal is index 1 based
1021       if (labels != null) {
1022         for (String label : labels) {
1023           int labelOrdinal = this.visibilityManager.getLabelOrdinal(label);
1024           if (labelOrdinal != 0) {
1025             bs.set(labelOrdinal);
1026           }
1027         }
1028       }
1029       visibilityLabelFilter = new VisibilityLabelFilter(bs, cfVsMaxVersions);
1030     }
1031     return visibilityLabelFilter;
1032   }
1033 
1034   private User getActiveUser() throws IOException {
1035     User user = RequestContext.getRequestUser();
1036     if (!RequestContext.isInRequestContext()) {
1037       // for non-rpc handling, fallback to system user
1038       user = User.getCurrent();
1039     }
1040     if (LOG.isTraceEnabled()) {
1041       LOG.trace("Current active user name is "+user.getShortName());
1042     }
1043     return user;
1044   }
1045 
1046   private List<String> getSystemAndSuperUsers() throws IOException {
1047     User user = User.getCurrent();
1048     if (user == null) {
1049       throw new IOException("Unable to obtain the current user, "
1050           + "authorization checks for internal operations will not work correctly!");
1051     }
1052     if (LOG.isTraceEnabled()) {
1053       LOG.trace("Current user name is "+user.getShortName());
1054     }
1055     String currentUser = user.getShortName();
1056     List<String> superUsers = Lists.asList(currentUser,
1057         this.conf.getStrings(AccessControlLists.SUPERUSER_CONF_KEY, new String[0]));
1058     return superUsers;
1059   }
1060 
1061   private boolean isSystemOrSuperUser() throws IOException {
1062     List<String> superUsers = getSystemAndSuperUsers();
1063     User activeUser = getActiveUser();
1064     return superUsers.contains(activeUser.getShortName());
1065   }
1066 
1067   @Override
1068   public Cell postMutationBeforeWAL(ObserverContext<RegionCoprocessorEnvironment> ctx,
1069       MutationType opType, Mutation mutation, Cell oldCell, Cell newCell) throws IOException {
1070     List<Tag> tags = Lists.newArrayList();
1071     CellVisibility cellVisibility = null;
1072     try {
1073       cellVisibility = mutation.getCellVisibility();
1074     } catch (DeserializationException e) {
1075       throw new IOException(e);
1076     }
1077     if (cellVisibility == null) {
1078       return newCell;
1079     }
1080     // Adding all other tags
1081     Iterator<Tag> tagsItr = CellUtil.tagsIterator(newCell.getTagsArray(), newCell.getTagsOffset(),
1082         newCell.getTagsLength());
1083     while (tagsItr.hasNext()) {
1084       Tag tag = tagsItr.next();
1085       if (tag.getType() != VisibilityUtils.VISIBILITY_TAG_TYPE) {
1086         tags.add(tag);
1087       }
1088     }
1089     try {
1090       tags.addAll(createVisibilityTags(cellVisibility.getExpression()));
1091     } catch (ParseException e) {
1092       throw new IOException(e);
1093     }
1094 
1095     // We need to create another KV, unfortunately, because the current new KV
1096     // has no space for tags
1097     KeyValue newKv = KeyValueUtil.ensureKeyValue(newCell);
1098     byte[] bytes = newKv.getBuffer();
1099     KeyValue rewriteKv = new KeyValue(bytes, newKv.getRowOffset(), newKv.getRowLength(), bytes,
1100         newKv.getFamilyOffset(), newKv.getFamilyLength(), bytes, newKv.getQualifierOffset(),
1101         newKv.getQualifierLength(), newKv.getTimestamp(), KeyValue.Type.codeToType(newKv
1102             .getTypeByte()), bytes, newKv.getValueOffset(), newKv.getValueLength(), tags);
1103     // Preserve mvcc data
1104     rewriteKv.setMvccVersion(newKv.getMvccVersion());
1105     return rewriteKv;
1106   }
1107 
1108   @Override
1109   public Service getService() {
1110     return VisibilityLabelsProtos.VisibilityLabelsService.newReflectiveService(this);
1111   }
1112 
1113   /****************************** VisibilityEndpoint service related methods ******************************/
1114   @Override
1115   public synchronized void addLabels(RpcController controller, VisibilityLabelsRequest request,
1116       RpcCallback<VisibilityLabelsResponse> done) {
1117     VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder();
1118     List<VisibilityLabel> labels = request.getVisLabelList();
1119     if (!initialized) {
1120       setExceptionResults(labels.size(), new CoprocessorException(
1121           "VisibilityController not yet initialized"), response);
1122     }
1123     try {
1124       checkCallingUserAuth();
1125       List<Mutation> puts = new ArrayList<Mutation>(labels.size());
1126       RegionActionResult successResult = RegionActionResult.newBuilder().build();
1127       for (VisibilityLabel visLabel : labels) {
1128         byte[] label = visLabel.getLabel().toByteArray();
1129         String labelStr = Bytes.toString(label);
1130         if (VisibilityLabelsValidator.isValidLabel(label)) {
1131           if (this.visibilityManager.getLabelOrdinal(labelStr) > 0) {
1132             RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
1133             failureResultBuilder.setException(ResponseConverter
1134                 .buildException(new LabelAlreadyExistsException("Label '" + labelStr
1135                     + "' already exists")));
1136             response.addResult(failureResultBuilder.build());
1137           } else {
1138             Put p = new Put(Bytes.toBytes(ordinalCounter));
1139             p.addImmutable(
1140                 LABELS_TABLE_FAMILY, LABEL_QUALIFIER, label, LABELS_TABLE_TAGS);
1141             if (LOG.isDebugEnabled()) {
1142               LOG.debug("Adding the label "+labelStr);
1143             }
1144             puts.add(p);
1145             ordinalCounter++;
1146             response.addResult(successResult);
1147           }
1148         } else {
1149           RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
1150           failureResultBuilder.setException(ResponseConverter
1151               .buildException(new InvalidLabelException("Invalid visibility label '" + labelStr
1152                   + "'")));
1153           response.addResult(failureResultBuilder.build());
1154         }
1155       }
1156       OperationStatus[] opStatus = this.regionEnv.getRegion().batchMutate(
1157           puts.toArray(new Mutation[puts.size()]));
1158       int i = 0;
1159       for (OperationStatus status : opStatus) {
1160         if (status.getOperationStatusCode() != SUCCESS) {
1161           while (response.getResult(i) != successResult)
1162             i++;
1163           RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
1164           failureResultBuilder.setException(ResponseConverter
1165               .buildException(new DoNotRetryIOException(status.getExceptionMsg())));
1166           response.setResult(i, failureResultBuilder.build());
1167         }
1168         i++;
1169       }
1170     } catch (IOException e) {
1171       LOG.error(e);
1172       setExceptionResults(labels.size(), e, response);
1173     }
1174     done.run(response.build());
1175   }
1176 
1177   private void setExceptionResults(int size, IOException e,
1178       VisibilityLabelsResponse.Builder response) {
1179     RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
1180     failureResultBuilder.setException(ResponseConverter.buildException(e));
1181     RegionActionResult failureResult = failureResultBuilder.build();
1182     for (int i = 0; i < size; i++) {
1183       response.addResult(i, failureResult);
1184     }
1185   }
1186 
1187   private void performACLCheck() throws IOException {
1188     // Do ACL check only when the security is enabled.
1189     if (this.acOn && !isSystemOrSuperUser()) {
1190       User user = getActiveUser();
1191       throw new AccessDeniedException("User '" + (user != null ? user.getShortName() : "null")
1192           + " is not authorized to perform this action.");
1193     }
1194   }
1195 
1196   private List<List<Cell>> getExistingLabelsWithAuths() throws IOException {
1197     Scan scan = new Scan();
1198     RegionScanner scanner = this.regionEnv.getRegion().getScanner(scan);
1199     List<List<Cell>> existingLabels = new ArrayList<List<Cell>>();
1200     try {
1201       while (true) {
1202         List<Cell> cells = new ArrayList<Cell>();
1203         scanner.next(cells);
1204         if (cells.isEmpty()) {
1205           break;
1206         }
1207         existingLabels.add(cells);
1208       }
1209     } finally {
1210       scanner.close();
1211     }
1212     return existingLabels;
1213   }
1214 
1215   @Override
1216   public synchronized void setAuths(RpcController controller, SetAuthsRequest request,
1217       RpcCallback<VisibilityLabelsResponse> done) {
1218     VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder();
1219     List<ByteString> auths = request.getAuthList();
1220     if (!initialized) {
1221       setExceptionResults(auths.size(), new CoprocessorException(
1222           "VisibilityController not yet initialized"), response);
1223     }
1224     byte[] user = request.getUser().toByteArray();
1225     try {
1226       checkCallingUserAuth();
1227       List<Mutation> puts = new ArrayList<Mutation>(auths.size());
1228       RegionActionResult successResult = RegionActionResult.newBuilder().build();
1229       for (ByteString authBS : auths) {
1230         byte[] auth = authBS.toByteArray();
1231         String authStr = Bytes.toString(auth);
1232         int labelOrdinal = this.visibilityManager.getLabelOrdinal(authStr);
1233         if (labelOrdinal == 0) {
1234           // This label is not yet added. 1st this should be added to the system
1235           RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
1236           failureResultBuilder.setException(ResponseConverter
1237               .buildException(new InvalidLabelException("Label '" + authStr + "' doesn't exist")));
1238           response.addResult(failureResultBuilder.build());
1239         } else {
1240           Put p = new Put(Bytes.toBytes(labelOrdinal));
1241           p.addImmutable(
1242               LABELS_TABLE_FAMILY, user, DUMMY_VALUE, LABELS_TABLE_TAGS);
1243           puts.add(p);
1244           response.addResult(successResult);
1245         }
1246       }
1247       OperationStatus[] opStatus = this.regionEnv.getRegion().batchMutate(
1248           puts.toArray(new Mutation[puts.size()]));
1249       int i = 0;
1250       for (OperationStatus status : opStatus) {
1251         if (status.getOperationStatusCode() != SUCCESS) {
1252           while (response.getResult(i) != successResult) i++;
1253           RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
1254           failureResultBuilder.setException(ResponseConverter
1255               .buildException(new DoNotRetryIOException(status.getExceptionMsg())));
1256           response.setResult(i, failureResultBuilder.build());
1257         }
1258         i++;
1259       }
1260     } catch (IOException e) {
1261       LOG.error(e);
1262       setExceptionResults(auths.size(), e, response);
1263     }
1264     done.run(response.build());
1265   }
1266 
1267   @Override
1268   public synchronized void getAuths(RpcController controller, GetAuthsRequest request,
1269       RpcCallback<GetAuthsResponse> done) {
1270     byte[] user = request.getUser().toByteArray();
1271     GetAuthsResponse.Builder response = GetAuthsResponse.newBuilder();
1272     response.setUser(request.getUser());
1273     try {
1274       List<String> labels = getUserAuthsFromLabelsTable(user);
1275       for (String label : labels) {
1276         response.addAuth(HBaseZeroCopyByteString.wrap(Bytes.toBytes(label)));
1277       }
1278     } catch (IOException e) {
1279       ResponseConverter.setControllerException(controller, e);
1280     }
1281     done.run(response.build());
1282   }
1283 
1284   private List<String> getUserAuthsFromLabelsTable(byte[] user) throws IOException {
1285     Scan s = new Scan();
1286     s.addColumn(LABELS_TABLE_FAMILY, user);
1287     Filter filter = createVisibilityLabelFilter(this.regionEnv.getRegion(), new Authorizations(
1288         SYSTEM_LABEL));
1289     s.setFilter(filter);
1290     List<String> auths = new ArrayList<String>();
1291     // We do ACL check here as we create scanner directly on region. It will not make calls to
1292     // AccessController CP methods.
1293     performACLCheck();
1294     RegionScanner scanner = this.regionEnv.getRegion().getScanner(s);
1295     List<Cell> results = new ArrayList<Cell>(1);
1296     while (true) {
1297       scanner.next(results);
1298       if (results.isEmpty()) break;
1299       Cell cell = results.get(0);
1300       int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
1301       String label = this.visibilityManager.getLabel(ordinal);
1302       if (label != null) {
1303         auths.add(label);
1304       }
1305       results.clear();
1306     }
1307     return auths;
1308   }
1309 
1310   @Override
1311   public synchronized void clearAuths(RpcController controller, SetAuthsRequest request,
1312       RpcCallback<VisibilityLabelsResponse> done) {
1313     VisibilityLabelsResponse.Builder response = VisibilityLabelsResponse.newBuilder();
1314     List<ByteString> auths = request.getAuthList();
1315     if (!initialized) {
1316       setExceptionResults(auths.size(), new CoprocessorException(
1317           "VisibilityController not yet initialized"), response);
1318     }
1319     byte[] user = request.getUser().toByteArray();
1320     try {
1321       checkCallingUserAuth();
1322       List<String> currentAuths = this.getUserAuthsFromLabelsTable(user);
1323       List<Mutation> deletes = new ArrayList<Mutation>(auths.size());
1324       RegionActionResult successResult = RegionActionResult.newBuilder().build();
1325       for (ByteString authBS : auths) {
1326         byte[] auth = authBS.toByteArray();
1327         String authStr = Bytes.toString(auth);
1328         if (currentAuths.contains(authStr)) {
1329           int labelOrdinal = this.visibilityManager.getLabelOrdinal(authStr);
1330           assert labelOrdinal > 0;
1331           Delete d = new Delete(Bytes.toBytes(labelOrdinal));
1332           d.deleteColumns(LABELS_TABLE_FAMILY, user);
1333           deletes.add(d);
1334           response.addResult(successResult);
1335         } else {
1336           // This label is not set for the user.
1337           RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
1338           failureResultBuilder.setException(ResponseConverter
1339               .buildException(new InvalidLabelException("Label '" + authStr
1340                   + "' is not set for the user " + Bytes.toString(user))));
1341           response.addResult(failureResultBuilder.build());
1342         }
1343       }
1344       OperationStatus[] opStatus = this.regionEnv.getRegion().batchMutate(
1345           deletes.toArray(new Mutation[deletes.size()]));
1346       int i = 0;
1347       for (OperationStatus status : opStatus) {
1348         if (status.getOperationStatusCode() != SUCCESS) {
1349           while (response.getResult(i) != successResult) i++;
1350           RegionActionResult.Builder failureResultBuilder = RegionActionResult.newBuilder();
1351           failureResultBuilder.setException(ResponseConverter
1352               .buildException(new DoNotRetryIOException(status.getExceptionMsg())));
1353           response.setResult(i, failureResultBuilder.build());
1354         }
1355         i++;
1356       }
1357     } catch (IOException e) {
1358       LOG.error(e);
1359       setExceptionResults(auths.size(), e, response);
1360     }
1361     done.run(response.build());
1362   }
1363 
1364   private void checkCallingUserAuth() throws IOException {
1365     if (!this.acOn) {
1366       User user = getActiveUser();
1367       if (user == null) {
1368         throw new IOException("Unable to retrieve calling user");
1369       }
1370       List<String> auths = this.visibilityManager.getAuths(user.getShortName());
1371       if (LOG.isTraceEnabled()) {
1372         LOG.trace("The list of auths are "+auths);
1373       }
1374       if (!auths.contains(SYSTEM_LABEL)) {
1375         throw new AccessDeniedException("User '" + user.getShortName()
1376             + "' is not authorized to perform this action.");
1377       }
1378     }
1379   }
1380 }