View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.security.visibility;
19  
20  import static org.apache.hadoop.hbase.TagType.VISIBILITY_TAG_TYPE;
21  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
22  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
23  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
24  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT;
25  import static org.apache.hadoop.hbase.security.visibility.VisibilityUtils.SYSTEM_LABEL;
26  
27  import java.io.ByteArrayOutputStream;
28  import java.io.DataOutputStream;
29  import java.io.IOException;
30  import java.util.ArrayList;
31  import java.util.BitSet;
32  import java.util.Collections;
33  import java.util.HashMap;
34  import java.util.HashSet;
35  import java.util.Iterator;
36  import java.util.List;
37  import java.util.Map;
38  import java.util.Set;
39  import java.util.regex.Pattern;
40  
41  import org.apache.commons.logging.Log;
42  import org.apache.commons.logging.LogFactory;
43  import org.apache.hadoop.conf.Configuration;
44  import org.apache.hadoop.hbase.AuthUtil;
45  import org.apache.hadoop.hbase.Cell;
46  import org.apache.hadoop.hbase.CellUtil;
47  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
48  import org.apache.hadoop.hbase.Tag;
49  import org.apache.hadoop.hbase.TagType;
50  import org.apache.hadoop.hbase.classification.InterfaceAudience;
51  import org.apache.hadoop.hbase.client.Delete;
52  import org.apache.hadoop.hbase.client.Mutation;
53  import org.apache.hadoop.hbase.client.Put;
54  import org.apache.hadoop.hbase.client.Scan;
55  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
56  import org.apache.hadoop.hbase.filter.Filter;
57  import org.apache.hadoop.hbase.io.util.StreamUtils;
58  import org.apache.hadoop.hbase.regionserver.HRegion;
59  import org.apache.hadoop.hbase.regionserver.OperationStatus;
60  import org.apache.hadoop.hbase.regionserver.RegionScanner;
61  import org.apache.hadoop.hbase.security.Superusers;
62  import org.apache.hadoop.hbase.security.User;
63  import org.apache.hadoop.hbase.util.Bytes;
64  import org.apache.hadoop.hbase.util.Pair;
65  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
66  
67  @InterfaceAudience.Private
68  public class DefaultVisibilityLabelServiceImpl implements VisibilityLabelService {
69  
70    private static final Log LOG = LogFactory.getLog(DefaultVisibilityLabelServiceImpl.class);
71  
72    // "system" label is having an ordinal value 1.
73    private static final int SYSTEM_LABEL_ORDINAL = 1;
74    private static final Tag[] LABELS_TABLE_TAGS = new Tag[1];
75    private static final byte[] DUMMY_VALUE = new byte[0];
76  
77    private volatile int ordinalCounter = -1;
78    private Configuration conf;
79    private HRegion labelsRegion;
80    private VisibilityLabelsCache labelsCache;
81    private List<ScanLabelGenerator> scanLabelGenerators;
82  
83    static {
84      ByteArrayOutputStream baos = new ByteArrayOutputStream();
85      DataOutputStream dos = new DataOutputStream(baos);
86      try {
87        StreamUtils.writeRawVInt32(dos, SYSTEM_LABEL_ORDINAL);
88      } catch (IOException e) {
89        // We write to a byte array. No Exception can happen.
90      }
91      LABELS_TABLE_TAGS[0] = new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray());
92    }
93  
94    public DefaultVisibilityLabelServiceImpl() {
95  
96    }
97  
98    @Override
99    public void setConf(Configuration conf) {
100     this.conf = conf;
101   }
102 
103   @Override
104   public Configuration getConf() {
105     return this.conf;
106   }
107 
108   @Override
109   public void init(RegionCoprocessorEnvironment e) throws IOException {
110     ZooKeeperWatcher zk = e.getRegionServerServices().getZooKeeper();
111     try {
112       labelsCache = VisibilityLabelsCache.createAndGet(zk, this.conf);
113     } catch (IOException ioe) {
114       LOG.error("Error creating VisibilityLabelsCache", ioe);
115       throw ioe;
116     }
117     this.scanLabelGenerators = VisibilityUtils.getScanLabelGenerators(this.conf);
118     if (e.getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) {
119       this.labelsRegion = e.getRegion();
120       Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths =
121           extractLabelsAndAuths(getExistingLabelsWithAuths());
122       Map<String, Integer> labels = labelsAndUserAuths.getFirst();
123       Map<String, List<Integer>> userAuths = labelsAndUserAuths.getSecond();
124       // Add the "system" label if it is not added into the system yet
125       addSystemLabel(this.labelsRegion, labels, userAuths);
126       int ordinal = SYSTEM_LABEL_ORDINAL; // Ordinal 1 is reserved for "system" label.
127       for (Integer i : labels.values()) {
128         if (i > ordinal) {
129           ordinal = i;
130         }
131       }
132       this.ordinalCounter = ordinal + 1;
133       if (labels.size() > 0) {
134         // If there is no data need not write to zk
135         byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(labels);
136         this.labelsCache.writeToZookeeper(serialized, true);
137         this.labelsCache.refreshLabelsCache(serialized);
138       }
139       if (userAuths.size() > 0) {
140         byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths);
141         this.labelsCache.writeToZookeeper(serialized, false);
142         this.labelsCache.refreshUserAuthsCache(serialized);
143       }
144     }
145   }
146 
147   protected List<List<Cell>> getExistingLabelsWithAuths() throws IOException {
148     Scan scan = new Scan();
149     RegionScanner scanner = labelsRegion.getScanner(scan);
150     List<List<Cell>> existingLabels = new ArrayList<List<Cell>>();
151     try {
152       while (true) {
153         List<Cell> cells = new ArrayList<Cell>();
154         scanner.next(cells);
155         if (cells.isEmpty()) {
156           break;
157         }
158         existingLabels.add(cells);
159       }
160     } finally {
161       scanner.close();
162     }
163     return existingLabels;
164   }
165 
166   protected Pair<Map<String, Integer>, Map<String, List<Integer>>> extractLabelsAndAuths(
167       List<List<Cell>> labelDetails) {
168     Map<String, Integer> labels = new HashMap<String, Integer>();
169     Map<String, List<Integer>> userAuths = new HashMap<String, List<Integer>>();
170     for (List<Cell> cells : labelDetails) {
171       for (Cell cell : cells) {
172         if (Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
173             cell.getQualifierLength(), LABEL_QUALIFIER, 0, LABEL_QUALIFIER.length)) {
174           labels.put(
175               Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()),
176               Bytes.toInt(cell.getRowArray(), cell.getRowOffset()));
177         } else {
178           // These are user cells who has authorization for this label
179           String user = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(),
180               cell.getQualifierLength());
181           List<Integer> auths = userAuths.get(user);
182           if (auths == null) {
183             auths = new ArrayList<Integer>();
184             userAuths.put(user, auths);
185           }
186           auths.add(Bytes.toInt(cell.getRowArray(), cell.getRowOffset()));
187         }
188       }
189     }
190     return new Pair<Map<String, Integer>, Map<String, List<Integer>>>(labels, userAuths);
191   }
192 
193   protected void addSystemLabel(HRegion region, Map<String, Integer> labels,
194       Map<String, List<Integer>> userAuths) throws IOException {
195     if (!labels.containsKey(SYSTEM_LABEL)) {
196       Put p = new Put(Bytes.toBytes(SYSTEM_LABEL_ORDINAL));
197       p.addImmutable(LABELS_TABLE_FAMILY, LABEL_QUALIFIER, Bytes.toBytes(SYSTEM_LABEL));
198       region.put(p);
199       labels.put(SYSTEM_LABEL, SYSTEM_LABEL_ORDINAL);
200     }
201   }
202 
203   @Override
204   public OperationStatus[] addLabels(List<byte[]> labels) throws IOException {
205     assert labelsRegion != null;
206     OperationStatus[] finalOpStatus = new OperationStatus[labels.size()];
207     List<Mutation> puts = new ArrayList<Mutation>(labels.size());
208     int i = 0;
209     for (byte[] label : labels) {
210       String labelStr = Bytes.toString(label);
211       if (this.labelsCache.getLabelOrdinal(labelStr) > 0) {
212         finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE,
213             new LabelAlreadyExistsException("Label '" + labelStr + "' already exists"));
214       } else {
215         Put p = new Put(Bytes.toBytes(ordinalCounter));
216         p.addImmutable(LABELS_TABLE_FAMILY, LABEL_QUALIFIER, label, LABELS_TABLE_TAGS);
217         if (LOG.isDebugEnabled()) {
218           LOG.debug("Adding the label " + labelStr);
219         }
220         puts.add(p);
221         ordinalCounter++;
222       }
223       i++;
224     }
225     if (mutateLabelsRegion(puts, finalOpStatus)) {
226       updateZk(true);
227     }
228     return finalOpStatus;
229   }
230 
231   @Override
232   public OperationStatus[] setAuths(byte[] user, List<byte[]> authLabels) throws IOException {
233     assert labelsRegion != null;
234     OperationStatus[] finalOpStatus = new OperationStatus[authLabels.size()];
235     List<Mutation> puts = new ArrayList<Mutation>(authLabels.size());
236     int i = 0;
237     for (byte[] auth : authLabels) {
238       String authStr = Bytes.toString(auth);
239       int labelOrdinal = this.labelsCache.getLabelOrdinal(authStr);
240       if (labelOrdinal == 0) {
241         // This label is not yet added. 1st this should be added to the system
242         finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE,
243             new InvalidLabelException("Label '" + authStr + "' doesn't exists"));
244       } else {
245         Put p = new Put(Bytes.toBytes(labelOrdinal));
246         p.addImmutable(LABELS_TABLE_FAMILY, user, DUMMY_VALUE, LABELS_TABLE_TAGS);
247         puts.add(p);
248       }
249       i++;
250     }
251     if (mutateLabelsRegion(puts, finalOpStatus)) {
252       updateZk(false);
253     }
254     return finalOpStatus;
255   }
256 
257   @Override
258   public OperationStatus[] clearAuths(byte[] user, List<byte[]> authLabels) throws IOException {
259     assert labelsRegion != null;
260     OperationStatus[] finalOpStatus = new OperationStatus[authLabels.size()];
261     List<String> currentAuths;
262     if (AuthUtil.isGroupPrincipal(Bytes.toString(user))) {
263       String group = AuthUtil.getGroupName(Bytes.toString(user));
264       currentAuths = this.getGroupAuths(new String[]{group}, true);
265     }
266     else {
267       currentAuths = this.getUserAuths(user, true);
268     }
269     List<Mutation> deletes = new ArrayList<Mutation>(authLabels.size());
270     int i = 0;
271     for (byte[] authLabel : authLabels) {
272       String authLabelStr = Bytes.toString(authLabel);
273       if (currentAuths.contains(authLabelStr)) {
274         int labelOrdinal = this.labelsCache.getLabelOrdinal(authLabelStr);
275         assert labelOrdinal > 0;
276         Delete d = new Delete(Bytes.toBytes(labelOrdinal));
277         d.deleteColumns(LABELS_TABLE_FAMILY, user);
278         deletes.add(d);
279       } else {
280         // This label is not set for the user.
281         finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE,
282             new InvalidLabelException("Label '" + authLabelStr + "' is not set for the user "
283                 + Bytes.toString(user)));
284       }
285       i++;
286     }
287     if (mutateLabelsRegion(deletes, finalOpStatus)) {
288       updateZk(false);
289     }
290     return finalOpStatus;
291   }
292 
293   /**
294    * Adds the mutations to labels region and set the results to the finalOpStatus. finalOpStatus
295    * might have some entries in it where the OpStatus is FAILURE. We will leave those and set in
296    * others in the order.
297    * @param mutations
298    * @param finalOpStatus
299    * @return whether we need a ZK update or not.
300    */
301   private boolean mutateLabelsRegion(List<Mutation> mutations, OperationStatus[] finalOpStatus)
302       throws IOException {
303     OperationStatus[] opStatus = this.labelsRegion.batchMutate(mutations
304         .toArray(new Mutation[mutations.size()]));
305     int i = 0;
306     boolean updateZk = false;
307     for (OperationStatus status : opStatus) {
308       // Update the zk when atleast one of the mutation was added successfully.
309       updateZk = updateZk || (status.getOperationStatusCode() == OperationStatusCode.SUCCESS);
310       for (; i < finalOpStatus.length; i++) {
311         if (finalOpStatus[i] == null) {
312           finalOpStatus[i] = status;
313           break;
314         }
315       }
316     }
317     return updateZk;
318   }
319 
320   @Override
321   @Deprecated
322   public List<String> getAuths(byte[] user, boolean systemCall)
323       throws IOException {
324     return getUserAuths(user, systemCall);
325   }
326 
327   @Override
328   public List<String> getUserAuths(byte[] user, boolean systemCall)
329       throws IOException {
330     assert (labelsRegion != null || systemCall);
331     if (systemCall || labelsRegion == null) {
332       return this.labelsCache.getUserAuths(Bytes.toString(user));
333     }
334     Scan s = new Scan();
335     if (user != null && user.length > 0) {
336       s.addColumn(LABELS_TABLE_FAMILY, user);
337     }
338     Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion,
339         new Authorizations(SYSTEM_LABEL));
340     s.setFilter(filter);
341     ArrayList<String> auths = new ArrayList<String>();
342     RegionScanner scanner = this.labelsRegion.getScanner(s);
343     List<Cell> results = new ArrayList<Cell>(1);
344     while (true) {
345       scanner.next(results);
346       if (results.isEmpty()) break;
347       Cell cell = results.get(0);
348       int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
349       String label = this.labelsCache.getLabel(ordinal);
350       if (label != null) {
351         auths.add(label);
352       }
353       results.clear();
354     }
355     return auths;
356   }
357 
358   @Override
359   public List<String> getGroupAuths(String[] groups, boolean systemCall)
360       throws IOException {
361     assert (labelsRegion != null || systemCall);
362     if (systemCall || labelsRegion == null) {
363       return this.labelsCache.getGroupAuths(groups);
364     }
365     Scan s = new Scan();
366     if (groups != null && groups.length > 0) {
367       for (String group : groups) {
368         s.addColumn(LABELS_TABLE_FAMILY, Bytes.toBytes(AuthUtil.toGroupEntry(group)));
369       }
370     }
371     Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion,
372         new Authorizations(SYSTEM_LABEL));
373     s.setFilter(filter);
374     Set<String> auths = new HashSet<String>();
375     RegionScanner scanner = this.labelsRegion.getScanner(s);
376     try {
377       List<Cell> results = new ArrayList<Cell>(1);
378       while (true) {
379         scanner.next(results);
380         if (results.isEmpty()) break;
381         Cell cell = results.get(0);
382         int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
383         String label = this.labelsCache.getLabel(ordinal);
384         if (label != null) {
385           auths.add(label);
386         }
387         results.clear();
388       }
389     } finally {
390       scanner.close();
391     }
392     return new ArrayList<String>(auths);
393   }
394 
395   @Override
396   public List<String> listLabels(String regex) throws IOException {
397     assert (labelsRegion != null);
398     Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths =
399         extractLabelsAndAuths(getExistingLabelsWithAuths());
400     Map<String, Integer> labels = labelsAndUserAuths.getFirst();
401     labels.remove(SYSTEM_LABEL);
402     if (regex != null) {
403       Pattern pattern = Pattern.compile(regex);
404       ArrayList<String> matchedLabels = new ArrayList<String>();
405       for (String label : labels.keySet()) {
406         if (pattern.matcher(label).matches()) {
407           matchedLabels.add(label);
408         }
409       }
410       return matchedLabels;
411     }
412     return new ArrayList<String>(labels.keySet());
413   }
414 
415   @Override
416   public List<Tag> createVisibilityExpTags(String visExpression, boolean withSerializationFormat,
417       boolean checkAuths) throws IOException {
418     Set<Integer> auths = new HashSet<Integer>();
419     if (checkAuths) {
420       User user = VisibilityUtils.getActiveUser();
421       auths.addAll(this.labelsCache.getUserAuthsAsOrdinals(user.getShortName()));
422       auths.addAll(this.labelsCache.getGroupAuthsAsOrdinals(user.getGroupNames()));
423     }
424     return VisibilityUtils.createVisibilityExpTags(visExpression, withSerializationFormat,
425         checkAuths, auths, labelsCache);
426   }
427 
428   protected void updateZk(boolean labelAddition) throws IOException {
429     // We will add to zookeeper here.
430     // TODO we should add the delta only to zk. Else this will be a very heavy op and when there are
431     // so many labels and auth in the system, we will end up adding lots of data to zk. Most
432     // possibly we will exceed zk node data limit!
433     Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths =
434         extractLabelsAndAuths(getExistingLabelsWithAuths());
435     Map<String, Integer> existingLabels = labelsAndUserAuths.getFirst();
436     Map<String, List<Integer>> userAuths = labelsAndUserAuths.getSecond();
437     if (labelAddition) {
438       byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(existingLabels);
439       this.labelsCache.writeToZookeeper(serialized, true);
440     } else {
441       byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths);
442       this.labelsCache.writeToZookeeper(serialized, false);
443     }
444   }
445 
446   @Override
447   public VisibilityExpEvaluator getVisibilityExpEvaluator(Authorizations authorizations)
448       throws IOException {
449     // If a super user issues a get/scan, he should be able to scan the cells
450     // irrespective of the Visibility labels
451     if (isReadFromSystemAuthUser()) {
452       return new VisibilityExpEvaluator() {
453         @Override
454         public boolean evaluate(Cell cell) throws IOException {
455           return true;
456         }
457       };
458     }
459     List<String> authLabels = null;
460     for (ScanLabelGenerator scanLabelGenerator : scanLabelGenerators) {
461       try {
462         // null authorizations to be handled inside SLG impl.
463         authLabels = scanLabelGenerator.getLabels(VisibilityUtils.getActiveUser(), authorizations);
464         authLabels = (authLabels == null) ? new ArrayList<String>() : authLabels;
465         authorizations = new Authorizations(authLabels);
466       } catch (Throwable t) {
467         LOG.error(t);
468         throw new IOException(t);
469       }
470     }
471     int labelsCount = this.labelsCache.getLabelsCount();
472     final BitSet bs = new BitSet(labelsCount + 1); // ordinal is index 1 based
473     if (authLabels != null) {
474       for (String authLabel : authLabels) {
475         int labelOrdinal = this.labelsCache.getLabelOrdinal(authLabel);
476         if (labelOrdinal != 0) {
477           bs.set(labelOrdinal);
478         }
479       }
480     }
481 
482     return new VisibilityExpEvaluator() {
483       @Override
484       public boolean evaluate(Cell cell) throws IOException {
485         boolean visibilityTagPresent = false;
486         // Save an object allocation where we can
487         if (cell.getTagsLengthUnsigned() > 0) {
488           Iterator<Tag> tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
489               cell.getTagsLengthUnsigned());
490           while (tagsItr.hasNext()) {
491             boolean includeKV = true;
492             Tag tag = tagsItr.next();
493             if (tag.getType() == VISIBILITY_TAG_TYPE) {
494               visibilityTagPresent = true;
495               int offset = tag.getTagOffset();
496               int endOffset = offset + tag.getTagLength();
497               while (offset < endOffset) {
498                 Pair<Integer, Integer> result = StreamUtils
499                     .readRawVarint32(tag.getBuffer(), offset);
500                 int currLabelOrdinal = result.getFirst();
501                 if (currLabelOrdinal < 0) {
502                   // check for the absence of this label in the Scan Auth labels
503                   // ie. to check BitSet corresponding bit is 0
504                   int temp = -currLabelOrdinal;
505                   if (bs.get(temp)) {
506                     includeKV = false;
507                     break;
508                   }
509                 } else {
510                   if (!bs.get(currLabelOrdinal)) {
511                     includeKV = false;
512                     break;
513                   }
514                 }
515                 offset += result.getSecond();
516               }
517               if (includeKV) {
518                 // We got one visibility expression getting evaluated to true. Good to include this
519                 // KV in the result then.
520                 return true;
521               }
522             }
523           }
524         }
525         return !(visibilityTagPresent);
526       }
527     };
528   }
529 
530   protected boolean isReadFromSystemAuthUser() throws IOException {
531     User user = VisibilityUtils.getActiveUser();
532     return havingSystemAuth(user);
533   }
534 
535   @Override
536   @Deprecated
537   public boolean havingSystemAuth(byte[] user) throws IOException {
538     // Implementation for backward compatibility
539     if (Superusers.isSuperUser(Bytes.toString(user))) {
540       return true;
541     }
542     List<String> auths = this.getUserAuths(user, true);
543     if (LOG.isTraceEnabled()) {
544       LOG.trace("The auths for user " + Bytes.toString(user) + " are " + auths);
545     }
546     return auths.contains(SYSTEM_LABEL);
547   }
548 
549   @Override
550   public boolean havingSystemAuth(User user) throws IOException {
551     // A super user has 'system' auth.
552     if (Superusers.isSuperUser(user)) {
553       return true;
554     }
555     // A user can also be explicitly granted 'system' auth.
556     List<String> auths = this.getUserAuths(Bytes.toBytes(user.getShortName()), true);
557     if (LOG.isTraceEnabled()) {
558       LOG.trace("The auths for user " + user.getShortName() + " are " + auths);
559     }
560     if (auths.contains(SYSTEM_LABEL)) {
561       return true;
562     }
563     auths = this.getGroupAuths(user.getGroupNames(), true);
564     if (LOG.isTraceEnabled()) {
565       LOG.trace("The auths for groups of user " + user.getShortName() + " are " + auths);
566     }
567     return auths.contains(SYSTEM_LABEL);
568   }
569 
570   @Override
571   public boolean matchVisibility(List<Tag> putVisTags, Byte putTagsFormat, List<Tag> deleteVisTags,
572       Byte deleteTagsFormat) throws IOException {
573     if ((deleteTagsFormat != null && deleteTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT)
574         && (putTagsFormat == null || putTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT)) {
575       if (putVisTags.size() == 0) {
576         // Early out if there are no tags in the cell
577         return false;
578       }
579       if (putTagsFormat == null) {
580         return matchUnSortedVisibilityTags(putVisTags, deleteVisTags);
581       } else {
582         return matchOrdinalSortedVisibilityTags(putVisTags, deleteVisTags);
583       }
584     }
585     throw new IOException("Unexpected tag format passed for comparison, deleteTagsFormat : "
586         + deleteTagsFormat + ", putTagsFormat : " + putTagsFormat);
587   }
588 
589   /**
590    * @param putVisTags Visibility tags in Put Mutation
591    * @param deleteVisTags Visibility tags in Delete Mutation
592    * @return true when all the visibility tags in Put matches with visibility tags in Delete.
593    * This is used when, at least one set of tags are not sorted based on the label ordinal.
594    */
595   private static boolean matchUnSortedVisibilityTags(List<Tag> putVisTags,
596       List<Tag> deleteVisTags) throws IOException {
597     return compareTagsOrdinals(sortTagsBasedOnOrdinal(putVisTags),
598         sortTagsBasedOnOrdinal(deleteVisTags));
599   }
600 
601   /**
602    * @param putVisTags Visibility tags in Put Mutation
603    * @param deleteVisTags Visibility tags in Delete Mutation
604    * @return true when all the visibility tags in Put matches with visibility tags in Delete.
605    * This is used when both the set of tags are sorted based on the label ordinal.
606    */
607   private static boolean matchOrdinalSortedVisibilityTags(List<Tag> putVisTags,
608       List<Tag> deleteVisTags) {
609     boolean matchFound = false;
610     // If the size does not match. Definitely we are not comparing the equal tags.
611     if ((deleteVisTags.size()) == putVisTags.size()) {
612       for (Tag tag : deleteVisTags) {
613         matchFound = false;
614         for (Tag givenTag : putVisTags) {
615           if (Bytes.equals(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength(),
616               givenTag.getBuffer(), givenTag.getTagOffset(), givenTag.getTagLength())) {
617             matchFound = true;
618             break;
619           }
620         }
621         if (!matchFound) break;
622       }
623     }
624     return matchFound;
625   }
626 
627   private static List<List<Integer>> sortTagsBasedOnOrdinal(List<Tag> tags) throws IOException {
628     List<List<Integer>> fullTagsList = new ArrayList<List<Integer>>();
629     for (Tag tag : tags) {
630       if (tag.getType() == VISIBILITY_TAG_TYPE) {
631         getSortedTagOrdinals(fullTagsList, tag);
632       }
633     }
634     return fullTagsList;
635   }
636 
637   private static void getSortedTagOrdinals(List<List<Integer>> fullTagsList, Tag tag)
638       throws IOException {
639     List<Integer> tagsOrdinalInSortedOrder = new ArrayList<Integer>();
640     int offset = tag.getTagOffset();
641     int endOffset = offset + tag.getTagLength();
642     while (offset < endOffset) {
643       Pair<Integer, Integer> result = StreamUtils.readRawVarint32(tag.getBuffer(), offset);
644       tagsOrdinalInSortedOrder.add(result.getFirst());
645       offset += result.getSecond();
646     }
647     Collections.sort(tagsOrdinalInSortedOrder);
648     fullTagsList.add(tagsOrdinalInSortedOrder);
649   }
650 
651   /*
652    * @return true when all the visibility tags in Put matches with visibility tags in Delete.
653    */
654   private static boolean compareTagsOrdinals(List<List<Integer>> putVisTags,
655       List<List<Integer>> deleteVisTags) {
656     boolean matchFound = false;
657     if (deleteVisTags.size() == putVisTags.size()) {
658       for (List<Integer> deleteTagOrdinals : deleteVisTags) {
659         matchFound = false;
660         for (List<Integer> tagOrdinals : putVisTags) {
661           if (deleteTagOrdinals.equals(tagOrdinals)) {
662             matchFound = true;
663             break;
664           }
665         }
666         if (!matchFound) break;
667       }
668     }
669     return matchFound;
670   }
671 
672   @Override
673   public byte[] encodeVisibilityForReplication(final List<Tag> tags, final Byte serializationFormat)
674       throws IOException {
675     if (tags.size() > 0
676         && (serializationFormat == null ||
677         serializationFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT)) {
678       return createModifiedVisExpression(tags);
679     }
680     return null;
681   }
682 
683   /**
684    * @param tags
685    *          - all the visibility tags associated with the current Cell
686    * @return - the modified visibility expression as byte[]
687    */
688   private byte[] createModifiedVisExpression(final List<Tag> tags)
689       throws IOException {
690     StringBuilder visibilityString = new StringBuilder();
691     for (Tag tag : tags) {
692       if (tag.getType() == TagType.VISIBILITY_TAG_TYPE) {
693         if (visibilityString.length() != 0) {
694           visibilityString.append(VisibilityConstants.CLOSED_PARAN).append(
695               VisibilityConstants.OR_OPERATOR);
696         }
697         int offset = tag.getTagOffset();
698         int endOffset = offset + tag.getTagLength();
699         boolean expressionStart = true;
700         while (offset < endOffset) {
701           Pair<Integer, Integer> result = StreamUtils.readRawVarint32(tag.getBuffer(), offset);
702           int currLabelOrdinal = result.getFirst();
703           if (currLabelOrdinal < 0) {
704             int temp = -currLabelOrdinal;
705             String label = this.labelsCache.getLabel(temp);
706             if (expressionStart) {
707               // Quote every label in case of unicode characters if present
708               visibilityString.append(VisibilityConstants.OPEN_PARAN)
709                   .append(VisibilityConstants.NOT_OPERATOR).append(CellVisibility.quote(label));
710             } else {
711               visibilityString.append(VisibilityConstants.AND_OPERATOR)
712                   .append(VisibilityConstants.NOT_OPERATOR).append(CellVisibility.quote(label));
713             }
714           } else {
715             String label = this.labelsCache.getLabel(currLabelOrdinal);
716             if (expressionStart) {
717               visibilityString.append(VisibilityConstants.OPEN_PARAN).append(
718                   CellVisibility.quote(label));
719             } else {
720               visibilityString.append(VisibilityConstants.AND_OPERATOR).append(
721                   CellVisibility.quote(label));
722             }
723           }
724           expressionStart = false;
725           offset += result.getSecond();
726         }
727       }
728     }
729     if (visibilityString.length() != 0) {
730       visibilityString.append(VisibilityConstants.CLOSED_PARAN);
731       // Return the string formed as byte[]
732       return Bytes.toBytes(visibilityString.toString());
733     }
734     return null;
735   }
736 }