View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.security.visibility;
19  
20  import static org.apache.hadoop.hbase.TagType.VISIBILITY_TAG_TYPE;
21  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
22  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
23  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
24  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.SORTED_ORDINAL_SERIALIZATION_FORMAT;
25  import static org.apache.hadoop.hbase.security.visibility.VisibilityUtils.SYSTEM_LABEL;
26  
27  import java.io.ByteArrayOutputStream;
28  import java.io.DataOutputStream;
29  import java.io.IOException;
30  import java.util.ArrayList;
31  import java.util.BitSet;
32  import java.util.Collections;
33  import java.util.HashMap;
34  import java.util.Iterator;
35  import java.util.List;
36  import java.util.Map;
37  import java.util.Set;
38  import java.util.regex.Pattern;
39  
40  import org.apache.commons.logging.Log;
41  import org.apache.commons.logging.LogFactory;
42  import org.apache.hadoop.conf.Configuration;
43  import org.apache.hadoop.hbase.Cell;
44  import org.apache.hadoop.hbase.CellUtil;
45  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
46  import org.apache.hadoop.hbase.Tag;
47  import org.apache.hadoop.hbase.TagType;
48  import org.apache.hadoop.hbase.classification.InterfaceAudience;
49  import org.apache.hadoop.hbase.client.Delete;
50  import org.apache.hadoop.hbase.client.Mutation;
51  import org.apache.hadoop.hbase.client.Put;
52  import org.apache.hadoop.hbase.client.Scan;
53  import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
54  import org.apache.hadoop.hbase.filter.Filter;
55  import org.apache.hadoop.hbase.io.util.StreamUtils;
56  import org.apache.hadoop.hbase.regionserver.HRegion;
57  import org.apache.hadoop.hbase.regionserver.OperationStatus;
58  import org.apache.hadoop.hbase.regionserver.RegionScanner;
59  import org.apache.hadoop.hbase.security.User;
60  import org.apache.hadoop.hbase.security.access.AccessControlLists;
61  import org.apache.hadoop.hbase.util.Bytes;
62  import org.apache.hadoop.hbase.util.Pair;
63  import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
64  
65  import com.google.common.collect.Lists;
66  
67  @InterfaceAudience.Private
68  public class DefaultVisibilityLabelServiceImpl implements VisibilityLabelService {
69  
70    private static final Log LOG = LogFactory.getLog(DefaultVisibilityLabelServiceImpl.class);
71  
72    // "system" label is having an ordinal value 1.
73    private static final int SYSTEM_LABEL_ORDINAL = 1;
74    private static final Tag[] LABELS_TABLE_TAGS = new Tag[1];
75    private static final byte[] DUMMY_VALUE = new byte[0];
76  
77    private volatile int ordinalCounter = -1;
78    private Configuration conf;
79    private HRegion labelsRegion;
80    private VisibilityLabelsCache labelsCache;
81    private List<ScanLabelGenerator> scanLabelGenerators;
82  
83    static {
84      ByteArrayOutputStream baos = new ByteArrayOutputStream();
85      DataOutputStream dos = new DataOutputStream(baos);
86      try {
87        StreamUtils.writeRawVInt32(dos, SYSTEM_LABEL_ORDINAL);
88      } catch (IOException e) {
89        // We write to a byte array. No Exception can happen.
90      }
91      LABELS_TABLE_TAGS[0] = new Tag(VISIBILITY_TAG_TYPE, baos.toByteArray());
92    }
93  
94    public DefaultVisibilityLabelServiceImpl() {
95  
96    }
97  
98    @Override
99    public void setConf(Configuration conf) {
100     this.conf = conf;
101   }
102 
103   @Override
104   public Configuration getConf() {
105     return this.conf;
106   }
107 
108   @Override
109   public void init(RegionCoprocessorEnvironment e) throws IOException {
110     ZooKeeperWatcher zk = e.getRegionServerServices().getZooKeeper();
111     try {
112       labelsCache = VisibilityLabelsCache.createAndGet(zk, this.conf);
113     } catch (IOException ioe) {
114       LOG.error("Error creating VisibilityLabelsCache", ioe);
115       throw ioe;
116     }
117     this.scanLabelGenerators = VisibilityUtils.getScanLabelGenerators(this.conf);
118     if (e.getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) {
119       this.labelsRegion = e.getRegion();
120       Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths =
121           extractLabelsAndAuths(getExistingLabelsWithAuths());
122       Map<String, Integer> labels = labelsAndUserAuths.getFirst();
123       Map<String, List<Integer>> userAuths = labelsAndUserAuths.getSecond();
124       // Add the "system" label if it is not added into the system yet
125       addSystemLabel(this.labelsRegion, labels, userAuths);
126       int ordinal = SYSTEM_LABEL_ORDINAL; // Ordinal 1 is reserved for "system" label.
127       for (Integer i : labels.values()) {
128         if (i > ordinal) {
129           ordinal = i;
130         }
131       }
132       this.ordinalCounter = ordinal + 1;
133       if (labels.size() > 0) {
134         // If there is no data need not write to zk
135         byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(labels);
136         this.labelsCache.writeToZookeeper(serialized, true);
137         this.labelsCache.refreshLabelsCache(serialized);
138       }
139       if (userAuths.size() > 0) {
140         byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths);
141         this.labelsCache.writeToZookeeper(serialized, false);
142         this.labelsCache.refreshUserAuthsCache(serialized);
143       }
144     }
145   }
146 
147   protected List<List<Cell>> getExistingLabelsWithAuths() throws IOException {
148     Scan scan = new Scan();
149     RegionScanner scanner = labelsRegion.getScanner(scan);
150     List<List<Cell>> existingLabels = new ArrayList<List<Cell>>();
151     try {
152       while (true) {
153         List<Cell> cells = new ArrayList<Cell>();
154         scanner.next(cells);
155         if (cells.isEmpty()) {
156           break;
157         }
158         existingLabels.add(cells);
159       }
160     } finally {
161       scanner.close();
162     }
163     return existingLabels;
164   }
165 
166   protected Pair<Map<String, Integer>, Map<String, List<Integer>>> extractLabelsAndAuths(
167       List<List<Cell>> labelDetails) {
168     Map<String, Integer> labels = new HashMap<String, Integer>();
169     Map<String, List<Integer>> userAuths = new HashMap<String, List<Integer>>();
170     for (List<Cell> cells : labelDetails) {
171       for (Cell cell : cells) {
172         if (Bytes.equals(cell.getQualifierArray(), cell.getQualifierOffset(),
173             cell.getQualifierLength(), LABEL_QUALIFIER, 0, LABEL_QUALIFIER.length)) {
174           labels.put(
175               Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()),
176               Bytes.toInt(cell.getRowArray(), cell.getRowOffset()));
177         } else {
178           // These are user cells who has authorization for this label
179           String user = Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(),
180               cell.getQualifierLength());
181           List<Integer> auths = userAuths.get(user);
182           if (auths == null) {
183             auths = new ArrayList<Integer>();
184             userAuths.put(user, auths);
185           }
186           auths.add(Bytes.toInt(cell.getRowArray(), cell.getRowOffset()));
187         }
188       }
189     }
190     return new Pair<Map<String, Integer>, Map<String, List<Integer>>>(labels, userAuths);
191   }
192 
193   protected void addSystemLabel(HRegion region, Map<String, Integer> labels,
194       Map<String, List<Integer>> userAuths) throws IOException {
195     if (!labels.containsKey(SYSTEM_LABEL)) {
196       Put p = new Put(Bytes.toBytes(SYSTEM_LABEL_ORDINAL));
197       p.addImmutable(LABELS_TABLE_FAMILY, LABEL_QUALIFIER, Bytes.toBytes(SYSTEM_LABEL));
198       // Set auth for "system" label for all super users.
199       List<String> superUsers = getSystemAndSuperUsers();
200       for (String superUser : superUsers) {
201         p.addImmutable(LABELS_TABLE_FAMILY, Bytes.toBytes(superUser), DUMMY_VALUE,
202             LABELS_TABLE_TAGS);
203       }
204       region.put(p);
205       labels.put(SYSTEM_LABEL, SYSTEM_LABEL_ORDINAL);
206       for (String superUser : superUsers) {
207         List<Integer> auths = userAuths.get(superUser);
208         if (auths == null) {
209           auths = new ArrayList<Integer>(1);
210           userAuths.put(superUser, auths);
211         }
212         auths.add(SYSTEM_LABEL_ORDINAL);
213       }
214     }
215   }
216 
217   protected List<String> getSystemAndSuperUsers() throws IOException {
218     User user = User.getCurrent();
219     if (user == null) {
220       throw new IOException("Unable to obtain the current user, "
221           + "authorization checks for internal operations will not work correctly!");
222     }
223     if (LOG.isTraceEnabled()) {
224       LOG.trace("Current user name is " + user.getShortName());
225     }
226     String currentUser = user.getShortName();
227     List<String> superUsers = Lists.asList(currentUser,
228         this.conf.getStrings(AccessControlLists.SUPERUSER_CONF_KEY, new String[0]));
229     return superUsers;
230   }
231 
232   @Override
233   public OperationStatus[] addLabels(List<byte[]> labels) throws IOException {
234     assert labelsRegion != null;
235     OperationStatus[] finalOpStatus = new OperationStatus[labels.size()];
236     List<Mutation> puts = new ArrayList<Mutation>(labels.size());
237     int i = 0;
238     for (byte[] label : labels) {
239       String labelStr = Bytes.toString(label);
240       if (this.labelsCache.getLabelOrdinal(labelStr) > 0) {
241         finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE,
242             new LabelAlreadyExistsException("Label '" + labelStr + "' already exists"));
243       } else {
244         Put p = new Put(Bytes.toBytes(ordinalCounter));
245         p.addImmutable(LABELS_TABLE_FAMILY, LABEL_QUALIFIER, label, LABELS_TABLE_TAGS);
246         if (LOG.isDebugEnabled()) {
247           LOG.debug("Adding the label " + labelStr);
248         }
249         puts.add(p);
250         ordinalCounter++;
251       }
252       i++;
253     }
254     if (mutateLabelsRegion(puts, finalOpStatus)) {
255       updateZk(true);
256     }
257     return finalOpStatus;
258   }
259 
260   @Override
261   public OperationStatus[] setAuths(byte[] user, List<byte[]> authLabels) throws IOException {
262     assert labelsRegion != null;
263     OperationStatus[] finalOpStatus = new OperationStatus[authLabels.size()];
264     List<Mutation> puts = new ArrayList<Mutation>(authLabels.size());
265     int i = 0;
266     for (byte[] auth : authLabels) {
267       String authStr = Bytes.toString(auth);
268       int labelOrdinal = this.labelsCache.getLabelOrdinal(authStr);
269       if (labelOrdinal == 0) {
270         // This label is not yet added. 1st this should be added to the system
271         finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE,
272             new InvalidLabelException("Label '" + authStr + "' doesn't exists"));
273       } else {
274         Put p = new Put(Bytes.toBytes(labelOrdinal));
275         p.addImmutable(LABELS_TABLE_FAMILY, user, DUMMY_VALUE, LABELS_TABLE_TAGS);
276         puts.add(p);
277       }
278       i++;
279     }
280     if (mutateLabelsRegion(puts, finalOpStatus)) {
281       updateZk(false);
282     }
283     return finalOpStatus;
284   }
285 
286   @Override
287   public OperationStatus[] clearAuths(byte[] user, List<byte[]> authLabels) throws IOException {
288     assert labelsRegion != null;
289     OperationStatus[] finalOpStatus = new OperationStatus[authLabels.size()];
290     List<String> currentAuths = this.getAuths(user, true);
291     List<Mutation> deletes = new ArrayList<Mutation>(authLabels.size());
292     int i = 0;
293     for (byte[] authLabel : authLabels) {
294       String authLabelStr = Bytes.toString(authLabel);
295       if (currentAuths.contains(authLabelStr)) {
296         int labelOrdinal = this.labelsCache.getLabelOrdinal(authLabelStr);
297         assert labelOrdinal > 0;
298         Delete d = new Delete(Bytes.toBytes(labelOrdinal));
299         d.deleteColumns(LABELS_TABLE_FAMILY, user);
300         deletes.add(d);
301       } else {
302         // This label is not set for the user.
303         finalOpStatus[i] = new OperationStatus(OperationStatusCode.FAILURE,
304             new InvalidLabelException("Label '" + authLabelStr + "' is not set for the user "
305                 + Bytes.toString(user)));
306       }
307       i++;
308     }
309     if (mutateLabelsRegion(deletes, finalOpStatus)) {
310       updateZk(false);
311     }
312     return finalOpStatus;
313   }
314 
315   /**
316    * Adds the mutations to labels region and set the results to the finalOpStatus. finalOpStatus
317    * might have some entries in it where the OpStatus is FAILURE. We will leave those and set in
318    * others in the order.
319    * @param mutations
320    * @param finalOpStatus
321    * @return whether we need a ZK update or not.
322    */
323   private boolean mutateLabelsRegion(List<Mutation> mutations, OperationStatus[] finalOpStatus)
324       throws IOException {
325     OperationStatus[] opStatus = this.labelsRegion.batchMutate(mutations
326         .toArray(new Mutation[mutations.size()]));
327     int i = 0;
328     boolean updateZk = false;
329     for (OperationStatus status : opStatus) {
330       // Update the zk when atleast one of the mutation was added successfully.
331       updateZk = updateZk || (status.getOperationStatusCode() == OperationStatusCode.SUCCESS);
332       for (; i < finalOpStatus.length; i++) {
333         if (finalOpStatus[i] == null) {
334           finalOpStatus[i] = status;
335           break;
336         }
337       }
338     }
339     return updateZk;
340   }
341 
342   @Override
343   public List<String> getAuths(byte[] user, boolean systemCall) throws IOException {
344     assert (labelsRegion != null || systemCall);
345     if (systemCall || labelsRegion == null) {
346       return this.labelsCache.getAuths(Bytes.toString(user));
347     }
348     Scan s = new Scan();
349     s.addColumn(LABELS_TABLE_FAMILY, user);
350     Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion,
351         new Authorizations(SYSTEM_LABEL));
352     s.setFilter(filter);
353     List<String> auths = new ArrayList<String>();
354     RegionScanner scanner = this.labelsRegion.getScanner(s);
355     List<Cell> results = new ArrayList<Cell>(1);
356     while (true) {
357       scanner.next(results);
358       if (results.isEmpty()) break;
359       Cell cell = results.get(0);
360       int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
361       String label = this.labelsCache.getLabel(ordinal);
362       if (label != null) {
363         auths.add(label);
364       }
365       results.clear();
366     }
367     return auths;
368   }
369 
370   @Override
371   public List<String> listLabels(String regex) throws IOException {
372     assert (labelsRegion != null);
373     Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths =
374         extractLabelsAndAuths(getExistingLabelsWithAuths());
375     Map<String, Integer> labels = labelsAndUserAuths.getFirst();
376     labels.remove(SYSTEM_LABEL);
377     if (regex != null) {
378       Pattern pattern = Pattern.compile(regex);
379       ArrayList<String> matchedLabels = new ArrayList<String>();
380       for (String label : labels.keySet()) {
381         if (pattern.matcher(label).matches()) {
382           matchedLabels.add(label);
383         }
384       }
385       return matchedLabels;
386     }
387     return new ArrayList<String>(labels.keySet());
388   }
389 
390   @Override
391   public List<Tag> createVisibilityExpTags(String visExpression, boolean withSerializationFormat,
392       boolean checkAuths) throws IOException {
393     Set<Integer> auths = null;
394     if (checkAuths) {
395       auths = this.labelsCache.getAuthsAsOrdinals(VisibilityUtils.getActiveUser().getShortName());
396     }
397     return VisibilityUtils.createVisibilityExpTags(visExpression, withSerializationFormat,
398         checkAuths, auths, labelsCache);
399   }
400 
401   protected void updateZk(boolean labelAddition) throws IOException {
402     // We will add to zookeeper here.
403     // TODO we should add the delta only to zk. Else this will be a very heavy op and when there are
404     // so many labels and auth in the system, we will end up adding lots of data to zk. Most
405     // possibly we will exceed zk node data limit!
406     Pair<Map<String, Integer>, Map<String, List<Integer>>> labelsAndUserAuths =
407         extractLabelsAndAuths(getExistingLabelsWithAuths());
408     Map<String, Integer> existingLabels = labelsAndUserAuths.getFirst();
409     Map<String, List<Integer>> userAuths = labelsAndUserAuths.getSecond();
410     if (labelAddition) {
411       byte[] serialized = VisibilityUtils.getDataToWriteToZooKeeper(existingLabels);
412       this.labelsCache.writeToZookeeper(serialized, true);
413     } else {
414       byte[] serialized = VisibilityUtils.getUserAuthsDataToWriteToZooKeeper(userAuths);
415       this.labelsCache.writeToZookeeper(serialized, false);
416     }
417   }
418 
419   @Override
420   public VisibilityExpEvaluator getVisibilityExpEvaluator(Authorizations authorizations)
421       throws IOException {
422     // If a super user issues a get/scan, he should be able to scan the cells
423     // irrespective of the Visibility labels
424     if (isReadFromSuperUser()) {
425       return new VisibilityExpEvaluator() {
426         @Override
427         public boolean evaluate(Cell cell) throws IOException {
428           return true;
429         }
430       };
431     }
432     List<String> authLabels = null;
433     for (ScanLabelGenerator scanLabelGenerator : scanLabelGenerators) {
434       try {
435         // null authorizations to be handled inside SLG impl.
436         authLabels = scanLabelGenerator.getLabels(VisibilityUtils.getActiveUser(), authorizations);
437         authLabels = (authLabels == null) ? new ArrayList<String>() : authLabels;
438         authorizations = new Authorizations(authLabels);
439       } catch (Throwable t) {
440         LOG.error(t);
441         throw new IOException(t);
442       }
443     }
444     int labelsCount = this.labelsCache.getLabelsCount();
445     final BitSet bs = new BitSet(labelsCount + 1); // ordinal is index 1 based
446     if (authLabels != null) {
447       for (String authLabel : authLabels) {
448         int labelOrdinal = this.labelsCache.getLabelOrdinal(authLabel);
449         if (labelOrdinal != 0) {
450           bs.set(labelOrdinal);
451         }
452       }
453     }
454 
455     return new VisibilityExpEvaluator() {
456       @Override
457       public boolean evaluate(Cell cell) throws IOException {
458         boolean visibilityTagPresent = false;
459         // Save an object allocation where we can
460         if (cell.getTagsLengthUnsigned() > 0) {
461           Iterator<Tag> tagsItr = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(),
462               cell.getTagsLengthUnsigned());
463           while (tagsItr.hasNext()) {
464             boolean includeKV = true;
465             Tag tag = tagsItr.next();
466             if (tag.getType() == VISIBILITY_TAG_TYPE) {
467               visibilityTagPresent = true;
468               int offset = tag.getTagOffset();
469               int endOffset = offset + tag.getTagLength();
470               while (offset < endOffset) {
471                 Pair<Integer, Integer> result = StreamUtils
472                     .readRawVarint32(tag.getBuffer(), offset);
473                 int currLabelOrdinal = result.getFirst();
474                 if (currLabelOrdinal < 0) {
475                   // check for the absence of this label in the Scan Auth labels
476                   // ie. to check BitSet corresponding bit is 0
477                   int temp = -currLabelOrdinal;
478                   if (bs.get(temp)) {
479                     includeKV = false;
480                     break;
481                   }
482                 } else {
483                   if (!bs.get(currLabelOrdinal)) {
484                     includeKV = false;
485                     break;
486                   }
487                 }
488                 offset += result.getSecond();
489               }
490               if (includeKV) {
491                 // We got one visibility expression getting evaluated to true. Good to include this
492                 // KV in the result then.
493                 return true;
494               }
495             }
496           }
497         }
498         return !(visibilityTagPresent);
499       }
500     };
501   }
502 
503   protected boolean isReadFromSuperUser() throws IOException {
504     byte[] user = Bytes.toBytes(VisibilityUtils.getActiveUser().getShortName());
505     return havingSystemAuth(user);
506   }
507 
508   @Override
509   public boolean havingSystemAuth(byte[] user) throws IOException {
510     List<String> auths = this.getAuths(user, true);
511     if (LOG.isTraceEnabled()) {
512       LOG.trace("The auths for user " + Bytes.toString(user) + " are " + auths);
513     }
514     return auths.contains(SYSTEM_LABEL);
515   }
516 
517   @Override
518   public boolean matchVisibility(List<Tag> putVisTags, Byte putTagsFormat, List<Tag> deleteVisTags,
519       Byte deleteTagsFormat) throws IOException {
520     if ((deleteTagsFormat != null && deleteTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT)
521         && (putTagsFormat == null || putTagsFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT)) {
522       if (putVisTags.size() == 0) {
523         // Early out if there are no tags in the cell
524         return false;
525       }
526       if (putTagsFormat == null) {
527         return matchUnSortedVisibilityTags(putVisTags, deleteVisTags);
528       } else {
529         return matchOrdinalSortedVisibilityTags(putVisTags, deleteVisTags);
530       }
531     }
532     throw new IOException("Unexpected tag format passed for comparison, deleteTagsFormat : "
533         + deleteTagsFormat + ", putTagsFormat : " + putTagsFormat);
534   }
535 
536   /**
537    * @param putVisTags Visibility tags in Put Mutation
538    * @param deleteVisTags Visibility tags in Delete Mutation
539    * @return true when all the visibility tags in Put matches with visibility tags in Delete.
540    * This is used when, at least one set of tags are not sorted based on the label ordinal.
541    */
542   private static boolean matchUnSortedVisibilityTags(List<Tag> putVisTags,
543       List<Tag> deleteVisTags) throws IOException {
544     return compareTagsOrdinals(sortTagsBasedOnOrdinal(putVisTags),
545         sortTagsBasedOnOrdinal(deleteVisTags));
546   }
547 
548   /**
549    * @param putVisTags Visibility tags in Put Mutation
550    * @param deleteVisTags Visibility tags in Delete Mutation
551    * @return true when all the visibility tags in Put matches with visibility tags in Delete.
552    * This is used when both the set of tags are sorted based on the label ordinal.
553    */
554   private static boolean matchOrdinalSortedVisibilityTags(List<Tag> putVisTags,
555       List<Tag> deleteVisTags) {
556     boolean matchFound = false;
557     // If the size does not match. Definitely we are not comparing the equal tags.
558     if ((deleteVisTags.size()) == putVisTags.size()) {
559       for (Tag tag : deleteVisTags) {
560         matchFound = false;
561         for (Tag givenTag : putVisTags) {
562           if (Bytes.equals(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength(),
563               givenTag.getBuffer(), givenTag.getTagOffset(), givenTag.getTagLength())) {
564             matchFound = true;
565             break;
566           }
567         }
568         if (!matchFound) break;
569       }
570     }
571     return matchFound;
572   }
573 
574   private static List<List<Integer>> sortTagsBasedOnOrdinal(List<Tag> tags) throws IOException {
575     List<List<Integer>> fullTagsList = new ArrayList<List<Integer>>();
576     for (Tag tag : tags) {
577       if (tag.getType() == VISIBILITY_TAG_TYPE) {
578         getSortedTagOrdinals(fullTagsList, tag);
579       }
580     }
581     return fullTagsList;
582   }
583 
584   private static void getSortedTagOrdinals(List<List<Integer>> fullTagsList, Tag tag)
585       throws IOException {
586     List<Integer> tagsOrdinalInSortedOrder = new ArrayList<Integer>();
587     int offset = tag.getTagOffset();
588     int endOffset = offset + tag.getTagLength();
589     while (offset < endOffset) {
590       Pair<Integer, Integer> result = StreamUtils.readRawVarint32(tag.getBuffer(), offset);
591       tagsOrdinalInSortedOrder.add(result.getFirst());
592       offset += result.getSecond();
593     }
594     Collections.sort(tagsOrdinalInSortedOrder);
595     fullTagsList.add(tagsOrdinalInSortedOrder);
596   }
597 
598   /*
599    * @return true when all the visibility tags in Put matches with visibility tags in Delete.
600    */
601   private static boolean compareTagsOrdinals(List<List<Integer>> putVisTags,
602       List<List<Integer>> deleteVisTags) {
603     boolean matchFound = false;
604     if (deleteVisTags.size() == putVisTags.size()) {
605       for (List<Integer> deleteTagOrdinals : deleteVisTags) {
606         matchFound = false;
607         for (List<Integer> tagOrdinals : putVisTags) {
608           if (deleteTagOrdinals.equals(tagOrdinals)) {
609             matchFound = true;
610             break;
611           }
612         }
613         if (!matchFound) break;
614       }
615     }
616     return matchFound;
617   }
618 
619   @Override
620   public byte[] encodeVisibilityForReplication(final List<Tag> tags, final Byte serializationFormat)
621       throws IOException {
622     if (tags.size() > 0
623         && (serializationFormat == null ||
624         serializationFormat == SORTED_ORDINAL_SERIALIZATION_FORMAT)) {
625       return createModifiedVisExpression(tags);
626     }
627     return null;
628   }
629 
630   /**
631    * @param tags
632    *          - all the visibility tags associated with the current Cell
633    * @return - the modified visibility expression as byte[]
634    */
635   private byte[] createModifiedVisExpression(final List<Tag> tags)
636       throws IOException {
637     StringBuilder visibilityString = new StringBuilder();
638     for (Tag tag : tags) {
639       if (tag.getType() == TagType.VISIBILITY_TAG_TYPE) {
640         if (visibilityString.length() != 0) {
641           visibilityString.append(VisibilityConstants.CLOSED_PARAN).append(
642               VisibilityConstants.OR_OPERATOR);
643         }
644         int offset = tag.getTagOffset();
645         int endOffset = offset + tag.getTagLength();
646         boolean expressionStart = true;
647         while (offset < endOffset) {
648           Pair<Integer, Integer> result = StreamUtils.readRawVarint32(tag.getBuffer(), offset);
649           int currLabelOrdinal = result.getFirst();
650           if (currLabelOrdinal < 0) {
651             int temp = -currLabelOrdinal;
652             String label = this.labelsCache.getLabel(temp);
653             if (expressionStart) {
654               // Quote every label in case of unicode characters if present
655               visibilityString.append(VisibilityConstants.OPEN_PARAN)
656                   .append(VisibilityConstants.NOT_OPERATOR).append(CellVisibility.quote(label));
657             } else {
658               visibilityString.append(VisibilityConstants.AND_OPERATOR)
659                   .append(VisibilityConstants.NOT_OPERATOR).append(CellVisibility.quote(label));
660             }
661           } else {
662             String label = this.labelsCache.getLabel(currLabelOrdinal);
663             if (expressionStart) {
664               visibilityString.append(VisibilityConstants.OPEN_PARAN).append(
665                   CellVisibility.quote(label));
666             } else {
667               visibilityString.append(VisibilityConstants.AND_OPERATOR).append(
668                   CellVisibility.quote(label));
669             }
670           }
671           expressionStart = false;
672           offset += result.getSecond();
673         }
674       }
675     }
676     if (visibilityString.length() != 0) {
677       visibilityString.append(VisibilityConstants.CLOSED_PARAN);
678       // Return the string formed as byte[]
679       return Bytes.toBytes(visibilityString.toString());
680     }
681     return null;
682   }
683 }