View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.security.visibility;
19  
20  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_FAMILY;
21  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABELS_TABLE_NAME;
22  import static org.apache.hadoop.hbase.security.visibility.VisibilityConstants.LABEL_QUALIFIER;
23  import static org.junit.Assert.assertEquals;
24  import static org.junit.Assert.assertFalse;
25  import static org.junit.Assert.assertNotNull;
26  import static org.junit.Assert.assertNull;
27  import static org.junit.Assert.assertTrue;
28  import static org.junit.Assert.fail;
29  
30  import java.io.IOException;
31  import java.security.PrivilegedExceptionAction;
32  import java.util.ArrayList;
33  import java.util.List;
34  
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.hbase.Cell;
37  import org.apache.hadoop.hbase.CellScanner;
38  import org.apache.hadoop.hbase.HBaseTestingUtility;
39  import org.apache.hadoop.hbase.HColumnDescriptor;
40  import org.apache.hadoop.hbase.HConstants;
41  import org.apache.hadoop.hbase.HTableDescriptor;
42  import org.apache.hadoop.hbase.TableName;
43  import org.apache.hadoop.hbase.client.Append;
44  import org.apache.hadoop.hbase.client.Get;
45  import org.apache.hadoop.hbase.client.HBaseAdmin;
46  import org.apache.hadoop.hbase.client.HTable;
47  import org.apache.hadoop.hbase.client.Increment;
48  import org.apache.hadoop.hbase.client.Put;
49  import org.apache.hadoop.hbase.client.Result;
50  import org.apache.hadoop.hbase.client.ResultScanner;
51  import org.apache.hadoop.hbase.client.RowMutations;
52  import org.apache.hadoop.hbase.client.Scan;
53  import org.apache.hadoop.hbase.client.security.SecurityCapability;
54  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.RegionActionResult;
55  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.GetAuthsResponse;
56  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
57  import org.apache.hadoop.hbase.regionserver.BloomType;
58  import org.apache.hadoop.hbase.regionserver.HRegion;
59  import org.apache.hadoop.hbase.regionserver.HRegionServer;
60  import org.apache.hadoop.hbase.security.User;
61  import org.apache.hadoop.hbase.util.Bytes;
62  import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
63  import org.junit.After;
64  import org.junit.AfterClass;
65  import org.junit.Rule;
66  import org.junit.Test;
67  import org.junit.rules.TestName;
68  
69  import com.google.protobuf.ByteString;
70  
71  /**
72   * Base test class for visibility labels basic features
73   */
74  public abstract class TestVisibilityLabels {
75  
76    public static final String TOPSECRET = "topsecret";
77    public static final String PUBLIC = "public";
78    public static final String PRIVATE = "private";
79    public static final String CONFIDENTIAL = "confidential";
80    public static final String SECRET = "secret";
81    public static final String COPYRIGHT = "\u00A9ABC";
82    public static final String ACCENT = "\u0941";
83    public static final String UNICODE_VIS_TAG = COPYRIGHT + "\"" + ACCENT + "\\" + SECRET + "\""
84        + "\u0027&\\";
85    public static final String UC1 = "\u0027\"\u002b";
86    public static final String UC2 = "\u002d\u003f";
87    public static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
88    public static final byte[] row1 = Bytes.toBytes("row1");
89    public static final byte[] row2 = Bytes.toBytes("row2");
90    public static final byte[] row3 = Bytes.toBytes("row3");
91    public static final byte[] row4 = Bytes.toBytes("row4");
92    public final static byte[] fam = Bytes.toBytes("info");
93    public final static byte[] qual = Bytes.toBytes("qual");
94    public final static byte[] value = Bytes.toBytes("value");
95    public static Configuration conf;
96  
97    private volatile boolean killedRS = false;
98    @Rule 
99    public final TestName TEST_NAME = new TestName();
100   public static User SUPERUSER, USER1;
101 
102   @AfterClass
103   public static void tearDownAfterClass() throws Exception {
104     TEST_UTIL.shutdownMiniCluster();
105   }
106 
107   @After
108   public void tearDown() throws Exception {
109     killedRS = false;
110   }
111 
112   @Test
113   public void testSecurityCapabilities() throws Exception {
114     List<SecurityCapability> capabilities = TEST_UTIL.getHBaseAdmin().getSecurityCapabilities();
115     assertTrue("CELL_VISIBILITY capability is missing",
116       capabilities.contains(SecurityCapability.CELL_VISIBILITY));
117   }
118 
119   @Test
120   public void testSimpleVisibilityLabels() throws Exception {
121     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
122     HTable table = createTableAndWriteDataWithLabels(tableName, SECRET + "|" + CONFIDENTIAL,
123         PRIVATE + "|" + CONFIDENTIAL);
124     try {
125       Scan s = new Scan();
126       s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE));
127       ResultScanner scanner = table.getScanner(s);
128       Result[] next = scanner.next(3);
129 
130       assertTrue(next.length == 2);
131       CellScanner cellScanner = next[0].cellScanner();
132       cellScanner.advance();
133       Cell current = cellScanner.current();
134       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
135           current.getRowLength(), row1, 0, row1.length));
136       cellScanner = next[1].cellScanner();
137       cellScanner.advance();
138       current = cellScanner.current();
139       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
140           current.getRowLength(), row2, 0, row2.length));
141     } finally {
142       if (table != null) {
143         table.close();
144       }
145     }
146   }
147   
148   @Test
149   public void testSimpleVisibilityLabelsWithUniCodeCharacters() throws Exception {
150     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
151     HTable table = createTableAndWriteDataWithLabels(tableName,
152         SECRET + "|" + CellVisibility.quote(COPYRIGHT), "(" + CellVisibility.quote(COPYRIGHT) + "&"
153             + CellVisibility.quote(ACCENT) + ")|" + CONFIDENTIAL,
154         CellVisibility.quote(UNICODE_VIS_TAG) + "&" + SECRET);
155     try {
156       Scan s = new Scan();
157       s.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL, PRIVATE, COPYRIGHT, ACCENT,
158           UNICODE_VIS_TAG));
159       ResultScanner scanner = table.getScanner(s);
160       Result[] next = scanner.next(3);
161       assertTrue(next.length == 3);
162       CellScanner cellScanner = next[0].cellScanner();
163       cellScanner.advance();
164       Cell current = cellScanner.current();
165       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
166           current.getRowLength(), row1, 0, row1.length));
167       cellScanner = next[1].cellScanner();
168       cellScanner.advance();
169       current = cellScanner.current();
170       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
171           current.getRowLength(), row2, 0, row2.length));
172       cellScanner = next[2].cellScanner();
173       cellScanner.advance();
174       current = cellScanner.current();
175       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
176           current.getRowLength(), row3, 0, row3.length));
177     } finally {
178       if (table != null) {
179         table.close();
180       }
181     }
182   }
183 
184   @Test
185   public void testAuthorizationsWithSpecialUnicodeCharacters() throws Exception {
186     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
187     HTable table = createTableAndWriteDataWithLabels(tableName,
188         CellVisibility.quote(UC1) + "|" + CellVisibility.quote(UC2), CellVisibility.quote(UC1),
189         CellVisibility.quote(UNICODE_VIS_TAG));
190     try {
191       Scan s = new Scan();
192       s.setAuthorizations(new Authorizations(UC1, UC2, ACCENT,
193           UNICODE_VIS_TAG));
194       ResultScanner scanner = table.getScanner(s);
195       Result[] next = scanner.next(3);
196       assertTrue(next.length == 3);
197       CellScanner cellScanner = next[0].cellScanner();
198       cellScanner.advance();
199       Cell current = cellScanner.current();
200       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
201           current.getRowLength(), row1, 0, row1.length));
202       cellScanner = next[1].cellScanner();
203       cellScanner.advance();
204       current = cellScanner.current();
205       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
206           current.getRowLength(), row2, 0, row2.length));
207       cellScanner = next[2].cellScanner();
208       cellScanner.advance();
209       current = cellScanner.current();
210       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
211           current.getRowLength(), row3, 0, row3.length));
212     } finally {
213       if (table != null) {
214         table.close();
215       }
216     }
217   }
218 
219   @Test
220   public void testVisibilityLabelsWithComplexLabels() throws Exception {
221     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
222     HTable table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
223         + ")" + "&" + "!" + TOPSECRET, "(" + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")", "("
224         + PRIVATE + "&" + CONFIDENTIAL + "&" + SECRET + ")", "(" + PRIVATE + "&" + CONFIDENTIAL
225         + "&" + SECRET + ")");
226     try {
227       Scan s = new Scan();
228       s.setAuthorizations(new Authorizations(TOPSECRET, CONFIDENTIAL, PRIVATE, PUBLIC, SECRET));
229       ResultScanner scanner = table.getScanner(s);
230       Result[] next = scanner.next(4);
231       assertEquals(3, next.length);
232       CellScanner cellScanner = next[0].cellScanner();
233       cellScanner.advance();
234       Cell current = cellScanner.current();
235       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
236           current.getRowLength(), row2, 0, row2.length));
237       cellScanner = next[1].cellScanner();
238       cellScanner.advance();
239       current = cellScanner.current();
240       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
241           current.getRowLength(), row3, 0, row3.length));
242       cellScanner = next[2].cellScanner();
243       cellScanner.advance();
244       current = cellScanner.current();
245       assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(),
246           current.getRowLength(), row4, 0, row4.length));
247     } finally {
248       if (table != null) {
249         table.close();
250       }
251     }
252   }
253 
254   @Test
255   public void testVisibilityLabelsThatDoesNotPassTheCriteria() throws Exception {
256     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
257     HTable table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
258         + ")", PRIVATE);
259     try {
260       Scan s = new Scan();
261       s.setAuthorizations(new Authorizations(PUBLIC));
262       ResultScanner scanner = table.getScanner(s);
263       Result[] next = scanner.next(3);
264       assertTrue(next.length == 0);
265     } finally {
266       if (table != null) {
267         table.close();
268       }
269     }
270   }
271 
272   @Test
273   public void testVisibilityLabelsInPutsThatDoesNotMatchAnyDefinedLabels() throws Exception {
274     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
275     try {
276       createTableAndWriteDataWithLabels(tableName, "SAMPLE_LABEL", "TEST");
277       fail("Should have failed with failed sanity check exception");
278     } catch (Exception e) {
279     }
280   }
281 
282   @Test
283   public void testVisibilityLabelsInScanThatDoesNotMatchAnyDefinedLabels() throws Exception {
284     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
285     HTable table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
286         + ")", PRIVATE);
287     try {
288       Scan s = new Scan();
289       s.setAuthorizations(new Authorizations("SAMPLE"));
290       ResultScanner scanner = table.getScanner(s);
291       Result[] next = scanner.next(3);
292       assertTrue(next.length == 0);
293     } finally {
294       if (table != null) {
295         table.close();
296       }
297     }
298   }
299 
300   @Test
301   public void testVisibilityLabelsWithGet() throws Exception {
302     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
303     HTable table = createTableAndWriteDataWithLabels(tableName, SECRET + "&" + CONFIDENTIAL + "&!"
304         + PRIVATE, SECRET + "&" + CONFIDENTIAL + "&" + PRIVATE);
305     try {
306       Get get = new Get(row1);
307       get.setAuthorizations(new Authorizations(SECRET, CONFIDENTIAL));
308       Result result = table.get(get);
309       assertTrue(!result.isEmpty());
310       Cell cell = result.getColumnLatestCell(fam, qual);
311       assertTrue(Bytes.equals(value, 0, value.length, cell.getValueArray(), cell.getValueOffset(),
312           cell.getValueLength()));
313     } finally {
314       if (table != null) {
315         table.close();
316       }
317     }
318   }
319 
320   @Test
321   public void testVisibilityLabelsOnKillingOfRSContainingLabelsTable() throws Exception {
322     List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
323         .getRegionServerThreads();
324     int liveRS = 0;
325     for (RegionServerThread rsThreads : regionServerThreads) {
326       if (!rsThreads.getRegionServer().isAborted()) {
327         liveRS++;
328       }
329     }
330     if (liveRS == 1) {
331       TEST_UTIL.getHBaseCluster().startRegionServer();
332     }
333     Thread t1 = new Thread() {
334       public void run() {
335         List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
336             .getRegionServerThreads();
337         for (RegionServerThread rsThread : regionServerThreads) {
338           List<HRegion> onlineRegions = rsThread.getRegionServer().getOnlineRegions(
339               LABELS_TABLE_NAME);
340           if (onlineRegions.size() > 0) {
341             rsThread.getRegionServer().abort("Aborting ");
342             killedRS = true;
343             break;
344           }
345         }
346       }
347 
348     };
349     t1.start();
350     final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
351     Thread t = new Thread() {
352       public void run() {
353         try {
354           while (!killedRS) {
355             Thread.sleep(1);
356           }
357           createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL + ")",
358               PRIVATE);
359         } catch (Exception e) {
360         }
361       }
362     };
363     t.start();
364     regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
365     while (!killedRS) {
366       Thread.sleep(10);
367     }
368     regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
369     for (RegionServerThread rsThread : regionServerThreads) {
370       while (true) {
371         if (!rsThread.getRegionServer().isAborted()) {
372           List<HRegion> onlineRegions = rsThread.getRegionServer().getOnlineRegions(
373               LABELS_TABLE_NAME);
374           if (onlineRegions.size() > 0) {
375             break;
376           } else {
377             Thread.sleep(10);
378           }
379         } else {
380           break;
381         }
382       }
383     }
384     TEST_UTIL.waitTableEnabled(LABELS_TABLE_NAME.getName(), 50000);
385     t.join();
386     HTable table = null;
387     try {
388       table = new HTable(TEST_UTIL.getConfiguration(), tableName);
389       Scan s = new Scan();
390       s.setAuthorizations(new Authorizations(SECRET));
391       ResultScanner scanner = table.getScanner(s);
392       Result[] next = scanner.next(3);
393       assertTrue(next.length == 1);
394     } finally {
395       if (table != null) {
396         table.close();
397       }
398     }
399   }
400 
401   @Test(timeout = 60 * 1000)
402   public void testVisibilityLabelsOnRSRestart() throws Exception {
403     final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
404     HTable table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
405         + ")", PRIVATE);
406     List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster()
407         .getRegionServerThreads();
408     for (RegionServerThread rsThread : regionServerThreads) {
409       rsThread.getRegionServer().abort("Aborting ");
410     }
411     // Start one new RS
412     RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
413     waitForLabelsRegionAvailability(rs.getRegionServer());
414     try {
415       Scan s = new Scan();
416       s.setAuthorizations(new Authorizations(SECRET));
417       ResultScanner scanner = table.getScanner(s);
418       Result[] next = scanner.next(3);
419       assertTrue(next.length == 1);
420     } finally {
421       if (table != null) {
422         table.close();
423       }
424     }
425   }
426 
427   protected void waitForLabelsRegionAvailability(HRegionServer regionServer) {
428     while (!regionServer.isOnline()) {
429       try {
430         Thread.sleep(10);
431       } catch (InterruptedException e) {
432       }
433     }
434     while (regionServer.getOnlineRegions(LABELS_TABLE_NAME).isEmpty()) {
435       try {
436         Thread.sleep(10);
437       } catch (InterruptedException e) {
438       }
439     }
440     HRegion labelsTableRegion = regionServer.getOnlineRegions(LABELS_TABLE_NAME).get(0);
441     while (labelsTableRegion.isRecovering()) {
442       try {
443         Thread.sleep(10);
444       } catch (InterruptedException e) {
445       }
446     }
447   }
448 
449   @Test
450   public void testVisibilityLabelsInGetThatDoesNotMatchAnyDefinedLabels() throws Exception {
451     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
452     HTable table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL
453         + ")", PRIVATE);
454     try {
455       Get get = new Get(row1);
456       get.setAuthorizations(new Authorizations("SAMPLE"));
457       Result result = table.get(get);
458       assertTrue(result.isEmpty());
459     } finally {
460       if (table != null) {
461         table.close();
462       }
463     }
464   }
465 
466   @Test
467   public void testSetAndGetUserAuths() throws Throwable {
468     final String user = "user1";
469     PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
470       public Void run() throws Exception {
471         String[] auths = { SECRET, CONFIDENTIAL };
472         try {
473           VisibilityClient.setAuths(conf, auths, user);
474         } catch (Throwable e) {
475         }
476         return null;
477       }
478     };
479     SUPERUSER.runAs(action);
480     HTable ht = null;
481     try {
482       ht = new HTable(conf, LABELS_TABLE_NAME);
483       Scan scan = new Scan();
484       scan.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL));
485       ResultScanner scanner = ht.getScanner(scan);
486       Result result = null;
487       List<Result> results = new ArrayList<Result>();
488       while ((result = scanner.next()) != null) {
489         results.add(result);
490       }
491       List<String> auths = extractAuths(user, results);
492       assertTrue(auths.contains(SECRET));
493       assertTrue(auths.contains(CONFIDENTIAL));
494       assertEquals(2, auths.size());
495     } finally {
496       if (ht != null) {
497         ht.close();
498       }
499     }
500 
501     action = new PrivilegedExceptionAction<Void>() {
502       public Void run() throws Exception {
503         GetAuthsResponse authsResponse = null;
504         try {
505           authsResponse = VisibilityClient.getAuths(conf, user);
506         } catch (Throwable e) {
507           fail("Should not have failed");
508         }
509         List<String> authsList = new ArrayList<String>();
510         for (ByteString authBS : authsResponse.getAuthList()) {
511           authsList.add(Bytes.toString(authBS.toByteArray()));
512         }
513         assertEquals(2, authsList.size());
514         assertTrue(authsList.contains(SECRET));
515         assertTrue(authsList.contains(CONFIDENTIAL));
516         return null;
517       }
518     };
519     SUPERUSER.runAs(action);
520 
521     // Try doing setAuths once again and there should not be any duplicates
522     action = new PrivilegedExceptionAction<Void>() {
523       public Void run() throws Exception {
524         String[] auths1 = { SECRET, CONFIDENTIAL };
525         GetAuthsResponse authsResponse = null;
526         try {
527           VisibilityClient.setAuths(conf, auths1, user);
528           try {
529             authsResponse = VisibilityClient.getAuths(conf, user);
530           } catch (Throwable e) {
531             fail("Should not have failed");
532           }
533         } catch (Throwable e) {
534         }
535         List<String> authsList = new ArrayList<String>();
536         for (ByteString authBS : authsResponse.getAuthList()) {
537           authsList.add(Bytes.toString(authBS.toByteArray()));
538         }
539         assertEquals(2, authsList.size());
540         assertTrue(authsList.contains(SECRET));
541         assertTrue(authsList.contains(CONFIDENTIAL));
542         return null;
543       }
544     };
545     SUPERUSER.runAs(action);
546   }
547 
548   protected List<String> extractAuths(String user, List<Result> results) {
549     List<String> auths = new ArrayList<String>();
550     for (Result result : results) {
551       Cell labelCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, LABEL_QUALIFIER);
552       Cell userAuthCell = result.getColumnLatestCell(LABELS_TABLE_FAMILY, user.getBytes());
553       if (userAuthCell != null) {
554         auths.add(Bytes.toString(labelCell.getValueArray(), labelCell.getValueOffset(),
555             labelCell.getValueLength()));
556       }
557     }
558     return auths;
559   }
560 
561   @Test
562   public void testClearUserAuths() throws Throwable {
563     PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {
564       public Void run() throws Exception {
565         String[] auths = { SECRET, CONFIDENTIAL, PRIVATE };
566         String user = "testUser";
567         try {
568           VisibilityClient.setAuths(conf, auths, user);
569         } catch (Throwable e) {
570           fail("Should not have failed");
571         }
572         // Removing the auths for SECRET and CONFIDENTIAL for the user.
573         // Passing a non existing auth also.
574         auths = new String[] { SECRET, PUBLIC, CONFIDENTIAL };
575         VisibilityLabelsResponse response = null;
576         try {
577           response = VisibilityClient.clearAuths(conf, auths, user);
578         } catch (Throwable e) {
579           fail("Should not have failed");
580         }
581         List<RegionActionResult> resultList = response.getResultList();
582         assertEquals(3, resultList.size());
583         assertTrue(resultList.get(0).getException().getValue().isEmpty());
584         assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException",
585             resultList.get(1).getException().getName());
586         assertTrue(Bytes.toString(resultList.get(1).getException().getValue().toByteArray())
587             .contains(
588                 "org.apache.hadoop.hbase.security.visibility.InvalidLabelException: "
589                     + "Label 'public' is not set for the user testUser"));
590         assertTrue(resultList.get(2).getException().getValue().isEmpty());
591         HTable ht = null;
592         try {
593           ht = new HTable(conf, LABELS_TABLE_NAME);
594           ResultScanner scanner = ht.getScanner(new Scan());
595           Result result = null;
596           List<Result> results = new ArrayList<Result>();
597           while ((result = scanner.next()) != null) {
598             results.add(result);
599           }
600           List<String> curAuths = extractAuths(user, results);
601           assertTrue(curAuths.contains(PRIVATE));
602           assertEquals(1, curAuths.size());
603         } finally {
604           if (ht != null) {
605             ht.close();
606           }
607         }
608 
609         GetAuthsResponse authsResponse = null;
610         try {
611           authsResponse = VisibilityClient.getAuths(conf, user);
612         } catch (Throwable e) {
613           fail("Should not have failed");
614         }
615         List<String> authsList = new ArrayList<String>();
616         for (ByteString authBS : authsResponse.getAuthList()) {
617           authsList.add(Bytes.toString(authBS.toByteArray()));
618         }
619         assertEquals(1, authsList.size());
620         assertTrue(authsList.contains(PRIVATE));
621         return null;
622       }
623     };
624     SUPERUSER.runAs(action);
625   }
626 
627   @Test
628   public void testLabelsWithCheckAndPut() throws Throwable {
629     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
630     HTable table = null;
631     try {
632       table = TEST_UTIL.createTable(tableName, fam);
633       byte[] row1 = Bytes.toBytes("row1");
634       Put put = new Put(row1);
635       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value);
636       put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
637       table.checkAndPut(row1, fam, qual, null, put);
638       byte[] row2 = Bytes.toBytes("row2");
639       put = new Put(row2);
640       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value);
641       put.setCellVisibility(new CellVisibility(SECRET));
642       table.checkAndPut(row2, fam, qual, null, put);
643       
644       Scan scan = new Scan();
645       scan.setAuthorizations(new Authorizations(SECRET));
646       ResultScanner scanner = table.getScanner(scan);
647       Result result = scanner.next();
648       assertTrue(!result.isEmpty());
649       assertTrue(Bytes.equals(row2, result.getRow()));
650       result = scanner.next();
651       assertNull(result);
652     } finally {
653       if (table != null) {
654         table.close();
655       }
656     }
657   }
658 
659   @Test
660   public void testLabelsWithIncrement() throws Throwable {
661     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
662     HTable table = null;
663     try {
664       table = TEST_UTIL.createTable(tableName, fam);
665       byte[] row1 = Bytes.toBytes("row1");
666       byte[] val = Bytes.toBytes(1L);
667       Put put = new Put(row1);
668       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, val);
669       put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
670       table.put(put);
671       Get get = new Get(row1);
672       get.setAuthorizations(new Authorizations(SECRET));
673       Result result = table.get(get);
674       assertTrue(result.isEmpty());
675       table.incrementColumnValue(row1, fam, qual, 2L);
676       result = table.get(get);
677       assertTrue(result.isEmpty());
678       Increment increment = new Increment(row1);
679       increment.addColumn(fam, qual, 2L);
680       increment.setCellVisibility(new CellVisibility(SECRET));
681       table.increment(increment);
682       result = table.get(get);
683       assertTrue(!result.isEmpty());
684     } finally {
685       if (table != null) {
686         table.close();
687       }
688     }
689   }
690 
691   @Test
692   public void testLabelsWithAppend() throws Throwable {
693     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
694     HTable table = null;
695     try {
696       table = TEST_UTIL.createTable(tableName, fam);
697       byte[] row1 = Bytes.toBytes("row1");
698       byte[] val = Bytes.toBytes("a");
699       Put put = new Put(row1);
700       put.add(fam, qual, HConstants.LATEST_TIMESTAMP, val);
701       put.setCellVisibility(new CellVisibility(SECRET + " & " + CONFIDENTIAL));
702       table.put(put);
703       Get get = new Get(row1);
704       get.setAuthorizations(new Authorizations(SECRET));
705       Result result = table.get(get);
706       assertTrue(result.isEmpty());
707       Append append = new Append(row1);
708       append.add(fam, qual, Bytes.toBytes("b"));
709       table.append(append);
710       result = table.get(get);
711       assertTrue(result.isEmpty());
712       append = new Append(row1);
713       append.add(fam, qual, Bytes.toBytes("c"));
714       append.setCellVisibility(new CellVisibility(SECRET));
715       table.append(append);
716       result = table.get(get);
717       assertTrue(!result.isEmpty());
718     } finally {
719       if (table != null) {
720         table.close();
721       }
722     }
723   }
724 
725   @Test
726   public void testUserShouldNotDoDDLOpOnLabelsTable() throws Exception {
727     HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
728     try {
729       admin.disableTable(LABELS_TABLE_NAME);
730       fail("Lables table should not get disabled by user.");
731     } catch (Exception e) {
732     }
733     try {
734       admin.deleteTable(LABELS_TABLE_NAME);
735       fail("Lables table should not get disabled by user.");
736     } catch (Exception e) {
737     }
738     try {
739       HColumnDescriptor hcd = new HColumnDescriptor("testFamily");
740       admin.addColumn(LABELS_TABLE_NAME, hcd);
741       fail("Lables table should not get altered by user.");
742     } catch (Exception e) {
743     }
744     try {
745       admin.deleteColumn(LABELS_TABLE_NAME, VisibilityConstants.LABELS_TABLE_FAMILY);
746       fail("Lables table should not get altered by user.");
747     } catch (Exception e) {
748     }
749     try {
750       HColumnDescriptor hcd = new HColumnDescriptor(VisibilityConstants.LABELS_TABLE_FAMILY);
751       hcd.setBloomFilterType(BloomType.ROWCOL);
752       admin.modifyColumn(LABELS_TABLE_NAME, hcd);
753       fail("Lables table should not get altered by user.");
754     } catch (Exception e) {
755     }
756     try {
757       HTableDescriptor htd = new HTableDescriptor(LABELS_TABLE_NAME);
758       htd.addFamily(new HColumnDescriptor("f1"));
759       htd.addFamily(new HColumnDescriptor("f2"));
760       admin.modifyTable(LABELS_TABLE_NAME, htd);
761       fail("Lables table should not get altered by user.");
762     } catch (Exception e) {
763     }
764   }
765 
766   @Test
767   public void testMultipleVersions() throws Exception {
768     final byte[] r1 = Bytes.toBytes("row1");
769     final byte[] r2 = Bytes.toBytes("row2");
770     final byte[] v1 = Bytes.toBytes("100");
771     final byte[] v2 = Bytes.toBytes("101");
772     final byte[] fam2 = Bytes.toBytes("info2");
773     final byte[] qual2 = Bytes.toBytes("qual2");
774     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
775     HTableDescriptor desc = new HTableDescriptor(tableName);
776     HColumnDescriptor col = new HColumnDescriptor(fam);// Default max versions is 1.
777     desc.addFamily(col);
778     col = new HColumnDescriptor(fam2);
779     col.setMaxVersions(5);
780     desc.addFamily(col);
781     TEST_UTIL.getHBaseAdmin().createTable(desc);
782     HTable table = null;
783     try {
784       table = new HTable(TEST_UTIL.getConfiguration(), tableName);
785       Put put = new Put(r1);
786       put.add(fam, qual, 3l, v1);
787       put.add(fam, qual2, 3l, v1);
788       put.add(fam2, qual, 3l, v1);
789       put.add(fam2, qual2, 3l, v1);
790       put.setCellVisibility(new CellVisibility(SECRET));
791       table.put(put);
792       put = new Put(r1);
793       put.add(fam, qual, 4l, v2);
794       put.add(fam, qual2, 4l, v2);
795       put.add(fam2, qual, 4l, v2);
796       put.add(fam2, qual2, 4l, v2);
797       put.setCellVisibility(new CellVisibility(PRIVATE));
798       table.put(put);
799 
800       put = new Put(r2);
801       put.add(fam, qual, 3l, v1);
802       put.add(fam, qual2, 3l, v1);
803       put.add(fam2, qual, 3l, v1);
804       put.add(fam2, qual2, 3l, v1);
805       put.setCellVisibility(new CellVisibility(SECRET));
806       table.put(put);
807       put = new Put(r2);
808       put.add(fam, qual, 4l, v2);
809       put.add(fam, qual2, 4l, v2);
810       put.add(fam2, qual, 4l, v2);
811       put.add(fam2, qual2, 4l, v2);
812       put.setCellVisibility(new CellVisibility(SECRET));
813       table.put(put);
814 
815       Scan s = new Scan();
816       s.setMaxVersions(1);
817       s.setAuthorizations(new Authorizations(SECRET));
818       ResultScanner scanner = table.getScanner(s);
819       Result result = scanner.next();
820       assertTrue(Bytes.equals(r1, result.getRow()));
821       // for cf 'fam' max versions in HCD is 1. So the old version cells, which are having matching
822       // CellVisibility with Authorizations, should not get considered in the label evaluation at
823       // all.
824       assertNull(result.getColumnLatestCell(fam, qual));
825       assertNull(result.getColumnLatestCell(fam, qual2));
826       // for cf 'fam2' max versions in HCD is > 1. So we can consider the old version cells, which
827       // are having matching CellVisibility with Authorizations, in the label evaluation. It can
828       // just skip those recent versions for which visibility is not there as per the new version's
829       // CellVisibility. The old versions which are having visibility can be send back
830       Cell cell = result.getColumnLatestCell(fam2, qual);
831       assertNotNull(cell);
832       assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
833           cell.getValueLength()));
834       cell = result.getColumnLatestCell(fam2, qual2);
835       assertNotNull(cell);
836       assertTrue(Bytes.equals(v1, 0, v1.length, cell.getValueArray(), cell.getValueOffset(),
837           cell.getValueLength()));
838 
839       result = scanner.next();
840       assertTrue(Bytes.equals(r2, result.getRow()));
841       cell = result.getColumnLatestCell(fam, qual);
842       assertNotNull(cell);
843       assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
844           cell.getValueLength()));
845       cell = result.getColumnLatestCell(fam, qual2);
846       assertNotNull(cell);
847       assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
848           cell.getValueLength()));
849       cell = result.getColumnLatestCell(fam2, qual);
850       assertNotNull(cell);
851       assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
852           cell.getValueLength()));
853       cell = result.getColumnLatestCell(fam2, qual2);
854       assertNotNull(cell);
855       assertTrue(Bytes.equals(v2, 0, v2.length, cell.getValueArray(), cell.getValueOffset(),
856           cell.getValueLength()));
857     } finally {
858       if (table != null) {
859         table.close();
860       }
861     }
862   }
863 
864   @Test
865   public void testMutateRow() throws Exception {
866     final byte[] qual2 = Bytes.toBytes("qual2");
867     TableName tableName = TableName.valueOf(TEST_NAME.getMethodName());
868     HTableDescriptor desc = new HTableDescriptor(tableName);
869     HColumnDescriptor col = new HColumnDescriptor(fam);
870     desc.addFamily(col);
871     TEST_UTIL.getHBaseAdmin().createTable(desc);
872     HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
873     try {
874       Put p1 = new Put(row1);
875       p1.add(fam, qual, value);
876       p1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
877 
878       Put p2 = new Put(row1);
879       p2.add(fam, qual2, value);
880       p2.setCellVisibility(new CellVisibility(SECRET));
881 
882       RowMutations rm = new RowMutations(row1);
883       rm.add(p1);
884       rm.add(p2);
885 
886       table.mutateRow(rm);
887 
888       Get get = new Get(row1);
889       get.setAuthorizations(new Authorizations(CONFIDENTIAL));
890       Result result = table.get(get);
891       assertTrue(result.containsColumn(fam, qual));
892       assertFalse(result.containsColumn(fam, qual2));
893 
894       get.setAuthorizations(new Authorizations(SECRET));
895       result = table.get(get);
896       assertFalse(result.containsColumn(fam, qual));
897       assertTrue(result.containsColumn(fam, qual2));
898     } finally {
899       table.close();
900     }
901   }
902 
903   static HTable createTableAndWriteDataWithLabels(TableName tableName, String... labelExps)
904       throws Exception {
905     HTable table = null;
906     try {
907       table = TEST_UTIL.createTable(tableName, fam);
908       int i = 1;
909       List<Put> puts = new ArrayList<Put>();
910       for (String labelExp : labelExps) {
911         Put put = new Put(Bytes.toBytes("row" + i));
912         put.add(fam, qual, HConstants.LATEST_TIMESTAMP, value);
913         put.setCellVisibility(new CellVisibility(labelExp));
914         puts.add(put);
915         i++;
916       }
917       table.put(puts);
918     } finally {
919       if (table != null) {
920         table.close();
921       }
922     }
923     return table;
924   }
925 
926   public static void addLabels() throws Exception {
927     PrivilegedExceptionAction<VisibilityLabelsResponse> action =
928         new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
929       public VisibilityLabelsResponse run() throws Exception {
930         String[] labels = { SECRET, TOPSECRET, CONFIDENTIAL, PUBLIC, PRIVATE, COPYRIGHT, ACCENT,
931             UNICODE_VIS_TAG, UC1, UC2 };
932         try {
933           VisibilityClient.addLabels(conf, labels);
934         } catch (Throwable t) {
935           throw new IOException(t);
936         }
937         return null;
938       }
939     };
940     SUPERUSER.runAs(action);
941   }
942 }