View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.thrift2;
19  
20  import static java.nio.ByteBuffer.wrap;
21  import static org.junit.Assert.assertArrayEquals;
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNull;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.nio.ByteBuffer;
28  import java.security.PrivilegedExceptionAction;
29  import java.util.ArrayList;
30  import java.util.Collections;
31  import java.util.Comparator;
32  import java.util.List;
33  
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  import org.apache.hadoop.conf.Configuration;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HColumnDescriptor;
39  import org.apache.hadoop.hbase.HTableDescriptor;
40  import org.apache.hadoop.hbase.MediumTests;
41  import org.apache.hadoop.hbase.TableName;
42  import org.apache.hadoop.hbase.client.HBaseAdmin;
43  import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
44  import org.apache.hadoop.hbase.security.User;
45  import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
46  import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
47  import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
48  import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
49  import org.apache.hadoop.hbase.security.visibility.VisibilityController;
50  import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
51  import org.apache.hadoop.hbase.thrift2.generated.TAppend;
52  import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
53  import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
54  import org.apache.hadoop.hbase.thrift2.generated.TColumn;
55  import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
56  import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
57  import org.apache.hadoop.hbase.thrift2.generated.TGet;
58  import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
59  import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
60  import org.apache.hadoop.hbase.thrift2.generated.TPut;
61  import org.apache.hadoop.hbase.thrift2.generated.TResult;
62  import org.apache.hadoop.hbase.thrift2.generated.TScan;
63  import org.apache.hadoop.hbase.util.Bytes;
64  import org.junit.AfterClass;
65  import org.junit.Assert;
66  import org.junit.Before;
67  import org.junit.BeforeClass;
68  import org.junit.Test;
69  import org.junit.experimental.categories.Category;
70  
71  @Category(MediumTests.class)
72  public class TestThriftHBaseServiceHandlerWithLabels {
73  
74  public static final Log LOG = LogFactory
75      .getLog(TestThriftHBaseServiceHandlerWithLabels.class);
76  private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
77  
78  // Static names for tables, columns, rows, and values
79  private static byte[] tableAname = Bytes.toBytes("tableA");
80  private static byte[] familyAname = Bytes.toBytes("familyA");
81  private static byte[] familyBname = Bytes.toBytes("familyB");
82  private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
83  private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
84  private static byte[] valueAname = Bytes.toBytes("valueA");
85  private static byte[] valueBname = Bytes.toBytes("valueB");
86  private static HColumnDescriptor[] families = new HColumnDescriptor[] {
87      new HColumnDescriptor(familyAname).setMaxVersions(3),
88      new HColumnDescriptor(familyBname).setMaxVersions(2) };
89  
90  private final static String TOPSECRET = "topsecret";
91  private final static String PUBLIC = "public";
92  private final static String PRIVATE = "private";
93  private final static String CONFIDENTIAL = "confidential";
94  private final static String SECRET = "secret";
95  private static User SUPERUSER;
96  
97  private static Configuration conf;
98  
99  public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
100     List<TColumnValue> columnValuesB) {
101   assertEquals(columnValuesA.size(), columnValuesB.size());
102   Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
103     @Override
104     public int compare(TColumnValue o1, TColumnValue o2) {
105       return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
106           Bytes.add(o2.getFamily(), o2.getQualifier()));
107     }
108   };
109   Collections.sort(columnValuesA, comparator);
110   Collections.sort(columnValuesB, comparator);
111 
112   for (int i = 0; i < columnValuesA.size(); i++) {
113     TColumnValue a = columnValuesA.get(i);
114     TColumnValue b = columnValuesB.get(i);
115     assertArrayEquals(a.getFamily(), b.getFamily());
116     assertArrayEquals(a.getQualifier(), b.getQualifier());
117     assertArrayEquals(a.getValue(), b.getValue());
118   }
119 }
120 
121 @BeforeClass
122 public static void beforeClass() throws Exception {
123   SUPERUSER = User.createUserForTesting(conf, "admin",
124       new String[] { "supergroup" });
125   conf = UTIL.getConfiguration();
126   conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS,
127       SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
128   conf.set("hbase.superuser", SUPERUSER.getShortName());
129   conf.set("hbase.coprocessor.master.classes",
130       VisibilityController.class.getName());
131   conf.set("hbase.coprocessor.region.classes",
132       VisibilityController.class.getName());
133   conf.setInt("hfile.format.version", 3);
134   UTIL.startMiniCluster(1);
135   // Wait for the labels table to become available
136   UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
137   createLabels();
138   HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration());
139   HTableDescriptor tableDescriptor = new HTableDescriptor(
140       TableName.valueOf(tableAname));
141   for (HColumnDescriptor family : families) {
142     tableDescriptor.addFamily(family);
143   }
144   admin.createTable(tableDescriptor);
145   setAuths();
146 }
147 
148 private static void createLabels() throws IOException, InterruptedException {
149   PrivilegedExceptionAction<VisibilityLabelsResponse> action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
150     public VisibilityLabelsResponse run() throws Exception {
151       String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
152       try {
153         VisibilityClient.addLabels(conf, labels);
154       } catch (Throwable t) {
155         throw new IOException(t);
156       }
157       return null;
158     }
159   };
160   SUPERUSER.runAs(action);
161 }
162 
163 private static void setAuths() throws IOException {
164   String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
165   try {
166     VisibilityClient.setAuths(conf, labels, User.getCurrent().getShortName());
167   } catch (Throwable t) {
168     throw new IOException(t);
169   }
170 }
171 
172 @AfterClass
173 public static void afterClass() throws Exception {
174   UTIL.shutdownMiniCluster();
175 }
176 
177 @Before
178 public void setup() throws Exception {
179 
180 }
181 
182 private ThriftHBaseServiceHandler createHandler() {
183   return new ThriftHBaseServiceHandler(UTIL.getConfiguration());
184 }
185 
186 @Test
187 public void testScanWithVisibilityLabels() throws Exception {
188   ThriftHBaseServiceHandler handler = createHandler();
189   ByteBuffer table = wrap(tableAname);
190 
191   // insert data
192   TColumnValue columnValue = new TColumnValue(wrap(familyAname),
193       wrap(qualifierAname), wrap(valueAname));
194   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
195   columnValues.add(columnValue);
196   for (int i = 0; i < 10; i++) {
197     TPut put = new TPut(wrap(("testScan" + i).getBytes()), columnValues);
198     if (i == 5) {
199       put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
200     } else {
201       put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
202           + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
203     }
204     handler.put(table, put);
205   }
206 
207   // create scan instance
208   TScan scan = new TScan();
209   List<TColumn> columns = new ArrayList<TColumn>();
210   TColumn column = new TColumn();
211   column.setFamily(familyAname);
212   column.setQualifier(qualifierAname);
213   columns.add(column);
214   scan.setColumns(columns);
215   scan.setStartRow("testScan".getBytes());
216   scan.setStopRow("testScan\uffff".getBytes());
217 
218   TAuthorization tauth = new TAuthorization();
219   List<String> labels = new ArrayList<String>();
220   labels.add(SECRET);
221   labels.add(PRIVATE);
222   tauth.setLabels(labels);
223   scan.setAuthorizations(tauth);
224   // get scanner and rows
225   int scanId = handler.openScanner(table, scan);
226   List<TResult> results = handler.getScannerRows(scanId, 10);
227   assertEquals(9, results.size());
228   Assert.assertFalse(Bytes.equals(results.get(5).getRow(),
229       ("testScan" + 5).getBytes()));
230   for (int i = 0; i < 9; i++) {
231     if (i < 5) {
232       assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
233     } else if (i == 5) {
234       continue;
235     } else {
236       assertArrayEquals(("testScan" + (i + 1)).getBytes(), results.get(i)
237           .getRow());
238     }
239   }
240 
241   // check that we are at the end of the scan
242   results = handler.getScannerRows(scanId, 9);
243   assertEquals(0, results.size());
244 
245   // close scanner and check that it was indeed closed
246   handler.closeScanner(scanId);
247   try {
248     handler.getScannerRows(scanId, 9);
249     fail("Scanner id should be invalid");
250   } catch (TIllegalArgument e) {
251   }
252 }
253 
254 @Test
255 public void testGetScannerResultsWithAuthorizations() throws Exception {
256   ThriftHBaseServiceHandler handler = createHandler();
257   ByteBuffer table = wrap(tableAname);
258 
259   // insert data
260   TColumnValue columnValue = new TColumnValue(wrap(familyAname),
261       wrap(qualifierAname), wrap(valueAname));
262   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
263   columnValues.add(columnValue);
264   for (int i = 0; i < 20; i++) {
265     TPut put = new TPut(
266         wrap(("testGetScannerResults" + pad(i, (byte) 2)).getBytes()),
267         columnValues);
268     if (i == 3) {
269       put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
270     } else {
271       put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
272           + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
273     }
274     handler.put(table, put);
275   }
276 
277   // create scan instance
278   TScan scan = new TScan();
279   List<TColumn> columns = new ArrayList<TColumn>();
280   TColumn column = new TColumn();
281   column.setFamily(familyAname);
282   column.setQualifier(qualifierAname);
283   columns.add(column);
284   scan.setColumns(columns);
285   scan.setStartRow("testGetScannerResults".getBytes());
286 
287   // get 5 rows and check the returned results
288   scan.setStopRow("testGetScannerResults05".getBytes());
289   TAuthorization tauth = new TAuthorization();
290   List<String> labels = new ArrayList<String>();
291   labels.add(SECRET);
292   labels.add(PRIVATE);
293   tauth.setLabels(labels);
294   scan.setAuthorizations(tauth);
295   List<TResult> results = handler.getScannerResults(table, scan, 5);
296   assertEquals(4, results.size());
297   for (int i = 0; i < 4; i++) {
298     if (i < 3) {
299       assertArrayEquals(
300           ("testGetScannerResults" + pad(i, (byte) 2)).getBytes(),
301           results.get(i).getRow());
302     } else if (i == 3) {
303       continue;
304     } else {
305       assertArrayEquals(
306           ("testGetScannerResults" + pad(i + 1, (byte) 2)).getBytes(), results
307               .get(i).getRow());
308     }
309   }
310 }
311 
312 @Test
313 public void testGetsWithLabels() throws Exception {
314   ThriftHBaseServiceHandler handler = createHandler();
315   byte[] rowName = "testPutGet".getBytes();
316   ByteBuffer table = wrap(tableAname);
317 
318   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
319   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
320       wrap(valueAname)));
321   columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),
322       wrap(valueBname)));
323   TPut put = new TPut(wrap(rowName), columnValues);
324 
325   put.setColumnValues(columnValues);
326   put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET + "|"
327       + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
328   handler.put(table, put);
329   TGet get = new TGet(wrap(rowName));
330   TAuthorization tauth = new TAuthorization();
331   List<String> labels = new ArrayList<String>();
332   labels.add(SECRET);
333   labels.add(PRIVATE);
334   tauth.setLabels(labels);
335   get.setAuthorizations(tauth);
336   TResult result = handler.get(table, get);
337   assertArrayEquals(rowName, result.getRow());
338   List<TColumnValue> returnedColumnValues = result.getColumnValues();
339   assertTColumnValuesEqual(columnValues, returnedColumnValues);
340 }
341 
342 @Test
343 public void testIncrementWithTags() throws Exception {
344   ThriftHBaseServiceHandler handler = createHandler();
345   byte[] rowName = "testIncrementWithTags".getBytes();
346   ByteBuffer table = wrap(tableAname);
347 
348   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
349   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
350       wrap(Bytes.toBytes(1L))));
351   TPut put = new TPut(wrap(rowName), columnValues);
352   put.setColumnValues(columnValues);
353   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
354   handler.put(table, put);
355 
356   List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
357   incrementColumns.add(new TColumnIncrement(wrap(familyAname),
358       wrap(qualifierAname)));
359   TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
360   increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
361   handler.increment(table, increment);
362 
363   TGet get = new TGet(wrap(rowName));
364   TAuthorization tauth = new TAuthorization();
365   List<String> labels = new ArrayList<String>();
366   labels.add(SECRET);
367   tauth.setLabels(labels);
368   get.setAuthorizations(tauth);
369   TResult result = handler.get(table, get);
370 
371   assertArrayEquals(rowName, result.getRow());
372   assertEquals(1, result.getColumnValuesSize());
373   TColumnValue columnValue = result.getColumnValues().get(0);
374   assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
375 }
376 
377 @Test
378 public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
379   ThriftHBaseServiceHandler handler = createHandler();
380   byte[] rowName = "testIncrementWithTagsWithNotMatchLabels".getBytes();
381   ByteBuffer table = wrap(tableAname);
382 
383   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
384   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
385       wrap(Bytes.toBytes(1L))));
386   TPut put = new TPut(wrap(rowName), columnValues);
387   put.setColumnValues(columnValues);
388   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
389   handler.put(table, put);
390 
391   List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
392   incrementColumns.add(new TColumnIncrement(wrap(familyAname),
393       wrap(qualifierAname)));
394   TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
395   increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
396   handler.increment(table, increment);
397 
398   TGet get = new TGet(wrap(rowName));
399   TAuthorization tauth = new TAuthorization();
400   List<String> labels = new ArrayList<String>();
401   labels.add(PUBLIC);
402   tauth.setLabels(labels);
403   get.setAuthorizations(tauth);
404   TResult result = handler.get(table, get);
405   assertNull(result.getRow());
406 }
407 
408 @Test
409 public void testAppend() throws Exception {
410   ThriftHBaseServiceHandler handler = createHandler();
411   byte[] rowName = "testAppend".getBytes();
412   ByteBuffer table = wrap(tableAname);
413   byte[] v1 = Bytes.toBytes(1L);
414   byte[] v2 = Bytes.toBytes(5L);
415   List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
416   columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
417       wrap(Bytes.toBytes(1L))));
418   TPut put = new TPut(wrap(rowName), columnValues);
419   put.setColumnValues(columnValues);
420   put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
421   handler.put(table, put);
422 
423   List<TColumnValue> appendColumns = new ArrayList<TColumnValue>();
424   appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
425       wrap(v2)));
426   TAppend append = new TAppend(wrap(rowName), appendColumns);
427   append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
428   handler.append(table, append);
429 
430   TGet get = new TGet(wrap(rowName));
431   TAuthorization tauth = new TAuthorization();
432   List<String> labels = new ArrayList<String>();
433   labels.add(SECRET);
434   tauth.setLabels(labels);
435   get.setAuthorizations(tauth);
436   TResult result = handler.get(table, get);
437 
438   assertArrayEquals(rowName, result.getRow());
439   assertEquals(1, result.getColumnValuesSize());
440   TColumnValue columnValue = result.getColumnValues().get(0);
441   assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
442 }
443 
444 /**
445  * Padding numbers to make comparison of sort order easier in a for loop
446  * 
447  * @param n
448  *          The number to pad.
449  * @param pad
450  *          The length to pad up to.
451  * @return The padded number as a string.
452  */
453 private String pad(int n, byte pad) {
454   String res = Integer.toString(n);
455   while (res.length() < pad)
456     res = "0" + res;
457   return res;
458 }
459 }