1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.thrift2;
19
20 import static java.nio.ByteBuffer.wrap;
21 import static org.junit.Assert.assertArrayEquals;
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertNull;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.nio.ByteBuffer;
28 import java.security.PrivilegedExceptionAction;
29 import java.util.ArrayList;
30 import java.util.Collections;
31 import java.util.Comparator;
32 import java.util.List;
33
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.hbase.HBaseTestingUtility;
38 import org.apache.hadoop.hbase.HColumnDescriptor;
39 import org.apache.hadoop.hbase.HTableDescriptor;
40 import org.apache.hadoop.hbase.MediumTests;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.client.HBaseAdmin;
43 import org.apache.hadoop.hbase.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
44 import org.apache.hadoop.hbase.security.User;
45 import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
46 import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
47 import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
48 import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
49 import org.apache.hadoop.hbase.security.visibility.VisibilityController;
50 import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
51 import org.apache.hadoop.hbase.thrift2.generated.TAppend;
52 import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
53 import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
54 import org.apache.hadoop.hbase.thrift2.generated.TColumn;
55 import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
56 import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
57 import org.apache.hadoop.hbase.thrift2.generated.TGet;
58 import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
59 import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
60 import org.apache.hadoop.hbase.thrift2.generated.TPut;
61 import org.apache.hadoop.hbase.thrift2.generated.TResult;
62 import org.apache.hadoop.hbase.thrift2.generated.TScan;
63 import org.apache.hadoop.hbase.util.Bytes;
64 import org.junit.AfterClass;
65 import org.junit.Assert;
66 import org.junit.Before;
67 import org.junit.BeforeClass;
68 import org.junit.Test;
69 import org.junit.experimental.categories.Category;
70
71 @Category(MediumTests.class)
72 public class TestThriftHBaseServiceHandlerWithLabels {
73
74 public static final Log LOG = LogFactory
75 .getLog(TestThriftHBaseServiceHandlerWithLabels.class);
76 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
77
78
79 private static byte[] tableAname = Bytes.toBytes("tableA");
80 private static byte[] familyAname = Bytes.toBytes("familyA");
81 private static byte[] familyBname = Bytes.toBytes("familyB");
82 private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
83 private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
84 private static byte[] valueAname = Bytes.toBytes("valueA");
85 private static byte[] valueBname = Bytes.toBytes("valueB");
86 private static HColumnDescriptor[] families = new HColumnDescriptor[] {
87 new HColumnDescriptor(familyAname).setMaxVersions(3),
88 new HColumnDescriptor(familyBname).setMaxVersions(2) };
89
90 private final static String TOPSECRET = "topsecret";
91 private final static String PUBLIC = "public";
92 private final static String PRIVATE = "private";
93 private final static String CONFIDENTIAL = "confidential";
94 private final static String SECRET = "secret";
95 private static User SUPERUSER;
96
97 private static Configuration conf;
98
99 public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
100 List<TColumnValue> columnValuesB) {
101 assertEquals(columnValuesA.size(), columnValuesB.size());
102 Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
103 @Override
104 public int compare(TColumnValue o1, TColumnValue o2) {
105 return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
106 Bytes.add(o2.getFamily(), o2.getQualifier()));
107 }
108 };
109 Collections.sort(columnValuesA, comparator);
110 Collections.sort(columnValuesB, comparator);
111
112 for (int i = 0; i < columnValuesA.size(); i++) {
113 TColumnValue a = columnValuesA.get(i);
114 TColumnValue b = columnValuesB.get(i);
115 assertArrayEquals(a.getFamily(), b.getFamily());
116 assertArrayEquals(a.getQualifier(), b.getQualifier());
117 assertArrayEquals(a.getValue(), b.getValue());
118 }
119 }
120
121 @BeforeClass
122 public static void beforeClass() throws Exception {
123 SUPERUSER = User.createUserForTesting(conf, "admin",
124 new String[] { "supergroup" });
125 conf = UTIL.getConfiguration();
126 conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS,
127 SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
128 conf.set("hbase.superuser", SUPERUSER.getShortName());
129 conf.set("hbase.coprocessor.master.classes",
130 VisibilityController.class.getName());
131 conf.set("hbase.coprocessor.region.classes",
132 VisibilityController.class.getName());
133 conf.setInt("hfile.format.version", 3);
134 UTIL.startMiniCluster(1);
135
136 UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
137 createLabels();
138 HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration());
139 HTableDescriptor tableDescriptor = new HTableDescriptor(
140 TableName.valueOf(tableAname));
141 for (HColumnDescriptor family : families) {
142 tableDescriptor.addFamily(family);
143 }
144 admin.createTable(tableDescriptor);
145 setAuths();
146 }
147
148 private static void createLabels() throws IOException, InterruptedException {
149 PrivilegedExceptionAction<VisibilityLabelsResponse> action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
150 public VisibilityLabelsResponse run() throws Exception {
151 String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
152 try {
153 VisibilityClient.addLabels(conf, labels);
154 } catch (Throwable t) {
155 throw new IOException(t);
156 }
157 return null;
158 }
159 };
160 SUPERUSER.runAs(action);
161 }
162
163 private static void setAuths() throws IOException {
164 String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
165 try {
166 VisibilityClient.setAuths(conf, labels, User.getCurrent().getShortName());
167 } catch (Throwable t) {
168 throw new IOException(t);
169 }
170 }
171
172 @AfterClass
173 public static void afterClass() throws Exception {
174 UTIL.shutdownMiniCluster();
175 }
176
177 @Before
178 public void setup() throws Exception {
179
180 }
181
182 private ThriftHBaseServiceHandler createHandler() {
183 return new ThriftHBaseServiceHandler(UTIL.getConfiguration());
184 }
185
186 @Test
187 public void testScanWithVisibilityLabels() throws Exception {
188 ThriftHBaseServiceHandler handler = createHandler();
189 ByteBuffer table = wrap(tableAname);
190
191
192 TColumnValue columnValue = new TColumnValue(wrap(familyAname),
193 wrap(qualifierAname), wrap(valueAname));
194 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
195 columnValues.add(columnValue);
196 for (int i = 0; i < 10; i++) {
197 TPut put = new TPut(wrap(("testScan" + i).getBytes()), columnValues);
198 if (i == 5) {
199 put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
200 } else {
201 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
202 + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
203 }
204 handler.put(table, put);
205 }
206
207
208 TScan scan = new TScan();
209 List<TColumn> columns = new ArrayList<TColumn>();
210 TColumn column = new TColumn();
211 column.setFamily(familyAname);
212 column.setQualifier(qualifierAname);
213 columns.add(column);
214 scan.setColumns(columns);
215 scan.setStartRow("testScan".getBytes());
216 scan.setStopRow("testScan\uffff".getBytes());
217
218 TAuthorization tauth = new TAuthorization();
219 List<String> labels = new ArrayList<String>();
220 labels.add(SECRET);
221 labels.add(PRIVATE);
222 tauth.setLabels(labels);
223 scan.setAuthorizations(tauth);
224
225 int scanId = handler.openScanner(table, scan);
226 List<TResult> results = handler.getScannerRows(scanId, 10);
227 assertEquals(9, results.size());
228 Assert.assertFalse(Bytes.equals(results.get(5).getRow(),
229 ("testScan" + 5).getBytes()));
230 for (int i = 0; i < 9; i++) {
231 if (i < 5) {
232 assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
233 } else if (i == 5) {
234 continue;
235 } else {
236 assertArrayEquals(("testScan" + (i + 1)).getBytes(), results.get(i)
237 .getRow());
238 }
239 }
240
241
242 results = handler.getScannerRows(scanId, 9);
243 assertEquals(0, results.size());
244
245
246 handler.closeScanner(scanId);
247 try {
248 handler.getScannerRows(scanId, 9);
249 fail("Scanner id should be invalid");
250 } catch (TIllegalArgument e) {
251 }
252 }
253
254 @Test
255 public void testGetScannerResultsWithAuthorizations() throws Exception {
256 ThriftHBaseServiceHandler handler = createHandler();
257 ByteBuffer table = wrap(tableAname);
258
259
260 TColumnValue columnValue = new TColumnValue(wrap(familyAname),
261 wrap(qualifierAname), wrap(valueAname));
262 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
263 columnValues.add(columnValue);
264 for (int i = 0; i < 20; i++) {
265 TPut put = new TPut(
266 wrap(("testGetScannerResults" + pad(i, (byte) 2)).getBytes()),
267 columnValues);
268 if (i == 3) {
269 put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
270 } else {
271 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
272 + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
273 }
274 handler.put(table, put);
275 }
276
277
278 TScan scan = new TScan();
279 List<TColumn> columns = new ArrayList<TColumn>();
280 TColumn column = new TColumn();
281 column.setFamily(familyAname);
282 column.setQualifier(qualifierAname);
283 columns.add(column);
284 scan.setColumns(columns);
285 scan.setStartRow("testGetScannerResults".getBytes());
286
287
288 scan.setStopRow("testGetScannerResults05".getBytes());
289 TAuthorization tauth = new TAuthorization();
290 List<String> labels = new ArrayList<String>();
291 labels.add(SECRET);
292 labels.add(PRIVATE);
293 tauth.setLabels(labels);
294 scan.setAuthorizations(tauth);
295 List<TResult> results = handler.getScannerResults(table, scan, 5);
296 assertEquals(4, results.size());
297 for (int i = 0; i < 4; i++) {
298 if (i < 3) {
299 assertArrayEquals(
300 ("testGetScannerResults" + pad(i, (byte) 2)).getBytes(),
301 results.get(i).getRow());
302 } else if (i == 3) {
303 continue;
304 } else {
305 assertArrayEquals(
306 ("testGetScannerResults" + pad(i + 1, (byte) 2)).getBytes(), results
307 .get(i).getRow());
308 }
309 }
310 }
311
312 @Test
313 public void testGetsWithLabels() throws Exception {
314 ThriftHBaseServiceHandler handler = createHandler();
315 byte[] rowName = "testPutGet".getBytes();
316 ByteBuffer table = wrap(tableAname);
317
318 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
319 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
320 wrap(valueAname)));
321 columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),
322 wrap(valueBname)));
323 TPut put = new TPut(wrap(rowName), columnValues);
324
325 put.setColumnValues(columnValues);
326 put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET + "|"
327 + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
328 handler.put(table, put);
329 TGet get = new TGet(wrap(rowName));
330 TAuthorization tauth = new TAuthorization();
331 List<String> labels = new ArrayList<String>();
332 labels.add(SECRET);
333 labels.add(PRIVATE);
334 tauth.setLabels(labels);
335 get.setAuthorizations(tauth);
336 TResult result = handler.get(table, get);
337 assertArrayEquals(rowName, result.getRow());
338 List<TColumnValue> returnedColumnValues = result.getColumnValues();
339 assertTColumnValuesEqual(columnValues, returnedColumnValues);
340 }
341
342 @Test
343 public void testIncrementWithTags() throws Exception {
344 ThriftHBaseServiceHandler handler = createHandler();
345 byte[] rowName = "testIncrementWithTags".getBytes();
346 ByteBuffer table = wrap(tableAname);
347
348 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
349 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
350 wrap(Bytes.toBytes(1L))));
351 TPut put = new TPut(wrap(rowName), columnValues);
352 put.setColumnValues(columnValues);
353 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
354 handler.put(table, put);
355
356 List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
357 incrementColumns.add(new TColumnIncrement(wrap(familyAname),
358 wrap(qualifierAname)));
359 TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
360 increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
361 handler.increment(table, increment);
362
363 TGet get = new TGet(wrap(rowName));
364 TAuthorization tauth = new TAuthorization();
365 List<String> labels = new ArrayList<String>();
366 labels.add(SECRET);
367 tauth.setLabels(labels);
368 get.setAuthorizations(tauth);
369 TResult result = handler.get(table, get);
370
371 assertArrayEquals(rowName, result.getRow());
372 assertEquals(1, result.getColumnValuesSize());
373 TColumnValue columnValue = result.getColumnValues().get(0);
374 assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
375 }
376
377 @Test
378 public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
379 ThriftHBaseServiceHandler handler = createHandler();
380 byte[] rowName = "testIncrementWithTagsWithNotMatchLabels".getBytes();
381 ByteBuffer table = wrap(tableAname);
382
383 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
384 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
385 wrap(Bytes.toBytes(1L))));
386 TPut put = new TPut(wrap(rowName), columnValues);
387 put.setColumnValues(columnValues);
388 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
389 handler.put(table, put);
390
391 List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
392 incrementColumns.add(new TColumnIncrement(wrap(familyAname),
393 wrap(qualifierAname)));
394 TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
395 increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
396 handler.increment(table, increment);
397
398 TGet get = new TGet(wrap(rowName));
399 TAuthorization tauth = new TAuthorization();
400 List<String> labels = new ArrayList<String>();
401 labels.add(PUBLIC);
402 tauth.setLabels(labels);
403 get.setAuthorizations(tauth);
404 TResult result = handler.get(table, get);
405 assertNull(result.getRow());
406 }
407
408 @Test
409 public void testAppend() throws Exception {
410 ThriftHBaseServiceHandler handler = createHandler();
411 byte[] rowName = "testAppend".getBytes();
412 ByteBuffer table = wrap(tableAname);
413 byte[] v1 = Bytes.toBytes(1L);
414 byte[] v2 = Bytes.toBytes(5L);
415 List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
416 columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
417 wrap(Bytes.toBytes(1L))));
418 TPut put = new TPut(wrap(rowName), columnValues);
419 put.setColumnValues(columnValues);
420 put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
421 handler.put(table, put);
422
423 List<TColumnValue> appendColumns = new ArrayList<TColumnValue>();
424 appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
425 wrap(v2)));
426 TAppend append = new TAppend(wrap(rowName), appendColumns);
427 append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
428 handler.append(table, append);
429
430 TGet get = new TGet(wrap(rowName));
431 TAuthorization tauth = new TAuthorization();
432 List<String> labels = new ArrayList<String>();
433 labels.add(SECRET);
434 tauth.setLabels(labels);
435 get.setAuthorizations(tauth);
436 TResult result = handler.get(table, get);
437
438 assertArrayEquals(rowName, result.getRow());
439 assertEquals(1, result.getColumnValuesSize());
440 TColumnValue columnValue = result.getColumnValues().get(0);
441 assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
442 }
443
444
445
446
447
448
449
450
451
452
453 private String pad(int n, byte pad) {
454 String res = Integer.toString(n);
455 while (res.length() < pad)
456 res = "0" + res;
457 return res;
458 }
459 }