1   /*
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.thrift2;
21  
22  import java.io.IOException;
23  import java.nio.ByteBuffer;
24  import java.util.ArrayList;
25  import java.util.Collections;
26  import java.util.Comparator;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.HashMap;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.conf.Configuration;
34  import org.apache.hadoop.hbase.HBaseTestingUtility;
35  import org.apache.hadoop.hbase.HColumnDescriptor;
36  import org.apache.hadoop.hbase.HTableDescriptor;
37  import org.apache.hadoop.hbase.MediumTests;
38  import org.apache.hadoop.hbase.client.HBaseAdmin;
39  import org.apache.hadoop.hbase.client.Get;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.client.Scan;
42  import org.apache.hadoop.hbase.client.Delete;
43  import org.apache.hadoop.hbase.client.Durability;
44  import org.apache.hadoop.hbase.filter.ParseFilter;
45  import org.apache.hadoop.hbase.thrift.ThriftMetrics;
46  import org.apache.hadoop.hbase.thrift2.generated.TColumn;
47  import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
48  import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
49  import org.apache.hadoop.hbase.thrift2.generated.TDelete;
50  import org.apache.hadoop.hbase.thrift2.generated.TDeleteType;
51  import org.apache.hadoop.hbase.thrift2.generated.TGet;
52  import org.apache.hadoop.hbase.thrift2.generated.THBaseService;
53  import org.apache.hadoop.hbase.thrift2.generated.TIOError;
54  import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
55  import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
56  import org.apache.hadoop.hbase.thrift2.generated.TPut;
57  import org.apache.hadoop.hbase.thrift2.generated.TResult;
58  import org.apache.hadoop.hbase.thrift2.generated.TScan;
59  import org.apache.hadoop.hbase.thrift2.generated.TMutation;
60  import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
61  import org.apache.hadoop.hbase.thrift2.generated.TDurability;
62  import org.apache.hadoop.hbase.util.Bytes;
63  import org.apache.hadoop.metrics.ContextFactory;
64  import org.apache.hadoop.metrics.MetricsContext;
65  import org.apache.hadoop.metrics.MetricsUtil;
66  import org.apache.hadoop.metrics.spi.NoEmitMetricsContext;
67  import org.apache.hadoop.metrics.spi.OutputRecord;
68  import org.apache.thrift.TException;
69  import org.junit.AfterClass;
70  import org.junit.Before;
71  import org.junit.BeforeClass;
72  import org.junit.Test;
73  import org.junit.experimental.categories.Category;
74  
75  import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.getFromThrift;
76  import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.putFromThrift;
77  import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.scanFromThrift;
78  import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.deleteFromThrift;
79  import static org.junit.Assert.*;
80  import static java.nio.ByteBuffer.wrap;
81  
82  /**
83   * Unit testing for ThriftServer.HBaseHandler, a part of the org.apache.hadoop.hbase.thrift2 
84   * package.
85   */
86  @Category(MediumTests.class)
87  public class TestThriftHBaseServiceHandler {
88  
89    public static final Log LOG = LogFactory.getLog(TestThriftHBaseServiceHandler.class);
90    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
91  
92    // Static names for tables, columns, rows, and values
93    private static byte[] tableAname = Bytes.toBytes("tableA");
94    private static byte[] familyAname = Bytes.toBytes("familyA");
95    private static byte[] familyBname = Bytes.toBytes("familyB");
96    private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
97    private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
98    private static byte[] valueAname = Bytes.toBytes("valueA");
99    private static byte[] valueBname = Bytes.toBytes("valueB");
100   private static HColumnDescriptor[] families = new HColumnDescriptor[] {
101       new HColumnDescriptor(familyAname),
102       new HColumnDescriptor(familyBname)
103           .setMaxVersions(2)
104   };
105 
106   public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
107       List<TColumnValue> columnValuesB) {
108     assertEquals(columnValuesA.size(), columnValuesB.size());
109     Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
110       @Override
111       public int compare(TColumnValue o1, TColumnValue o2) {
112         return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
113             Bytes.add(o2.getFamily(), o2.getQualifier()));
114       }
115     };
116     Collections.sort(columnValuesA, comparator);
117     Collections.sort(columnValuesB, comparator);
118 
119     for (int i = 0; i < columnValuesA.size(); i++) {
120       TColumnValue a = columnValuesA.get(i);
121       TColumnValue b = columnValuesB.get(i);
122       assertArrayEquals(a.getFamily(), b.getFamily());
123       assertArrayEquals(a.getQualifier(), b.getQualifier());
124       assertArrayEquals(a.getValue(), b.getValue());
125     }
126   }
127 
128   @BeforeClass
129   public static void beforeClass() throws Exception {
130     UTIL.startMiniCluster();
131     HBaseAdmin admin = new HBaseAdmin(UTIL.getConfiguration());
132     HTableDescriptor tableDescriptor = new HTableDescriptor(tableAname);
133     for (HColumnDescriptor family : families) {
134       tableDescriptor.addFamily(family);
135     }
136     admin.createTable(tableDescriptor);
137   }
138 
139   @AfterClass
140   public static void afterClass() throws Exception {
141     UTIL.shutdownMiniCluster();
142   }
143 
144   @Before
145   public void setup() throws Exception {
146 
147   }
148 
149   private ThriftHBaseServiceHandler createHandler() {
150     return new ThriftHBaseServiceHandler(UTIL.getConfiguration());
151   }
152 
153   @Test
154   public void testExists() throws TIOError, TException {
155     ThriftHBaseServiceHandler handler = createHandler();
156     byte[] rowName = "testExists".getBytes();
157     ByteBuffer table = wrap(tableAname);
158 
159     TGet get = new TGet(wrap(rowName));
160     assertFalse(handler.exists(table, get));
161 
162     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
163     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
164     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
165     TPut put = new TPut(wrap(rowName), columnValues);
166     put.setColumnValues(columnValues);
167 
168     handler.put(table, put);
169 
170     assertTrue(handler.exists(table, get));
171   }
172 
173   @Test
174   public void testPutGet() throws Exception {
175     ThriftHBaseServiceHandler handler = createHandler();
176     byte[] rowName = "testPutGet".getBytes();
177     ByteBuffer table = wrap(tableAname);
178 
179     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
180     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
181     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
182     TPut put = new TPut(wrap(rowName), columnValues);
183 
184     put.setColumnValues(columnValues);
185 
186     handler.put(table, put);
187 
188     TGet get = new TGet(wrap(rowName));
189 
190     TResult result = handler.get(table, get);
191     assertArrayEquals(rowName, result.getRow());
192     List<TColumnValue> returnedColumnValues = result.getColumnValues();
193     assertTColumnValuesEqual(columnValues, returnedColumnValues);
194   }
195 
196   @Test
197   public void testPutGetMultiple() throws Exception {
198     ThriftHBaseServiceHandler handler = createHandler();
199     ByteBuffer table = wrap(tableAname);
200     byte[] rowName1 = "testPutGetMultiple1".getBytes();
201     byte[] rowName2 = "testPutGetMultiple2".getBytes();
202 
203     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
204     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
205     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
206     List<TPut> puts = new ArrayList<TPut>();
207     puts.add(new TPut(wrap(rowName1), columnValues));
208     puts.add(new TPut(wrap(rowName2), columnValues));
209 
210     handler.putMultiple(table, puts);
211 
212     List<TGet> gets = new ArrayList<TGet>();
213     gets.add(new TGet(wrap(rowName1)));
214     gets.add(new TGet(wrap(rowName2)));
215 
216     List<TResult> results = handler.getMultiple(table, gets);
217     assertEquals(2, results.size());
218 
219     assertArrayEquals(rowName1, results.get(0).getRow());
220     assertTColumnValuesEqual(columnValues, results.get(0).getColumnValues());
221 
222     assertArrayEquals(rowName2, results.get(1).getRow());
223     assertTColumnValuesEqual(columnValues, results.get(1).getColumnValues());
224   }
225 
226   @Test
227   public void testDeleteMultiple() throws Exception {
228     ThriftHBaseServiceHandler handler = createHandler();
229     ByteBuffer table = wrap(tableAname);
230     byte[] rowName1 = "testDeleteMultiple1".getBytes();
231     byte[] rowName2 = "testDeleteMultiple2".getBytes();
232 
233     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
234     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
235     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
236     List<TPut> puts = new ArrayList<TPut>();
237     puts.add(new TPut(wrap(rowName1), columnValues));
238     puts.add(new TPut(wrap(rowName2), columnValues));
239 
240     handler.putMultiple(table, puts);
241 
242     List<TDelete> deletes = new ArrayList<TDelete>();
243     deletes.add(new TDelete(wrap(rowName1)));
244     deletes.add(new TDelete(wrap(rowName2)));
245 
246     List<TDelete> deleteResults = handler.deleteMultiple(table, deletes);
247     // 0 means they were all successfully applies
248     assertEquals(0, deleteResults.size());
249 
250     assertFalse(handler.exists(table, new TGet(wrap(rowName1))));
251     assertFalse(handler.exists(table, new TGet(wrap(rowName2))));
252   }
253 
254   @Test
255   public void testDelete() throws Exception {
256     ThriftHBaseServiceHandler handler = createHandler();
257     byte[] rowName = "testDelete".getBytes();
258     ByteBuffer table = wrap(tableAname);
259 
260     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
261     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
262       wrap(valueAname));
263     TColumnValue columnValueB = new TColumnValue(wrap(familyBname), wrap(qualifierBname),
264       wrap(valueBname));
265     columnValues.add(columnValueA);
266     columnValues.add(columnValueB);
267     TPut put = new TPut(wrap(rowName), columnValues);
268 
269     put.setColumnValues(columnValues);
270 
271     handler.put(table, put);
272 
273     TDelete delete = new TDelete(wrap(rowName));
274     List<TColumn> deleteColumns = new ArrayList<TColumn>();
275     TColumn deleteColumn = new TColumn(wrap(familyAname));
276     deleteColumn.setQualifier(qualifierAname);
277     deleteColumns.add(deleteColumn);
278     delete.setColumns(deleteColumns);
279 
280     handler.deleteSingle(table, delete);
281 
282     TGet get = new TGet(wrap(rowName));
283     TResult result = handler.get(table, get);
284     assertArrayEquals(rowName, result.getRow());
285     List<TColumnValue> returnedColumnValues = result.getColumnValues();
286     List<TColumnValue> expectedColumnValues = new ArrayList<TColumnValue>();
287     expectedColumnValues.add(columnValueB);
288     assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
289   }
290 
291   @Test
292   public void testDeleteAllTimestamps() throws Exception {
293     ThriftHBaseServiceHandler handler = createHandler();
294     byte[] rowName = "testDeleteAllTimestamps".getBytes();
295     ByteBuffer table = wrap(tableAname);
296 
297     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
298     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
299       wrap(valueAname));
300     columnValueA.setTimestamp(System.currentTimeMillis() - 10);
301     columnValues.add(columnValueA);
302     TPut put = new TPut(wrap(rowName), columnValues);
303 
304     put.setColumnValues(columnValues);
305 
306     handler.put(table, put);
307     columnValueA.setTimestamp(System.currentTimeMillis());
308     handler.put(table, put);
309 
310     TGet get = new TGet(wrap(rowName));
311     get.setMaxVersions(2);
312     TResult result = handler.get(table, get);
313     assertEquals(2, result.getColumnValuesSize());
314 
315     TDelete delete = new TDelete(wrap(rowName));
316     List<TColumn> deleteColumns = new ArrayList<TColumn>();
317     TColumn deleteColumn = new TColumn(wrap(familyAname));
318     deleteColumn.setQualifier(qualifierAname);
319     deleteColumns.add(deleteColumn);
320     delete.setColumns(deleteColumns);
321     delete.setDeleteType(TDeleteType.DELETE_COLUMNS); // This is the default anyway.
322 
323     handler.deleteSingle(table, delete);
324 
325     get = new TGet(wrap(rowName));
326     result = handler.get(table, get);
327     assertNull(result.getRow());
328     assertEquals(0, result.getColumnValuesSize());
329   }
330 
331   @Test
332   public void testDeleteSingleTimestamp() throws Exception {
333     ThriftHBaseServiceHandler handler = createHandler();
334     byte[] rowName = "testDeleteSingleTimestamp".getBytes();
335     ByteBuffer table = wrap(tableAname);
336 
337     long timestamp1 = System.currentTimeMillis() - 10;
338     long timestamp2 = System.currentTimeMillis();
339     
340     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
341     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
342       wrap(valueAname));
343     columnValueA.setTimestamp(timestamp1);
344     columnValues.add(columnValueA);
345     TPut put = new TPut(wrap(rowName), columnValues);
346 
347     put.setColumnValues(columnValues);
348 
349     handler.put(table, put);
350     columnValueA.setTimestamp(timestamp2);
351     handler.put(table, put);
352 
353     TGet get = new TGet(wrap(rowName));
354     get.setMaxVersions(2);
355     TResult result = handler.get(table, get);
356     assertEquals(2, result.getColumnValuesSize());
357 
358     TDelete delete = new TDelete(wrap(rowName));
359     List<TColumn> deleteColumns = new ArrayList<TColumn>();
360     TColumn deleteColumn = new TColumn(wrap(familyAname));
361     deleteColumn.setQualifier(qualifierAname);
362     deleteColumns.add(deleteColumn);
363     delete.setColumns(deleteColumns);
364     delete.setDeleteType(TDeleteType.DELETE_COLUMN);
365 
366     handler.deleteSingle(table, delete);
367 
368     get = new TGet(wrap(rowName));
369     result = handler.get(table, get);
370     assertArrayEquals(rowName, result.getRow());
371     assertEquals(1, result.getColumnValuesSize());
372     // the older timestamp should remain.
373     assertEquals(timestamp1, result.getColumnValues().get(0).getTimestamp());
374   }
375 
376   @Test
377   public void testIncrement() throws Exception {
378     ThriftHBaseServiceHandler handler = createHandler();
379     byte[] rowName = "testIncrement".getBytes();
380     ByteBuffer table = wrap(tableAname);
381 
382     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
383     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
384       wrap(Bytes.toBytes(1L))));
385     TPut put = new TPut(wrap(rowName), columnValues);
386     put.setColumnValues(columnValues);
387     handler.put(table, put);
388 
389     List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
390     incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
391     TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
392     handler.increment(table, increment);
393 
394     TGet get = new TGet(wrap(rowName));
395     TResult result = handler.get(table, get);
396 
397     assertArrayEquals(rowName, result.getRow());
398     assertEquals(1, result.getColumnValuesSize());
399     TColumnValue columnValue = result.getColumnValues().get(0);
400     assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
401   }
402 
403   /**
404    * check that checkAndPut fails if the cell does not exist, then put in the cell, then check
405    * that the checkAndPut succeeds.
406    * 
407    * @throws Exception
408    */
409   @Test
410   public void testCheckAndPut() throws Exception {
411     ThriftHBaseServiceHandler handler = createHandler();
412     byte[] rowName = "testCheckAndPut".getBytes();
413     ByteBuffer table = wrap(tableAname);
414 
415     List<TColumnValue> columnValuesA = new ArrayList<TColumnValue>();
416     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
417       wrap(valueAname));
418     columnValuesA.add(columnValueA);
419     TPut putA = new TPut(wrap(rowName), columnValuesA);
420     putA.setColumnValues(columnValuesA);
421 
422     List<TColumnValue> columnValuesB = new ArrayList<TColumnValue>();
423     TColumnValue columnValueB = new TColumnValue(wrap(familyBname), wrap(qualifierBname),
424       wrap(valueBname));
425     columnValuesB.add(columnValueB);
426     TPut putB = new TPut(wrap(rowName), columnValuesB);
427     putB.setColumnValues(columnValuesB);
428 
429     assertFalse(handler.checkAndPut(table, wrap(rowName), wrap(familyAname),
430       wrap(qualifierAname), wrap(valueAname), putB));
431 
432     TGet get = new TGet(wrap(rowName));
433     TResult result = handler.get(table, get);
434     assertEquals(0, result.getColumnValuesSize());
435 
436     handler.put(table, putA);
437 
438     assertTrue(handler.checkAndPut(table, wrap(rowName), wrap(familyAname),
439       wrap(qualifierAname), wrap(valueAname), putB));
440 
441     result = handler.get(table, get);
442     assertArrayEquals(rowName, result.getRow());
443     List<TColumnValue> returnedColumnValues = result.getColumnValues();
444     List<TColumnValue> expectedColumnValues = new ArrayList<TColumnValue>();
445     expectedColumnValues.add(columnValueA);
446     expectedColumnValues.add(columnValueB);
447     assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
448   }
449 
450   /**
451    * check that checkAndDelete fails if the cell does not exist, then put in the cell, then
452    * check that the checkAndDelete succeeds.
453    * 
454    * @throws Exception
455    */
456   @Test
457   public void testCheckAndDelete() throws Exception {
458     ThriftHBaseServiceHandler handler = createHandler();
459     byte[] rowName = "testCheckAndDelete".getBytes();
460     ByteBuffer table = wrap(tableAname);
461 
462     List<TColumnValue> columnValuesA = new ArrayList<TColumnValue>();
463     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
464       wrap(valueAname));
465     columnValuesA.add(columnValueA);
466     TPut putA = new TPut(wrap(rowName), columnValuesA);
467     putA.setColumnValues(columnValuesA);
468 
469     List<TColumnValue> columnValuesB = new ArrayList<TColumnValue>();
470     TColumnValue columnValueB = new TColumnValue(wrap(familyBname), wrap(qualifierBname),
471       wrap(valueBname));
472     columnValuesB.add(columnValueB);
473     TPut putB = new TPut(wrap(rowName), columnValuesB);
474     putB.setColumnValues(columnValuesB);
475 
476     // put putB so that we know whether the row has been deleted or not
477     handler.put(table, putB);
478 
479     TDelete delete = new TDelete(wrap(rowName));
480 
481     assertFalse(handler.checkAndDelete(table, wrap(rowName), wrap(familyAname),
482       wrap(qualifierAname), wrap(valueAname), delete));
483 
484     TGet get = new TGet(wrap(rowName));
485     TResult result = handler.get(table, get);
486     assertArrayEquals(rowName, result.getRow());
487     assertTColumnValuesEqual(columnValuesB, result.getColumnValues());
488 
489     handler.put(table, putA);
490 
491     assertTrue(handler.checkAndDelete(table, wrap(rowName), wrap(familyAname),
492       wrap(qualifierAname), wrap(valueAname), delete));
493 
494     result = handler.get(table, get);
495     assertFalse(result.isSetRow());
496     assertEquals(0, result.getColumnValuesSize());
497   }
498 
499   @Test
500   public void testScan() throws Exception {
501     ThriftHBaseServiceHandler handler = createHandler();
502     ByteBuffer table = wrap(tableAname);
503 
504     // insert data
505     TColumnValue columnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
506       wrap(valueAname));
507     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
508     columnValues.add(columnValue);
509     for (int i = 0; i < 10; i++) {
510       TPut put = new TPut(wrap(("testScan" + i).getBytes()), columnValues);
511       handler.put(table, put);
512     }
513 
514     // create scan instance
515     TScan scan = new TScan();
516     List<TColumn> columns = new ArrayList<TColumn>();
517     TColumn column = new TColumn();
518     column.setFamily(familyAname);
519     column.setQualifier(qualifierAname);
520     columns.add(column);
521     scan.setColumns(columns);
522     scan.setStartRow("testScan".getBytes());
523     scan.setStopRow("testScan\uffff".getBytes());
524 
525     // get scanner and rows
526     int scanId = handler.openScanner(table, scan);
527     List<TResult> results = handler.getScannerRows(scanId, 10);
528     assertEquals(10, results.size());
529     for (int i = 0; i < 10; i++) {
530       // check if the rows are returned and in order
531       assertArrayEquals(("testScan" + i).getBytes(), results.get(i).getRow());
532     }
533 
534     // check that we are at the end of the scan
535     results = handler.getScannerRows(scanId, 10);
536     assertEquals(0, results.size());
537 
538     // close scanner and check that it was indeed closed
539     handler.closeScanner(scanId);
540     try {
541       handler.getScannerRows(scanId, 10);
542       fail("Scanner id should be invalid");
543     } catch (TIllegalArgument e) {
544     }
545   }
546 
547   @Test
548   public void testScanWithFilter() throws Exception {
549     ThriftHBaseServiceHandler handler = createHandler();
550     ByteBuffer table = wrap(tableAname);
551 
552     // insert data
553     TColumnValue columnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
554       wrap(valueAname));
555     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
556     columnValues.add(columnValue);
557     for (int i = 0; i < 10; i++) {
558       TPut put = new TPut(wrap(("testScanWithFilter" + i).getBytes()), columnValues);
559       handler.put(table, put);
560     }
561 
562     // create scan instance with filter
563     TScan scan = new TScan();
564     List<TColumn> columns = new ArrayList<TColumn>();
565     TColumn column = new TColumn();
566     column.setFamily(familyAname);
567     column.setQualifier(qualifierAname);
568     columns.add(column);
569     scan.setColumns(columns);
570     scan.setStartRow("testScanWithFilter".getBytes());
571     scan.setStopRow("testScanWithFilter\uffff".getBytes());
572     // only get the key part
573     scan.setFilterString(wrap(("KeyOnlyFilter()").getBytes()));
574 
575     // get scanner and rows
576     int scanId = handler.openScanner(table, scan);
577     List<TResult> results = handler.getScannerRows(scanId, 10);
578     assertEquals(10, results.size());
579     for (int i = 0; i < 10; i++) {
580       // check if the rows are returned and in order
581       assertArrayEquals(("testScanWithFilter" + i).getBytes(), results.get(i).getRow());
582       // check that the value is indeed stripped by the filter
583       assertEquals(0, results.get(i).getColumnValues().get(0).getValue().length);
584     }
585 
586     // check that we are at the end of the scan
587     results = handler.getScannerRows(scanId, 10);
588     assertEquals(0, results.size());
589 
590     // close scanner and check that it was indeed closed
591     handler.closeScanner(scanId);
592     try {
593       handler.getScannerRows(scanId, 10);
594       fail("Scanner id should be invalid");
595     } catch (TIllegalArgument e) {
596     }
597   }
598 
599   /**
600    * Padding numbers to make comparison of sort order easier in a for loop
601    *
602    * @param n  The number to pad.
603    * @param pad  The length to pad up to.
604    * @return The padded number as a string.
605    */
606   private String pad(int n, byte pad) {
607     String res = Integer.toString(n);
608     while (res.length() < pad) res = "0" + res;
609     return res;
610   }
611 
612   @Test
613   public void testScanWithBatchSize() throws Exception {
614     ThriftHBaseServiceHandler handler = createHandler();
615     ByteBuffer table = wrap(tableAname);
616 
617     // insert data
618     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
619     for (int i = 0; i < 100; i++) {
620       String colNum = pad(i, (byte) 3);
621       TColumnValue columnValue = new TColumnValue(wrap(familyAname),
622         wrap(("col" + colNum).getBytes()), wrap(("val" + colNum).getBytes()));
623       columnValues.add(columnValue);
624     }
625     TPut put = new TPut(wrap(("testScanWithBatchSize").getBytes()), columnValues);
626     handler.put(table, put);
627 
628     // create scan instance
629     TScan scan = new TScan();
630     List<TColumn> columns = new ArrayList<TColumn>();
631     TColumn column = new TColumn();
632     column.setFamily(familyAname);
633     columns.add(column);
634     scan.setColumns(columns);
635     scan.setStartRow("testScanWithBatchSize".getBytes());
636     scan.setStopRow("testScanWithBatchSize\uffff".getBytes());
637     // set batch size to 10 columns per call
638     scan.setBatchSize(10);
639 
640     // get scanner
641     int scanId = handler.openScanner(table, scan);
642     List<TResult> results = null;
643     for (int i = 0; i < 10; i++) {
644       // get batch for single row (10x10 is what we expect)
645       results = handler.getScannerRows(scanId, 1);
646       assertEquals(1, results.size());
647       // check length of batch
648       List<TColumnValue> cols = results.get(0).getColumnValues();
649       assertEquals(10, cols.size());
650       // check if the columns are returned and in order
651       for (int y = 0; y < 10; y++) {
652         int colNum = y + (10 * i);
653         String colNumPad = pad(colNum, (byte) 3);
654         assertArrayEquals(("col" + colNumPad).getBytes(), cols.get(y).getQualifier());
655       }
656     }
657 
658     // check that we are at the end of the scan
659     results = handler.getScannerRows(scanId, 1);
660     assertEquals(0, results.size());
661 
662     // close scanner and check that it was indeed closed
663     handler.closeScanner(scanId);
664     try {
665       handler.getScannerRows(scanId, 1);
666       fail("Scanner id should be invalid");
667     } catch (TIllegalArgument e) {
668     }
669   }
670   @Test
671   public void testGetScannerResults() throws Exception {
672     ThriftHBaseServiceHandler handler = createHandler();
673     ByteBuffer table = wrap(tableAname);
674 
675     // insert data
676     TColumnValue columnValue =
677         new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
678     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
679     columnValues.add(columnValue);
680     for (int i = 0; i < 20; i++) {
681       TPut put =
682           new TPut(wrap(("testGetScannerResults" + pad(i, (byte) 2)).getBytes()), columnValues);
683       handler.put(table, put);
684     }
685 
686     // create scan instance
687     TScan scan = new TScan();
688     List<TColumn> columns = new ArrayList<TColumn>();
689     TColumn column = new TColumn();
690     column.setFamily(familyAname);
691     column.setQualifier(qualifierAname);
692     columns.add(column);
693     scan.setColumns(columns);
694     scan.setStartRow("testGetScannerResults".getBytes());
695 
696     // get 5 rows and check the returned results
697     scan.setStopRow("testGetScannerResults05".getBytes());
698     List<TResult> results = handler.getScannerResults(table, scan, 5);
699     assertEquals(5, results.size());
700     for (int i = 0; i < 5; i++) {
701       // check if the rows are returned and in order
702       assertArrayEquals(("testGetScannerResults" + pad(i, (byte) 2)).getBytes(), results.get(i)
703           .getRow());
704     }
705 
706     // get 10 rows and check the returned results
707     scan.setStopRow("testGetScannerResults10".getBytes());
708     results = handler.getScannerResults(table, scan, 10);
709     assertEquals(10, results.size());
710     for (int i = 0; i < 10; i++) {
711       // check if the rows are returned and in order
712       assertArrayEquals(("testGetScannerResults" + pad(i, (byte) 2)).getBytes(), results.get(i)
713           .getRow());
714     }
715 
716     // get 20 rows and check the returned results
717     scan.setStopRow("testGetScannerResults20".getBytes());
718     results = handler.getScannerResults(table, scan, 20);
719     assertEquals(20, results.size());
720     for (int i = 0; i < 20; i++) {
721       // check if the rows are returned and in order
722       assertArrayEquals(("testGetScannerResults" + pad(i, (byte) 2)).getBytes(), results.get(i)
723           .getRow());
724     }
725   }
726 
727   @Test
728   public void testFilterRegistration() throws Exception {
729     Configuration conf = UTIL.getConfiguration();
730     conf.set("hbase.thrift.filters", "MyFilter:filterclass");
731     ThriftServer.registerFilters(conf);
732     Map<String, String> registeredFilters = ParseFilter.getAllFilters();
733     assertEquals("filterclass", registeredFilters.get("MyFilter"));
734   }
735 
736   @Test
737   public void testMetrics() throws Exception {
738     Configuration conf = UTIL.getConfiguration();
739     ThriftMetrics metrics = getMetrics(conf);
740     THBaseService.Iface handler =
741         ThriftHBaseServiceHandler.newInstance(conf, metrics);
742     byte[] rowName = "testMetrics".getBytes();
743     ByteBuffer table = wrap(tableAname);
744 
745     TGet get = new TGet(wrap(rowName));
746     assertFalse(handler.exists(table, get));
747 
748     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
749     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
750     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),  wrap(valueBname)));
751     TPut put = new TPut(wrap(rowName), columnValues);
752     put.setColumnValues(columnValues);
753 
754     handler.put(table, put);
755 
756     assertTrue(handler.exists(table, get));
757     logMetrics(metrics);
758     verifyMetrics(metrics, "put_num_ops", 1);
759     verifyMetrics(metrics, "exists_num_ops", 2);
760   }
761 
762   @Test
763   public void testAttribute() throws Exception {
764     byte[] rowName = "testAttribute".getBytes();
765     byte[] attributeKey = "attribute1".getBytes();
766     byte[] attributeValue = "value1".getBytes();
767     Map<ByteBuffer, ByteBuffer> attributes = new HashMap<ByteBuffer, ByteBuffer>();
768     attributes.put(wrap(attributeKey), wrap(attributeValue));
769 
770     TGet tGet = new TGet(wrap(rowName));
771     tGet.setAttributes(attributes);
772     Get get = getFromThrift(tGet);
773     assertArrayEquals(get.getAttribute("attribute1"), attributeValue);
774 
775     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
776     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
777     TPut tPut = new TPut(wrap(rowName) , columnValues);
778     tPut.setAttributes(attributes);
779     Put put = putFromThrift(tPut);
780     assertArrayEquals(put.getAttribute("attribute1"), attributeValue);
781 
782     TScan tScan = new TScan();
783     tScan.setAttributes(attributes);
784     Scan scan = scanFromThrift(tScan);
785     assertArrayEquals(scan.getAttribute("attribute1"), attributeValue);
786 
787     TDelete tDelete = new TDelete(wrap(rowName));
788     tDelete.setAttributes(attributes);
789     Delete delete = deleteFromThrift(tDelete);
790     assertArrayEquals(delete.getAttribute("attribute1"), attributeValue);
791   }
792 
793   /**
794    * Put valueA to a row, make sure put has happened, then create a mutation object to put valueB
795    * and delete ValueA, then check that the row value is only valueB.
796    *
797    * @throws Exception
798    */
799   @Test
800   public void testMutateRow() throws Exception {
801     ThriftHBaseServiceHandler handler = createHandler();
802     byte[] rowName = "testMutateRow".getBytes();
803     ByteBuffer table = wrap(tableAname);
804 
805     List<TColumnValue> columnValuesA = new ArrayList<TColumnValue>();
806     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
807         wrap(valueAname));
808     columnValuesA.add(columnValueA);
809     TPut putA = new TPut(wrap(rowName), columnValuesA);
810     putA.setColumnValues(columnValuesA);
811 
812     handler.put(table,putA);
813 
814     TGet get = new TGet(wrap(rowName));
815     TResult result = handler.get(table, get);
816     assertArrayEquals(rowName, result.getRow());
817     List<TColumnValue> returnedColumnValues = result.getColumnValues();
818 
819     List<TColumnValue> expectedColumnValues = new ArrayList<TColumnValue>();
820     expectedColumnValues.add(columnValueA);
821     assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
822 
823     List<TColumnValue> columnValuesB = new ArrayList<TColumnValue>();
824     TColumnValue columnValueB = new TColumnValue(wrap(familyAname), wrap(qualifierBname),
825         wrap(valueBname));
826     columnValuesB.add(columnValueB);
827     TPut putB = new TPut(wrap(rowName), columnValuesB);
828     putB.setColumnValues(columnValuesB);
829 
830     TDelete delete = new TDelete(wrap(rowName));
831     List<TColumn> deleteColumns = new ArrayList<TColumn>();
832     TColumn deleteColumn = new TColumn(wrap(familyAname));
833     deleteColumn.setQualifier(qualifierAname);
834     deleteColumns.add(deleteColumn);
835     delete.setColumns(deleteColumns);
836 
837     List<TMutation> mutations = new ArrayList<TMutation>();
838     TMutation mutationA = TMutation.put(putB);
839     mutations.add(mutationA);
840 
841     TMutation mutationB = TMutation.deleteSingle(delete);
842     mutations.add(mutationB);
843 
844     TRowMutations tRowMutations = new TRowMutations(wrap(rowName),mutations);
845     handler.mutateRow(table,tRowMutations);
846 
847     result = handler.get(table, get);
848     assertArrayEquals(rowName, result.getRow());
849     returnedColumnValues = result.getColumnValues();
850 
851     expectedColumnValues = new ArrayList<TColumnValue>();
852     expectedColumnValues.add(columnValueB);
853     assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
854   }
855 
856   /**
857    * Create TPut, TDelete , TIncrement objects, set durability then call ThriftUtility
858    * functions to get Put , Delete and Increment respectively. Use getDurability to make sure
859    * the returned objects have the appropriate durability setting.
860    *
861    * @throws Exception
862    */
863   @Test
864   public void testDurability() throws Exception {
865     byte[] rowName = "testDurability".getBytes();
866     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
867     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
868 
869     List<TColumnIncrement> incrementColumns = new ArrayList<TColumnIncrement>();
870     incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
871 
872     TDelete tDelete = new TDelete(wrap(rowName));
873 
874     //if not setting writeToWal, check for default value
875     Delete delete = deleteFromThrift(tDelete);
876     assertEquals(delete.getDurability(), Durability.USE_DEFAULT);
877 
878     //if setting writeToWal to true, durability should be CF default
879     tDelete.setWriteToWal(true);
880     delete = deleteFromThrift(tDelete);
881     assertEquals(delete.getDurability(), Durability.USE_DEFAULT);
882 
883     //if setting writeToWal to false, durability should be SKIP_WAL
884     tDelete.setWriteToWal(false);
885     delete = deleteFromThrift(tDelete);
886     assertEquals(delete.getDurability(), Durability.SKIP_WAL);
887 
888 
889     tDelete.setDurability(TDurability.SKIP_WAL);
890     delete = deleteFromThrift(tDelete);
891     assertEquals(delete.getDurability(), Durability.SKIP_WAL);
892 
893     tDelete.setDurability(TDurability.ASYNC_WAL);
894     delete = deleteFromThrift(tDelete);
895     assertEquals(delete.getDurability(), Durability.ASYNC_WAL);
896 
897     tDelete.setDurability(TDurability.SYNC_WAL);
898     delete = deleteFromThrift(tDelete);
899     assertEquals(delete.getDurability(), Durability.SYNC_WAL);
900 
901     tDelete.setDurability(TDurability.FSYNC_WAL);
902     delete = deleteFromThrift(tDelete);
903     assertEquals(delete.getDurability(), Durability.FSYNC_WAL);
904 
905     TPut tPut = new TPut(wrap(rowName), columnValues);
906 
907     //if not setting writeToWal, check for default value
908     Put put = putFromThrift(tPut);
909     assertEquals(put.getDurability(), Durability.USE_DEFAULT);
910 
911     //if setting writeToWal to true, durability should be CF default
912     tPut.setWriteToWal(true);
913     put = putFromThrift(tPut);
914     assertEquals(put.getDurability(), Durability.USE_DEFAULT);
915 
916     //if setting writeToWal to false, durability should be SKIP_WAL
917     tPut.setWriteToWal(false);
918     put = putFromThrift(tPut);
919     assertEquals(put.getDurability(), Durability.SKIP_WAL);
920 
921     tPut.setDurability(TDurability.SKIP_WAL);
922     put = putFromThrift(tPut);
923     assertEquals(put.getDurability(), Durability.SKIP_WAL);
924 
925     tPut.setDurability(TDurability.ASYNC_WAL);
926     put = putFromThrift(tPut);
927     assertEquals(put.getDurability(), Durability.ASYNC_WAL);
928 
929     tPut.setDurability(TDurability.SYNC_WAL);
930     put = putFromThrift(tPut);
931     assertEquals(put.getDurability(), Durability.SYNC_WAL);
932 
933     tPut.setDurability(TDurability.FSYNC_WAL);
934     put = putFromThrift(tPut);
935     assertEquals(put.getDurability(), Durability.FSYNC_WAL);
936   }
937 
938   private static ThriftMetrics getMetrics(Configuration conf) throws Exception {
939     setupMetricsContext();
940     return new ThriftMetrics(Integer.parseInt(ThriftServer.DEFAULT_LISTEN_PORT),
941         conf, THBaseService.Iface.class);
942   }
943  
944   private static void setupMetricsContext() throws IOException {
945     ContextFactory factory = ContextFactory.getFactory();
946     factory.setAttribute(ThriftMetrics.CONTEXT_NAME + ".class",
947         NoEmitMetricsContext.class.getName());
948     MetricsUtil.getContext(ThriftMetrics.CONTEXT_NAME)
949                .createRecord(ThriftMetrics.CONTEXT_NAME).remove();
950   }
951  
952   private static void logMetrics(ThriftMetrics metrics) throws Exception {
953     if (LOG.isDebugEnabled()) {
954       return;
955     }
956     MetricsContext context = MetricsUtil.getContext( 
957         ThriftMetrics.CONTEXT_NAME); 
958     metrics.doUpdates(context); 
959     for (String key : context.getAllRecords().keySet()) {
960       for (OutputRecord record : context.getAllRecords().get(key)) {
961         for (String name : record.getMetricNames()) {
962           LOG.debug("metrics:" + name + " value:" +
963               record.getMetric(name).intValue());
964         }
965       }
966     }
967   }
968 
969   private static void verifyMetrics(ThriftMetrics metrics, String name, int expectValue)
970       throws Exception { 
971     MetricsContext context = MetricsUtil.getContext( 
972         ThriftMetrics.CONTEXT_NAME); 
973     metrics.doUpdates(context); 
974     OutputRecord record = context.getAllRecords().get( 
975         ThriftMetrics.CONTEXT_NAME).iterator().next(); 
976     assertEquals(expectValue, record.getMetric(name).intValue()); 
977   }
978 
979   @org.junit.Rule
980   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
981     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
982 }
983