View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.coprocessor;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.util.Collections;
25  import java.util.Map;
26  import java.util.TreeMap;
27  
28  import org.apache.hadoop.hbase.util.ByteStringer;
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.HBaseTestingUtility;
33  import org.apache.hadoop.hbase.HColumnDescriptor;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.testclassification.MediumTests;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.client.HBaseAdmin;
38  import org.apache.hadoop.hbase.client.HTable;
39  import org.apache.hadoop.hbase.client.Put;
40  import org.apache.hadoop.hbase.client.coprocessor.Batch;
41  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationProtos;
42  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationProtos.SumResponse;
43  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithErrorsProtos;
44  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithErrorsProtos.ColumnAggregationWithErrorsSumRequest;
45  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithErrorsProtos.ColumnAggregationWithErrorsSumResponse;
46  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos;
47  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos.ColumnAggregationServiceNullResponse;
48  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos.ColumnAggregationNullResponseSumRequest;
49  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos.ColumnAggregationNullResponseSumResponse;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.junit.AfterClass;
52  import org.junit.BeforeClass;
53  import org.junit.Test;
54  import org.junit.experimental.categories.Category;
55  
56  import com.google.protobuf.ServiceException;
57  
58  /**
59   * TestEndpoint: test cases to verify the batch execution of coprocessor Endpoint
60   */
61  @Category(MediumTests.class)
62  public class TestBatchCoprocessorEndpoint {
63    private static final Log LOG = LogFactory.getLog(TestBatchCoprocessorEndpoint.class);
64  
65    private static final TableName TEST_TABLE =
66        TableName.valueOf("TestTable");
67    private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily");
68    private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier");
69    private static byte[] ROW = Bytes.toBytes("testRow");
70  
71    private static final int ROWSIZE = 20;
72    private static final int rowSeperator1 = 5;
73    private static final int rowSeperator2 = 12;
74    private static byte[][] ROWS = makeN(ROW, ROWSIZE);
75  
76    private static HBaseTestingUtility util = new HBaseTestingUtility();
77  
78    @BeforeClass
79    public static void setupBeforeClass() throws Exception {
80      // set configure to indicate which cp should be loaded
81      Configuration conf = util.getConfiguration();
82      conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
83          org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint.class.getName(),
84          ProtobufCoprocessorService.class.getName(),
85          ColumnAggregationEndpointWithErrors.class.getName(),
86          ColumnAggregationEndpointNullResponse.class.getName());
87      conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
88          ProtobufCoprocessorService.class.getName());
89      util.startMiniCluster(2);
90      HBaseAdmin admin = new HBaseAdmin(conf);
91      HTableDescriptor desc = new HTableDescriptor(TEST_TABLE);
92      desc.addFamily(new HColumnDescriptor(TEST_FAMILY));
93      admin.createTable(desc, new byte[][]{ROWS[rowSeperator1], ROWS[rowSeperator2]});
94      util.waitUntilAllRegionsAssigned(TEST_TABLE);
95      admin.close();
96  
97      HTable table = new HTable(conf, TEST_TABLE);
98      for (int i = 0; i < ROWSIZE; i++) {
99        Put put = new Put(ROWS[i]);
100       put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(i));
101       table.put(put);
102     }
103     table.close();
104   }
105 
106   @AfterClass
107   public static void tearDownAfterClass() throws Exception {
108     util.shutdownMiniCluster();
109   }
110 
111   @Test
112   public void testAggregationNullResponse() throws Throwable {
113     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
114     ColumnAggregationNullResponseSumRequest.Builder builder =
115         ColumnAggregationNullResponseSumRequest
116         .newBuilder();
117     builder.setFamily(ByteStringer.wrap(TEST_FAMILY));
118     if (TEST_QUALIFIER != null && TEST_QUALIFIER.length > 0) {
119       builder.setQualifier(ByteStringer.wrap(TEST_QUALIFIER));
120     }
121     Map<byte[], ColumnAggregationNullResponseSumResponse> results =
122         table.batchCoprocessorService(
123             ColumnAggregationServiceNullResponse.getDescriptor().findMethodByName("sum"),
124             builder.build(), ROWS[0], ROWS[ROWS.length - 1],
125             ColumnAggregationNullResponseSumResponse.getDefaultInstance());
126 
127     int sumResult = 0;
128     int expectedResult = 0;
129     for (Map.Entry<byte[], ColumnAggregationNullResponseSumResponse> e :
130         results.entrySet()) {
131       LOG.info("Got value " + e.getValue().getSum() + " for region "
132           + Bytes.toStringBinary(e.getKey()));
133       sumResult += e.getValue().getSum();
134     }
135     for (int i = 0; i < rowSeperator2; i++) {
136       expectedResult += i;
137     }
138     assertEquals("Invalid result", expectedResult, sumResult);
139     table.close();
140   }
141 
142   private static byte[][] makeN(byte[] base, int n) {
143     byte[][] ret = new byte[n][];
144     for (int i = 0; i < n; i++) {
145       ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%02d", i)));
146     }
147     return ret;
148   }
149 
150   private Map<byte[], SumResponse> sum(final HTable table, final byte[] family,
151       final byte[] qualifier, final byte[] start, final byte[] end) throws ServiceException,
152       Throwable {
153     ColumnAggregationProtos.SumRequest.Builder builder = ColumnAggregationProtos.SumRequest
154         .newBuilder();
155     builder.setFamily(ByteStringer.wrap(family));
156     if (qualifier != null && qualifier.length > 0) {
157       builder.setQualifier(ByteStringer.wrap(qualifier));
158     }
159     return table.batchCoprocessorService(
160         ColumnAggregationProtos.ColumnAggregationService.getDescriptor().findMethodByName("sum"),
161         builder.build(), start, end, ColumnAggregationProtos.SumResponse.getDefaultInstance());
162   }
163 
164   @Test
165   public void testAggregationWithReturnValue() throws Throwable {
166     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
167     Map<byte[], SumResponse> results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[0],
168         ROWS[ROWS.length - 1]);
169     int sumResult = 0;
170     int expectedResult = 0;
171     for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
172       LOG.info("Got value " + e.getValue().getSum() + " for region "
173           + Bytes.toStringBinary(e.getKey()));
174       sumResult += e.getValue().getSum();
175     }
176     for (int i = 0; i < ROWSIZE; i++) {
177       expectedResult += i;
178     }
179     assertEquals("Invalid result", expectedResult, sumResult);
180 
181     results.clear();
182 
183     // scan: for region 2 and region 3
184     results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[rowSeperator1],
185         ROWS[ROWS.length - 1]);
186     sumResult = 0;
187     expectedResult = 0;
188     for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
189       LOG.info("Got value " + e.getValue().getSum() + " for region "
190           + Bytes.toStringBinary(e.getKey()));
191       sumResult += e.getValue().getSum();
192     }
193     for (int i = rowSeperator1; i < ROWSIZE; i++) {
194       expectedResult += i;
195     }
196     assertEquals("Invalid result", expectedResult, sumResult);
197     table.close();
198   }
199 
200   @Test
201   public void testAggregation() throws Throwable {
202     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
203     Map<byte[], SumResponse> results = sum(table, TEST_FAMILY, TEST_QUALIFIER,
204         ROWS[0], ROWS[ROWS.length - 1]);
205     int sumResult = 0;
206     int expectedResult = 0;
207     for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
208       LOG.info("Got value " + e.getValue().getSum() + " for region "
209           + Bytes.toStringBinary(e.getKey()));
210       sumResult += e.getValue().getSum();
211     }
212     for (int i = 0; i < ROWSIZE; i++) {
213       expectedResult += i;
214     }
215     assertEquals("Invalid result", expectedResult, sumResult);
216 
217     // scan: for region 2 and region 3
218     results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[rowSeperator1], ROWS[ROWS.length - 1]);
219     sumResult = 0;
220     expectedResult = 0;
221     for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
222       LOG.info("Got value " + e.getValue().getSum() + " for region "
223           + Bytes.toStringBinary(e.getKey()));
224       sumResult += e.getValue().getSum();
225     }
226     for (int i = rowSeperator1; i < ROWSIZE; i++) {
227       expectedResult += i;
228     }
229     assertEquals("Invalid result", expectedResult, sumResult);
230     table.close();
231   }
232 
233   @Test
234   public void testAggregationWithErrors() throws Throwable {
235     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
236     final Map<byte[], ColumnAggregationWithErrorsSumResponse> results =
237         Collections.synchronizedMap(
238             new TreeMap<byte[], ColumnAggregationWithErrorsSumResponse>(
239                 Bytes.BYTES_COMPARATOR
240             ));
241     ColumnAggregationWithErrorsSumRequest.Builder builder =
242         ColumnAggregationWithErrorsSumRequest
243         .newBuilder();
244     builder.setFamily(ByteStringer.wrap(TEST_FAMILY));
245     if (TEST_QUALIFIER != null && TEST_QUALIFIER.length > 0) {
246       builder.setQualifier(ByteStringer.wrap(TEST_QUALIFIER));
247     }
248 
249     boolean hasError = false;
250     try {
251       table.batchCoprocessorService(
252           ColumnAggregationWithErrorsProtos.ColumnAggregationServiceWithErrors.getDescriptor()
253               .findMethodByName("sum"),
254           builder.build(), ROWS[0], ROWS[ROWS.length - 1],
255           ColumnAggregationWithErrorsSumResponse.getDefaultInstance(),
256           new Batch.Callback<ColumnAggregationWithErrorsSumResponse>() {
257 
258             @Override
259             public void update(byte[] region, byte[] row,
260                 ColumnAggregationWithErrorsSumResponse result) {
261               results.put(region, result);
262             }
263           });
264     } catch (Throwable t) {
265       LOG.info("Exceptions in coprocessor service", t);
266       hasError = true;
267     }
268 
269     int sumResult = 0;
270     int expectedResult = 0;
271     for (Map.Entry<byte[], ColumnAggregationWithErrorsSumResponse> e : results.entrySet()) {
272       LOG.info("Got value " + e.getValue().getSum() + " for region "
273           + Bytes.toStringBinary(e.getKey()));
274       sumResult += e.getValue().getSum();
275     }
276     for (int i = 0; i < rowSeperator2; i++) {
277       expectedResult += i;
278     }
279     assertEquals("Invalid result", expectedResult, sumResult);
280     assertTrue(hasError);
281     table.close();
282   }
283 }