View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.coprocessor;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertNotNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.util.Collections;
26  import java.util.Map;
27  import java.util.TreeMap;
28  
29  import com.google.protobuf.HBaseZeroCopyByteString;
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.HColumnDescriptor;
35  import org.apache.hadoop.hbase.HTableDescriptor;
36  import org.apache.hadoop.hbase.MediumTests;
37  import org.apache.hadoop.hbase.TableName;
38  import org.apache.hadoop.hbase.client.HBaseAdmin;
39  import org.apache.hadoop.hbase.client.HTable;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.client.coprocessor.Batch;
42  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationProtos;
43  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationProtos.SumResponse;
44  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithErrorsProtos;
45  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos;
46  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos.ColumnAggregationServiceNullResponse;
47  import org.apache.hadoop.hbase.util.Bytes;
48  import org.junit.AfterClass;
49  import org.junit.BeforeClass;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  
53  import com.google.protobuf.ServiceException;
54  
55  /**
56   * TestEndpoint: test cases to verify the batch execution of coprocessor Endpoint
57   */
58  @Category(MediumTests.class)
59  public class TestBatchCoprocessorEndpoint {
60    private static final Log LOG = LogFactory.getLog(TestBatchCoprocessorEndpoint.class);
61  
62    private static final TableName TEST_TABLE =
63        TableName.valueOf("TestTable");
64    private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily");
65    private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier");
66    private static byte[] ROW = Bytes.toBytes("testRow");
67  
68    private static final int ROWSIZE = 20;
69    private static final int rowSeperator1 = 5;
70    private static final int rowSeperator2 = 12;
71    private static byte[][] ROWS = makeN(ROW, ROWSIZE);
72  
73    private static HBaseTestingUtility util = new HBaseTestingUtility();
74  
75    @BeforeClass
76    public static void setupBeforeClass() throws Exception {
77      // set configure to indicate which cp should be loaded
78      Configuration conf = util.getConfiguration();
79      conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
80          org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint.class.getName(),
81          ProtobufCoprocessorService.class.getName(),
82          ColumnAggregationEndpointWithErrors.class.getName(),
83          ColumnAggregationEndpointNullResponse.class.getName());
84      conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
85          ProtobufCoprocessorService.class.getName());
86      util.startMiniCluster(2);
87      HBaseAdmin admin = new HBaseAdmin(conf);
88      HTableDescriptor desc = new HTableDescriptor(TEST_TABLE);
89      desc.addFamily(new HColumnDescriptor(TEST_FAMILY));
90      admin.createTable(desc, new byte[][]{ROWS[rowSeperator1], ROWS[rowSeperator2]});
91      util.waitUntilAllRegionsAssigned(TEST_TABLE);
92      admin.close();
93  
94      HTable table = new HTable(conf, TEST_TABLE);
95      for (int i = 0; i < ROWSIZE; i++) {
96        Put put = new Put(ROWS[i]);
97        put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(i));
98        table.put(put);
99      }
100     table.close();
101   }
102 
103   @AfterClass
104   public static void tearDownAfterClass() throws Exception {
105     util.shutdownMiniCluster();
106   }
107 
108   @Test
109   public void testAggregationNullResponse() throws Throwable {
110     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
111     ColumnAggregationWithNullResponseProtos.SumRequest.Builder builder =
112         ColumnAggregationWithNullResponseProtos.SumRequest
113         .newBuilder();
114     builder.setFamily(HBaseZeroCopyByteString.wrap(TEST_FAMILY));
115     if (TEST_QUALIFIER != null && TEST_QUALIFIER.length > 0) {
116       builder.setQualifier(HBaseZeroCopyByteString.wrap(TEST_QUALIFIER));
117     }
118     Map<byte[], ColumnAggregationWithNullResponseProtos.SumResponse> results =
119         table.batchCoprocessorService(
120             ColumnAggregationServiceNullResponse.getDescriptor().findMethodByName("sum"),
121             builder.build(), ROWS[0], ROWS[ROWS.length - 1],
122             ColumnAggregationWithNullResponseProtos.SumResponse.getDefaultInstance());
123 
124     int sumResult = 0;
125     int expectedResult = 0;
126     for (Map.Entry<byte[], ColumnAggregationWithNullResponseProtos.SumResponse> e :
127         results.entrySet()) {
128       LOG.info("Got value " + e.getValue().getSum() + " for region "
129           + Bytes.toStringBinary(e.getKey()));
130       sumResult += e.getValue().getSum();
131     }
132     for (int i = 0; i < rowSeperator2; i++) {
133       expectedResult += i;
134     }
135     assertEquals("Invalid result", expectedResult, sumResult);
136     table.close();
137   }
138 
139   private static byte[][] makeN(byte[] base, int n) {
140     byte[][] ret = new byte[n][];
141     for (int i = 0; i < n; i++) {
142       ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%02d", i)));
143     }
144     return ret;
145   }
146 
147   private Map<byte[], SumResponse> sum(final HTable table, final byte[] family,
148       final byte[] qualifier, final byte[] start, final byte[] end) throws ServiceException,
149       Throwable {
150     ColumnAggregationProtos.SumRequest.Builder builder = ColumnAggregationProtos.SumRequest
151         .newBuilder();
152     builder.setFamily(HBaseZeroCopyByteString.wrap(family));
153     if (qualifier != null && qualifier.length > 0) {
154       builder.setQualifier(HBaseZeroCopyByteString.wrap(qualifier));
155     }
156     return table.batchCoprocessorService(
157         ColumnAggregationProtos.ColumnAggregationService.getDescriptor().findMethodByName("sum"),
158         builder.build(), start, end, ColumnAggregationProtos.SumResponse.getDefaultInstance());
159   }
160 
161   @Test
162   public void testAggregationWithReturnValue() throws Throwable {
163     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
164     Map<byte[], SumResponse> results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[0],
165         ROWS[ROWS.length - 1]);
166     int sumResult = 0;
167     int expectedResult = 0;
168     for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
169       LOG.info("Got value " + e.getValue().getSum() + " for region "
170           + Bytes.toStringBinary(e.getKey()));
171       sumResult += e.getValue().getSum();
172     }
173     for (int i = 0; i < ROWSIZE; i++) {
174       expectedResult += i;
175     }
176     assertEquals("Invalid result", expectedResult, sumResult);
177 
178     results.clear();
179 
180     // scan: for region 2 and region 3
181     results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[rowSeperator1],
182         ROWS[ROWS.length - 1]);
183     sumResult = 0;
184     expectedResult = 0;
185     for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
186       LOG.info("Got value " + e.getValue().getSum() + " for region "
187           + Bytes.toStringBinary(e.getKey()));
188       sumResult += e.getValue().getSum();
189     }
190     for (int i = rowSeperator1; i < ROWSIZE; i++) {
191       expectedResult += i;
192     }
193     assertEquals("Invalid result", expectedResult, sumResult);
194     table.close();
195   }
196 
197   @Test
198   public void testAggregation() throws Throwable {
199     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
200     Map<byte[], SumResponse> results = sum(table, TEST_FAMILY, TEST_QUALIFIER,
201         ROWS[0], ROWS[ROWS.length - 1]);
202     int sumResult = 0;
203     int expectedResult = 0;
204     for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
205       LOG.info("Got value " + e.getValue().getSum() + " for region "
206           + Bytes.toStringBinary(e.getKey()));
207       sumResult += e.getValue().getSum();
208     }
209     for (int i = 0; i < ROWSIZE; i++) {
210       expectedResult += i;
211     }
212     assertEquals("Invalid result", expectedResult, sumResult);
213 
214     // scan: for region 2 and region 3
215     results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[rowSeperator1], ROWS[ROWS.length - 1]);
216     sumResult = 0;
217     expectedResult = 0;
218     for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
219       LOG.info("Got value " + e.getValue().getSum() + " for region "
220           + Bytes.toStringBinary(e.getKey()));
221       sumResult += e.getValue().getSum();
222     }
223     for (int i = rowSeperator1; i < ROWSIZE; i++) {
224       expectedResult += i;
225     }
226     assertEquals("Invalid result", expectedResult, sumResult);
227     table.close();
228   }
229 
230   @Test
231   public void testAggregationWithErrors() throws Throwable {
232     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
233     final Map<byte[], ColumnAggregationWithErrorsProtos.SumResponse> results =
234         Collections.synchronizedMap(
235             new TreeMap<byte[], ColumnAggregationWithErrorsProtos.SumResponse>(
236                 Bytes.BYTES_COMPARATOR
237             ));
238     ColumnAggregationWithErrorsProtos.SumRequest.Builder builder =
239         ColumnAggregationWithErrorsProtos.SumRequest
240         .newBuilder();
241     builder.setFamily(HBaseZeroCopyByteString.wrap(TEST_FAMILY));
242     if (TEST_QUALIFIER != null && TEST_QUALIFIER.length > 0) {
243       builder.setQualifier(HBaseZeroCopyByteString.wrap(TEST_QUALIFIER));
244     }
245 
246     boolean hasError = false;
247     try {
248       table.batchCoprocessorService(
249           ColumnAggregationWithErrorsProtos.ColumnAggregationServiceWithErrors.getDescriptor()
250               .findMethodByName("sum"),
251           builder.build(), ROWS[0], ROWS[ROWS.length - 1],
252           ColumnAggregationWithErrorsProtos.SumResponse.getDefaultInstance(),
253           new Batch.Callback<ColumnAggregationWithErrorsProtos.SumResponse>() {
254 
255             @Override
256             public void update(byte[] region, byte[] row,
257                 ColumnAggregationWithErrorsProtos.SumResponse result) {
258               results.put(region, result);
259             }
260           });
261     } catch (Throwable t) {
262       LOG.info("Exceptions in coprocessor service", t);
263       hasError = true;
264     }
265 
266     int sumResult = 0;
267     int expectedResult = 0;
268     for (Map.Entry<byte[], ColumnAggregationWithErrorsProtos.SumResponse> e : results.entrySet()) {
269       LOG.info("Got value " + e.getValue().getSum() + " for region "
270           + Bytes.toStringBinary(e.getKey()));
271       sumResult += e.getValue().getSum();
272     }
273     for (int i = 0; i < rowSeperator2; i++) {
274       expectedResult += i;
275     }
276     assertEquals("Invalid result", expectedResult, sumResult);
277     assertTrue(hasError);
278     table.close();
279   }
280 }