1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.coprocessor;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertTrue;
23
24 import java.util.Collections;
25 import java.util.Map;
26 import java.util.TreeMap;
27
28 import org.apache.hadoop.hbase.util.ByteStringer;
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.HColumnDescriptor;
34 import org.apache.hadoop.hbase.HTableDescriptor;
35 import org.apache.hadoop.hbase.testclassification.MediumTests;
36 import org.apache.hadoop.hbase.TableName;
37 import org.apache.hadoop.hbase.client.HBaseAdmin;
38 import org.apache.hadoop.hbase.client.HTable;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.coprocessor.Batch;
41 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationProtos;
42 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationProtos.SumResponse;
43 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithErrorsProtos;
44 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithErrorsProtos.ColumnAggregationWithErrorsSumRequest;
45 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithErrorsProtos.ColumnAggregationWithErrorsSumResponse;
46 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos;
47 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos.ColumnAggregationServiceNullResponse;
48 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos.ColumnAggregationNullResponseSumRequest;
49 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos.ColumnAggregationNullResponseSumResponse;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.junit.AfterClass;
52 import org.junit.BeforeClass;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55
56 import com.google.protobuf.ServiceException;
57
58
59
60
61 @Category(MediumTests.class)
62 public class TestBatchCoprocessorEndpoint {
63 private static final Log LOG = LogFactory.getLog(TestBatchCoprocessorEndpoint.class);
64
65 private static final TableName TEST_TABLE =
66 TableName.valueOf("TestTable");
67 private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily");
68 private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier");
69 private static byte[] ROW = Bytes.toBytes("testRow");
70
71 private static final int ROWSIZE = 20;
72 private static final int rowSeperator1 = 5;
73 private static final int rowSeperator2 = 12;
74 private static byte[][] ROWS = makeN(ROW, ROWSIZE);
75
76 private static HBaseTestingUtility util = new HBaseTestingUtility();
77
78 @BeforeClass
79 public static void setupBeforeClass() throws Exception {
80
81 Configuration conf = util.getConfiguration();
82 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
83 org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint.class.getName(),
84 ProtobufCoprocessorService.class.getName(),
85 ColumnAggregationEndpointWithErrors.class.getName(),
86 ColumnAggregationEndpointNullResponse.class.getName());
87 conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
88 ProtobufCoprocessorService.class.getName());
89 util.startMiniCluster(2);
90 HBaseAdmin admin = new HBaseAdmin(conf);
91 HTableDescriptor desc = new HTableDescriptor(TEST_TABLE);
92 desc.addFamily(new HColumnDescriptor(TEST_FAMILY));
93 admin.createTable(desc, new byte[][]{ROWS[rowSeperator1], ROWS[rowSeperator2]});
94 util.waitUntilAllRegionsAssigned(TEST_TABLE);
95 admin.close();
96
97 HTable table = new HTable(conf, TEST_TABLE);
98 for (int i = 0; i < ROWSIZE; i++) {
99 Put put = new Put(ROWS[i]);
100 put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(i));
101 table.put(put);
102 }
103 table.close();
104 }
105
106 @AfterClass
107 public static void tearDownAfterClass() throws Exception {
108 util.shutdownMiniCluster();
109 }
110
111 @Test
112 public void testAggregationNullResponse() throws Throwable {
113 HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
114 ColumnAggregationNullResponseSumRequest.Builder builder =
115 ColumnAggregationNullResponseSumRequest
116 .newBuilder();
117 builder.setFamily(ByteStringer.wrap(TEST_FAMILY));
118 if (TEST_QUALIFIER != null && TEST_QUALIFIER.length > 0) {
119 builder.setQualifier(ByteStringer.wrap(TEST_QUALIFIER));
120 }
121 Map<byte[], ColumnAggregationNullResponseSumResponse> results =
122 table.batchCoprocessorService(
123 ColumnAggregationServiceNullResponse.getDescriptor().findMethodByName("sum"),
124 builder.build(), ROWS[0], ROWS[ROWS.length - 1],
125 ColumnAggregationNullResponseSumResponse.getDefaultInstance());
126
127 int sumResult = 0;
128 int expectedResult = 0;
129 for (Map.Entry<byte[], ColumnAggregationNullResponseSumResponse> e :
130 results.entrySet()) {
131 LOG.info("Got value " + e.getValue().getSum() + " for region "
132 + Bytes.toStringBinary(e.getKey()));
133 sumResult += e.getValue().getSum();
134 }
135 for (int i = 0; i < rowSeperator2; i++) {
136 expectedResult += i;
137 }
138 assertEquals("Invalid result", expectedResult, sumResult);
139 table.close();
140 }
141
142 private static byte[][] makeN(byte[] base, int n) {
143 byte[][] ret = new byte[n][];
144 for (int i = 0; i < n; i++) {
145 ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%02d", i)));
146 }
147 return ret;
148 }
149
150 private Map<byte[], SumResponse> sum(final HTable table, final byte[] family,
151 final byte[] qualifier, final byte[] start, final byte[] end) throws ServiceException,
152 Throwable {
153 ColumnAggregationProtos.SumRequest.Builder builder = ColumnAggregationProtos.SumRequest
154 .newBuilder();
155 builder.setFamily(ByteStringer.wrap(family));
156 if (qualifier != null && qualifier.length > 0) {
157 builder.setQualifier(ByteStringer.wrap(qualifier));
158 }
159 return table.batchCoprocessorService(
160 ColumnAggregationProtos.ColumnAggregationService.getDescriptor().findMethodByName("sum"),
161 builder.build(), start, end, ColumnAggregationProtos.SumResponse.getDefaultInstance());
162 }
163
164 @Test
165 public void testAggregationWithReturnValue() throws Throwable {
166 HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
167 Map<byte[], SumResponse> results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[0],
168 ROWS[ROWS.length - 1]);
169 int sumResult = 0;
170 int expectedResult = 0;
171 for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
172 LOG.info("Got value " + e.getValue().getSum() + " for region "
173 + Bytes.toStringBinary(e.getKey()));
174 sumResult += e.getValue().getSum();
175 }
176 for (int i = 0; i < ROWSIZE; i++) {
177 expectedResult += i;
178 }
179 assertEquals("Invalid result", expectedResult, sumResult);
180
181 results.clear();
182
183
184 results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[rowSeperator1],
185 ROWS[ROWS.length - 1]);
186 sumResult = 0;
187 expectedResult = 0;
188 for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
189 LOG.info("Got value " + e.getValue().getSum() + " for region "
190 + Bytes.toStringBinary(e.getKey()));
191 sumResult += e.getValue().getSum();
192 }
193 for (int i = rowSeperator1; i < ROWSIZE; i++) {
194 expectedResult += i;
195 }
196 assertEquals("Invalid result", expectedResult, sumResult);
197 table.close();
198 }
199
200 @Test
201 public void testAggregation() throws Throwable {
202 HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
203 Map<byte[], SumResponse> results = sum(table, TEST_FAMILY, TEST_QUALIFIER,
204 ROWS[0], ROWS[ROWS.length - 1]);
205 int sumResult = 0;
206 int expectedResult = 0;
207 for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
208 LOG.info("Got value " + e.getValue().getSum() + " for region "
209 + Bytes.toStringBinary(e.getKey()));
210 sumResult += e.getValue().getSum();
211 }
212 for (int i = 0; i < ROWSIZE; i++) {
213 expectedResult += i;
214 }
215 assertEquals("Invalid result", expectedResult, sumResult);
216
217
218 results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[rowSeperator1], ROWS[ROWS.length - 1]);
219 sumResult = 0;
220 expectedResult = 0;
221 for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
222 LOG.info("Got value " + e.getValue().getSum() + " for region "
223 + Bytes.toStringBinary(e.getKey()));
224 sumResult += e.getValue().getSum();
225 }
226 for (int i = rowSeperator1; i < ROWSIZE; i++) {
227 expectedResult += i;
228 }
229 assertEquals("Invalid result", expectedResult, sumResult);
230 table.close();
231 }
232
233 @Test
234 public void testAggregationWithErrors() throws Throwable {
235 HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
236 final Map<byte[], ColumnAggregationWithErrorsSumResponse> results =
237 Collections.synchronizedMap(
238 new TreeMap<byte[], ColumnAggregationWithErrorsSumResponse>(
239 Bytes.BYTES_COMPARATOR
240 ));
241 ColumnAggregationWithErrorsSumRequest.Builder builder =
242 ColumnAggregationWithErrorsSumRequest
243 .newBuilder();
244 builder.setFamily(ByteStringer.wrap(TEST_FAMILY));
245 if (TEST_QUALIFIER != null && TEST_QUALIFIER.length > 0) {
246 builder.setQualifier(ByteStringer.wrap(TEST_QUALIFIER));
247 }
248
249 boolean hasError = false;
250 try {
251 table.batchCoprocessorService(
252 ColumnAggregationWithErrorsProtos.ColumnAggregationServiceWithErrors.getDescriptor()
253 .findMethodByName("sum"),
254 builder.build(), ROWS[0], ROWS[ROWS.length - 1],
255 ColumnAggregationWithErrorsSumResponse.getDefaultInstance(),
256 new Batch.Callback<ColumnAggregationWithErrorsSumResponse>() {
257
258 @Override
259 public void update(byte[] region, byte[] row,
260 ColumnAggregationWithErrorsSumResponse result) {
261 results.put(region, result);
262 }
263 });
264 } catch (Throwable t) {
265 LOG.info("Exceptions in coprocessor service", t);
266 hasError = true;
267 }
268
269 int sumResult = 0;
270 int expectedResult = 0;
271 for (Map.Entry<byte[], ColumnAggregationWithErrorsSumResponse> e : results.entrySet()) {
272 LOG.info("Got value " + e.getValue().getSum() + " for region "
273 + Bytes.toStringBinary(e.getKey()));
274 sumResult += e.getValue().getSum();
275 }
276 for (int i = 0; i < rowSeperator2; i++) {
277 expectedResult += i;
278 }
279 assertEquals("Invalid result", expectedResult, sumResult);
280 assertTrue(hasError);
281 table.close();
282 }
283 }