1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.coprocessor;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertNotNull;
23 import static org.junit.Assert.assertTrue;
24
25 import java.util.Collections;
26 import java.util.Map;
27 import java.util.TreeMap;
28
29 import com.google.protobuf.HBaseZeroCopyByteString;
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HTableDescriptor;
36 import org.apache.hadoop.hbase.MediumTests;
37 import org.apache.hadoop.hbase.TableName;
38 import org.apache.hadoop.hbase.client.HBaseAdmin;
39 import org.apache.hadoop.hbase.client.HTable;
40 import org.apache.hadoop.hbase.client.Put;
41 import org.apache.hadoop.hbase.client.coprocessor.Batch;
42 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationProtos;
43 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationProtos.SumResponse;
44 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithErrorsProtos;
45 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos;
46 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationWithNullResponseProtos.ColumnAggregationServiceNullResponse;
47 import org.apache.hadoop.hbase.util.Bytes;
48 import org.junit.AfterClass;
49 import org.junit.BeforeClass;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52
53 import com.google.protobuf.ServiceException;
54
55
56
57
58 @Category(MediumTests.class)
59 public class TestBatchCoprocessorEndpoint {
60 private static final Log LOG = LogFactory.getLog(TestBatchCoprocessorEndpoint.class);
61
62 private static final TableName TEST_TABLE =
63 TableName.valueOf("TestTable");
64 private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily");
65 private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier");
66 private static byte[] ROW = Bytes.toBytes("testRow");
67
68 private static final int ROWSIZE = 20;
69 private static final int rowSeperator1 = 5;
70 private static final int rowSeperator2 = 12;
71 private static byte[][] ROWS = makeN(ROW, ROWSIZE);
72
73 private static HBaseTestingUtility util = new HBaseTestingUtility();
74
75 @BeforeClass
76 public static void setupBeforeClass() throws Exception {
77
78 Configuration conf = util.getConfiguration();
79 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
80 org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint.class.getName(),
81 ProtobufCoprocessorService.class.getName(),
82 ColumnAggregationEndpointWithErrors.class.getName(),
83 ColumnAggregationEndpointNullResponse.class.getName());
84 conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
85 ProtobufCoprocessorService.class.getName());
86 util.startMiniCluster(2);
87 HBaseAdmin admin = new HBaseAdmin(conf);
88 HTableDescriptor desc = new HTableDescriptor(TEST_TABLE);
89 desc.addFamily(new HColumnDescriptor(TEST_FAMILY));
90 admin.createTable(desc, new byte[][]{ROWS[rowSeperator1], ROWS[rowSeperator2]});
91 util.waitUntilAllRegionsAssigned(TEST_TABLE);
92 admin.close();
93
94 HTable table = new HTable(conf, TEST_TABLE);
95 for (int i = 0; i < ROWSIZE; i++) {
96 Put put = new Put(ROWS[i]);
97 put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(i));
98 table.put(put);
99 }
100 table.close();
101 }
102
103 @AfterClass
104 public static void tearDownAfterClass() throws Exception {
105 util.shutdownMiniCluster();
106 }
107
108 @Test
109 public void testAggregationNullResponse() throws Throwable {
110 HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
111 ColumnAggregationWithNullResponseProtos.SumRequest.Builder builder =
112 ColumnAggregationWithNullResponseProtos.SumRequest
113 .newBuilder();
114 builder.setFamily(HBaseZeroCopyByteString.wrap(TEST_FAMILY));
115 if (TEST_QUALIFIER != null && TEST_QUALIFIER.length > 0) {
116 builder.setQualifier(HBaseZeroCopyByteString.wrap(TEST_QUALIFIER));
117 }
118 Map<byte[], ColumnAggregationWithNullResponseProtos.SumResponse> results =
119 table.batchCoprocessorService(
120 ColumnAggregationServiceNullResponse.getDescriptor().findMethodByName("sum"),
121 builder.build(), ROWS[0], ROWS[ROWS.length - 1],
122 ColumnAggregationWithNullResponseProtos.SumResponse.getDefaultInstance());
123
124 int sumResult = 0;
125 int expectedResult = 0;
126 for (Map.Entry<byte[], ColumnAggregationWithNullResponseProtos.SumResponse> e :
127 results.entrySet()) {
128 LOG.info("Got value " + e.getValue().getSum() + " for region "
129 + Bytes.toStringBinary(e.getKey()));
130 sumResult += e.getValue().getSum();
131 }
132 for (int i = 0; i < rowSeperator2; i++) {
133 expectedResult += i;
134 }
135 assertEquals("Invalid result", expectedResult, sumResult);
136 table.close();
137 }
138
139 private static byte[][] makeN(byte[] base, int n) {
140 byte[][] ret = new byte[n][];
141 for (int i = 0; i < n; i++) {
142 ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%02d", i)));
143 }
144 return ret;
145 }
146
147 private Map<byte[], SumResponse> sum(final HTable table, final byte[] family,
148 final byte[] qualifier, final byte[] start, final byte[] end) throws ServiceException,
149 Throwable {
150 ColumnAggregationProtos.SumRequest.Builder builder = ColumnAggregationProtos.SumRequest
151 .newBuilder();
152 builder.setFamily(HBaseZeroCopyByteString.wrap(family));
153 if (qualifier != null && qualifier.length > 0) {
154 builder.setQualifier(HBaseZeroCopyByteString.wrap(qualifier));
155 }
156 return table.batchCoprocessorService(
157 ColumnAggregationProtos.ColumnAggregationService.getDescriptor().findMethodByName("sum"),
158 builder.build(), start, end, ColumnAggregationProtos.SumResponse.getDefaultInstance());
159 }
160
161 @Test
162 public void testAggregationWithReturnValue() throws Throwable {
163 HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
164 Map<byte[], SumResponse> results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[0],
165 ROWS[ROWS.length - 1]);
166 int sumResult = 0;
167 int expectedResult = 0;
168 for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
169 LOG.info("Got value " + e.getValue().getSum() + " for region "
170 + Bytes.toStringBinary(e.getKey()));
171 sumResult += e.getValue().getSum();
172 }
173 for (int i = 0; i < ROWSIZE; i++) {
174 expectedResult += i;
175 }
176 assertEquals("Invalid result", expectedResult, sumResult);
177
178 results.clear();
179
180
181 results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[rowSeperator1],
182 ROWS[ROWS.length - 1]);
183 sumResult = 0;
184 expectedResult = 0;
185 for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
186 LOG.info("Got value " + e.getValue().getSum() + " for region "
187 + Bytes.toStringBinary(e.getKey()));
188 sumResult += e.getValue().getSum();
189 }
190 for (int i = rowSeperator1; i < ROWSIZE; i++) {
191 expectedResult += i;
192 }
193 assertEquals("Invalid result", expectedResult, sumResult);
194 table.close();
195 }
196
197 @Test
198 public void testAggregation() throws Throwable {
199 HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
200 Map<byte[], SumResponse> results = sum(table, TEST_FAMILY, TEST_QUALIFIER,
201 ROWS[0], ROWS[ROWS.length - 1]);
202 int sumResult = 0;
203 int expectedResult = 0;
204 for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
205 LOG.info("Got value " + e.getValue().getSum() + " for region "
206 + Bytes.toStringBinary(e.getKey()));
207 sumResult += e.getValue().getSum();
208 }
209 for (int i = 0; i < ROWSIZE; i++) {
210 expectedResult += i;
211 }
212 assertEquals("Invalid result", expectedResult, sumResult);
213
214
215 results = sum(table, TEST_FAMILY, TEST_QUALIFIER, ROWS[rowSeperator1], ROWS[ROWS.length - 1]);
216 sumResult = 0;
217 expectedResult = 0;
218 for (Map.Entry<byte[], SumResponse> e : results.entrySet()) {
219 LOG.info("Got value " + e.getValue().getSum() + " for region "
220 + Bytes.toStringBinary(e.getKey()));
221 sumResult += e.getValue().getSum();
222 }
223 for (int i = rowSeperator1; i < ROWSIZE; i++) {
224 expectedResult += i;
225 }
226 assertEquals("Invalid result", expectedResult, sumResult);
227 table.close();
228 }
229
230 @Test
231 public void testAggregationWithErrors() throws Throwable {
232 HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
233 final Map<byte[], ColumnAggregationWithErrorsProtos.SumResponse> results =
234 Collections.synchronizedMap(
235 new TreeMap<byte[], ColumnAggregationWithErrorsProtos.SumResponse>(
236 Bytes.BYTES_COMPARATOR
237 ));
238 ColumnAggregationWithErrorsProtos.SumRequest.Builder builder =
239 ColumnAggregationWithErrorsProtos.SumRequest
240 .newBuilder();
241 builder.setFamily(HBaseZeroCopyByteString.wrap(TEST_FAMILY));
242 if (TEST_QUALIFIER != null && TEST_QUALIFIER.length > 0) {
243 builder.setQualifier(HBaseZeroCopyByteString.wrap(TEST_QUALIFIER));
244 }
245
246 boolean hasError = false;
247 try {
248 table.batchCoprocessorService(
249 ColumnAggregationWithErrorsProtos.ColumnAggregationServiceWithErrors.getDescriptor()
250 .findMethodByName("sum"),
251 builder.build(), ROWS[0], ROWS[ROWS.length - 1],
252 ColumnAggregationWithErrorsProtos.SumResponse.getDefaultInstance(),
253 new Batch.Callback<ColumnAggregationWithErrorsProtos.SumResponse>() {
254
255 @Override
256 public void update(byte[] region, byte[] row,
257 ColumnAggregationWithErrorsProtos.SumResponse result) {
258 results.put(region, result);
259 }
260 });
261 } catch (Throwable t) {
262 LOG.info("Exceptions in coprocessor service", t);
263 hasError = true;
264 }
265
266 int sumResult = 0;
267 int expectedResult = 0;
268 for (Map.Entry<byte[], ColumnAggregationWithErrorsProtos.SumResponse> e : results.entrySet()) {
269 LOG.info("Got value " + e.getValue().getSum() + " for region "
270 + Bytes.toStringBinary(e.getKey()));
271 sumResult += e.getValue().getSum();
272 }
273 for (int i = 0; i < rowSeperator2; i++) {
274 expectedResult += i;
275 }
276 assertEquals("Invalid result", expectedResult, sumResult);
277 assertTrue(hasError);
278 table.close();
279 }
280 }