1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.coprocessor;
20
21 import com.google.protobuf.ByteString;
22 import com.google.protobuf.RpcController;
23 import com.google.protobuf.ServiceException;
24 import java.io.IOException;
25 import java.util.Collections;
26 import java.util.Map;
27 import java.util.NavigableMap;
28 import java.util.TreeMap;
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.hbase.TableName;
33 import org.apache.hadoop.hbase.HBaseTestingUtility;
34 import org.apache.hadoop.hbase.HColumnDescriptor;
35 import org.apache.hadoop.hbase.HConstants;
36 import org.apache.hadoop.hbase.HRegionInfo;
37 import org.apache.hadoop.hbase.HTableDescriptor;
38 import org.apache.hadoop.hbase.MediumTests;
39 import org.apache.hadoop.hbase.ServerName;
40 import org.apache.hadoop.hbase.client.HBaseAdmin;
41 import org.apache.hadoop.hbase.client.HTable;
42 import org.apache.hadoop.hbase.client.Put;
43 import org.apache.hadoop.hbase.client.coprocessor.Batch;
44 import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationProtos;
45 import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
46 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
47 import org.apache.hadoop.hbase.ipc.ServerRpcController;
48 import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos;
49 import org.apache.hadoop.hbase.ipc.protobuf.generated.TestRpcServiceProtos;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.junit.AfterClass;
52 import org.junit.BeforeClass;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55
56 import static org.junit.Assert.assertEquals;
57 import static org.junit.Assert.assertNotNull;
58 import static org.junit.Assert.assertNull;
59 import static org.junit.Assert.assertTrue;
60 import static org.junit.Assert.fail;
61
62
63
64
65 @Category(MediumTests.class)
66 public class TestCoprocessorEndpoint {
67 private static final Log LOG = LogFactory.getLog(TestCoprocessorEndpoint.class);
68
69 private static final TableName TEST_TABLE =
70 TableName.valueOf("TestTable");
71 private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily");
72 private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier");
73 private static byte[] ROW = Bytes.toBytes("testRow");
74
75 private static final int ROWSIZE = 20;
76 private static final int rowSeperator1 = 5;
77 private static final int rowSeperator2 = 12;
78 private static byte[][] ROWS = makeN(ROW, ROWSIZE);
79
80 private static HBaseTestingUtility util = new HBaseTestingUtility();
81
82 @BeforeClass
83 public static void setupBeforeClass() throws Exception {
84
85 Configuration conf = util.getConfiguration();
86 conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
87 org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint.class.getName(),
88 ProtobufCoprocessorService.class.getName());
89 conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
90 ProtobufCoprocessorService.class.getName());
91 util.startMiniCluster(2);
92 HBaseAdmin admin = new HBaseAdmin(conf);
93 HTableDescriptor desc = new HTableDescriptor(TEST_TABLE);
94 desc.addFamily(new HColumnDescriptor(TEST_FAMILY));
95 admin.createTable(desc, new byte[][]{ROWS[rowSeperator1], ROWS[rowSeperator2]});
96 util.waitUntilAllRegionsAssigned(TEST_TABLE);
97 admin.close();
98
99 HTable table = new HTable(conf, TEST_TABLE);
100 for (int i = 0; i < ROWSIZE; i++) {
101 Put put = new Put(ROWS[i]);
102 put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(i));
103 table.put(put);
104 }
105 table.close();
106 }
107
108 @AfterClass
109 public static void tearDownAfterClass() throws Exception {
110 util.shutdownMiniCluster();
111 }
112
113 private Map<byte [], Long> sum(final HTable table, final byte [] family,
114 final byte [] qualifier, final byte [] start, final byte [] end)
115 throws ServiceException, Throwable {
116 return table.coprocessorService(ColumnAggregationProtos.ColumnAggregationService.class,
117 start, end,
118 new Batch.Call<ColumnAggregationProtos.ColumnAggregationService, Long>() {
119 @Override
120 public Long call(ColumnAggregationProtos.ColumnAggregationService instance)
121 throws IOException {
122 BlockingRpcCallback<ColumnAggregationProtos.SumResponse> rpcCallback =
123 new BlockingRpcCallback<ColumnAggregationProtos.SumResponse>();
124 ColumnAggregationProtos.SumRequest.Builder builder =
125 ColumnAggregationProtos.SumRequest.newBuilder();
126 builder.setFamily(ByteString.copyFrom(family));
127 if (qualifier != null && qualifier.length > 0) {
128 builder.setQualifier(ByteString.copyFrom(qualifier));
129 }
130 instance.sum(null, builder.build(), rpcCallback);
131 return rpcCallback.get().getSum();
132 }
133 });
134 }
135
136 @Test
137 public void testAggregation() throws Throwable {
138 HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
139 Map<byte[], Long> results = sum(table, TEST_FAMILY, TEST_QUALIFIER,
140 ROWS[0], ROWS[ROWS.length-1]);
141 int sumResult = 0;
142 int expectedResult = 0;
143 for (Map.Entry<byte[], Long> e : results.entrySet()) {
144 LOG.info("Got value "+e.getValue()+" for region "+Bytes.toStringBinary(e.getKey()));
145 sumResult += e.getValue();
146 }
147 for (int i = 0; i < ROWSIZE; i++) {
148 expectedResult += i;
149 }
150 assertEquals("Invalid result", expectedResult, sumResult);
151
152 results.clear();
153
154
155 results = sum(table, TEST_FAMILY, TEST_QUALIFIER,
156 ROWS[rowSeperator1], ROWS[ROWS.length-1]);
157 sumResult = 0;
158 expectedResult = 0;
159 for (Map.Entry<byte[], Long> e : results.entrySet()) {
160 LOG.info("Got value "+e.getValue()+" for region "+Bytes.toStringBinary(e.getKey()));
161 sumResult += e.getValue();
162 }
163 for (int i = rowSeperator1; i < ROWSIZE; i++) {
164 expectedResult += i;
165 }
166 assertEquals("Invalid result", expectedResult, sumResult);
167 table.close();
168 }
169
170 @Test
171 public void testCoprocessorService() throws Throwable {
172 HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
173 NavigableMap<HRegionInfo,ServerName> regions = table.getRegionLocations();
174
175 final TestProtos.EchoRequestProto request =
176 TestProtos.EchoRequestProto.newBuilder().setMessage("hello").build();
177 final Map<byte[], String> results = Collections.synchronizedMap(
178 new TreeMap<byte[], String>(Bytes.BYTES_COMPARATOR));
179 try {
180
181 final RpcController controller = new ServerRpcController();
182 table.coprocessorService(TestRpcServiceProtos.TestProtobufRpcProto.class,
183 ROWS[0], ROWS[ROWS.length - 1],
184 new Batch.Call<TestRpcServiceProtos.TestProtobufRpcProto, TestProtos.EchoResponseProto>() {
185 public TestProtos.EchoResponseProto call(TestRpcServiceProtos.TestProtobufRpcProto instance)
186 throws IOException {
187 LOG.debug("Default response is " + TestProtos.EchoRequestProto.getDefaultInstance());
188 BlockingRpcCallback<TestProtos.EchoResponseProto> callback = new BlockingRpcCallback<TestProtos.EchoResponseProto>();
189 instance.echo(controller, request, callback);
190 TestProtos.EchoResponseProto response = callback.get();
191 LOG.debug("Batch.Call returning result " + response);
192 return response;
193 }
194 },
195 new Batch.Callback<TestProtos.EchoResponseProto>() {
196 public void update(byte[] region, byte[] row, TestProtos.EchoResponseProto result) {
197 assertNotNull(result);
198 assertEquals("hello", result.getMessage());
199 results.put(region, result.getMessage());
200 }
201 }
202 );
203 for (Map.Entry<byte[], String> e : results.entrySet()) {
204 LOG.info("Got value "+e.getValue()+" for region "+Bytes.toStringBinary(e.getKey()));
205 }
206 assertEquals(3, results.size());
207 for (HRegionInfo info : regions.navigableKeySet()) {
208 LOG.info("Region info is "+info.getRegionNameAsString());
209 assertTrue(results.containsKey(info.getRegionName()));
210 }
211 results.clear();
212
213
214 table.coprocessorService(TestRpcServiceProtos.TestProtobufRpcProto.class,
215 ROWS[rowSeperator1], ROWS[ROWS.length - 1],
216 new Batch.Call<TestRpcServiceProtos.TestProtobufRpcProto, TestProtos.EchoResponseProto>() {
217 public TestProtos.EchoResponseProto call(TestRpcServiceProtos.TestProtobufRpcProto instance)
218 throws IOException {
219 LOG.debug("Default response is " + TestProtos.EchoRequestProto.getDefaultInstance());
220 BlockingRpcCallback<TestProtos.EchoResponseProto> callback = new BlockingRpcCallback<TestProtos.EchoResponseProto>();
221 instance.echo(controller, request, callback);
222 TestProtos.EchoResponseProto response = callback.get();
223 LOG.debug("Batch.Call returning result " + response);
224 return response;
225 }
226 },
227 new Batch.Callback<TestProtos.EchoResponseProto>() {
228 public void update(byte[] region, byte[] row, TestProtos.EchoResponseProto result) {
229 assertNotNull(result);
230 assertEquals("hello", result.getMessage());
231 results.put(region, result.getMessage());
232 }
233 }
234 );
235 for (Map.Entry<byte[], String> e : results.entrySet()) {
236 LOG.info("Got value "+e.getValue()+" for region "+Bytes.toStringBinary(e.getKey()));
237 }
238 assertEquals(2, results.size());
239 } finally {
240 table.close();
241 }
242 }
243
244 @Test
245 public void testCoprocessorServiceNullResponse() throws Throwable {
246 HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
247 NavigableMap<HRegionInfo,ServerName> regions = table.getRegionLocations();
248
249 final TestProtos.EchoRequestProto request =
250 TestProtos.EchoRequestProto.newBuilder().setMessage("hello").build();
251 try {
252
253 final RpcController controller = new ServerRpcController();
254
255 Map<byte[], String> results = table.coprocessorService(TestRpcServiceProtos.TestProtobufRpcProto.class,
256 ROWS[0], ROWS[ROWS.length - 1],
257 new Batch.Call<TestRpcServiceProtos.TestProtobufRpcProto, String>() {
258 public String call(TestRpcServiceProtos.TestProtobufRpcProto instance)
259 throws IOException {
260 BlockingRpcCallback<TestProtos.EchoResponseProto> callback = new BlockingRpcCallback<TestProtos.EchoResponseProto>();
261 instance.echo(controller, request, callback);
262 TestProtos.EchoResponseProto response = callback.get();
263 LOG.debug("Batch.Call got result " + response);
264 return null;
265 }
266 }
267 );
268 for (Map.Entry<byte[], String> e : results.entrySet()) {
269 LOG.info("Got value "+e.getValue()+" for region "+Bytes.toStringBinary(e.getKey()));
270 }
271 assertEquals(3, results.size());
272 for (HRegionInfo info : regions.navigableKeySet()) {
273 LOG.info("Region info is "+info.getRegionNameAsString());
274 assertTrue(results.containsKey(info.getRegionName()));
275 assertNull(results.get(info.getRegionName()));
276 }
277 } finally {
278 table.close();
279 }
280 }
281
282 @Test
283 public void testMasterCoprocessorService() throws Throwable {
284 HBaseAdmin admin = util.getHBaseAdmin();
285 final TestProtos.EchoRequestProto request =
286 TestProtos.EchoRequestProto.newBuilder().setMessage("hello").build();
287 TestRpcServiceProtos.TestProtobufRpcProto.BlockingInterface service =
288 TestRpcServiceProtos.TestProtobufRpcProto.newBlockingStub(admin.coprocessorService());
289 assertEquals("hello", service.echo(null, request).getMessage());
290 admin.close();
291 }
292
293 @Test
294 public void testCoprocessorError() throws Exception {
295 Configuration configuration = new Configuration(util.getConfiguration());
296
297 configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
298 HTable table = new HTable(configuration, TEST_TABLE);
299
300 try {
301 CoprocessorRpcChannel protocol = table.coprocessorService(ROWS[0]);
302
303 TestRpcServiceProtos.TestProtobufRpcProto.BlockingInterface service =
304 TestRpcServiceProtos.TestProtobufRpcProto.newBlockingStub(protocol);
305
306 service.error(null, TestProtos.EmptyRequestProto.getDefaultInstance());
307 fail("Should have thrown an exception");
308 } catch (ServiceException e) {
309 } finally {
310 table.close();
311 }
312 }
313
314 @Test
315 public void testMasterCoprocessorError() throws Throwable {
316 HBaseAdmin admin = util.getHBaseAdmin();
317 TestRpcServiceProtos.TestProtobufRpcProto.BlockingInterface service =
318 TestRpcServiceProtos.TestProtobufRpcProto.newBlockingStub(admin.coprocessorService());
319 try {
320 service.error(null, TestProtos.EmptyRequestProto.getDefaultInstance());
321 fail("Should have thrown an exception");
322 } catch (ServiceException e) {
323 } finally {
324 admin.close();
325 }
326 }
327
328 private static byte[][] makeN(byte[] base, int n) {
329 byte[][] ret = new byte[n][];
330 for (int i = 0; i < n; i++) {
331 ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%02d", i)));
332 }
333 return ret;
334 }
335
336 }
337