View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.coprocessor;
20  
21  import com.google.protobuf.ByteString;
22  import com.google.protobuf.RpcController;
23  import com.google.protobuf.ServiceException;
24  import java.io.IOException;
25  import java.util.Collections;
26  import java.util.Map;
27  import java.util.NavigableMap;
28  import java.util.TreeMap;
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.hbase.TableName;
33  import org.apache.hadoop.hbase.HBaseTestingUtility;
34  import org.apache.hadoop.hbase.HColumnDescriptor;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HTableDescriptor;
38  import org.apache.hadoop.hbase.MediumTests;
39  import org.apache.hadoop.hbase.ServerName;
40  import org.apache.hadoop.hbase.client.HBaseAdmin;
41  import org.apache.hadoop.hbase.client.HTable;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.client.coprocessor.Batch;
44  import org.apache.hadoop.hbase.coprocessor.protobuf.generated.ColumnAggregationProtos;
45  import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
46  import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
47  import org.apache.hadoop.hbase.ipc.ServerRpcController;
48  import org.apache.hadoop.hbase.ipc.protobuf.generated.TestProtos;
49  import org.apache.hadoop.hbase.ipc.protobuf.generated.TestRpcServiceProtos;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.junit.AfterClass;
52  import org.junit.BeforeClass;
53  import org.junit.Test;
54  import org.junit.experimental.categories.Category;
55  
56  import static org.junit.Assert.assertEquals;
57  import static org.junit.Assert.assertNotNull;
58  import static org.junit.Assert.assertNull;
59  import static org.junit.Assert.assertTrue;
60  import static org.junit.Assert.fail;
61  
62  /**
63   * TestEndpoint: test cases to verify coprocessor Endpoint
64   */
65  @Category(MediumTests.class)
66  public class TestCoprocessorEndpoint {
67    private static final Log LOG = LogFactory.getLog(TestCoprocessorEndpoint.class);
68  
69    private static final TableName TEST_TABLE =
70        TableName.valueOf("TestTable");
71    private static final byte[] TEST_FAMILY = Bytes.toBytes("TestFamily");
72    private static final byte[] TEST_QUALIFIER = Bytes.toBytes("TestQualifier");
73    private static byte[] ROW = Bytes.toBytes("testRow");
74  
75    private static final int ROWSIZE = 20;
76    private static final int rowSeperator1 = 5;
77    private static final int rowSeperator2 = 12;
78    private static byte[][] ROWS = makeN(ROW, ROWSIZE);
79  
80    private static HBaseTestingUtility util = new HBaseTestingUtility();
81  
82    @BeforeClass
83    public static void setupBeforeClass() throws Exception {
84      // set configure to indicate which cp should be loaded
85      Configuration conf = util.getConfiguration();
86      conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
87          org.apache.hadoop.hbase.coprocessor.ColumnAggregationEndpoint.class.getName(),
88          ProtobufCoprocessorService.class.getName());
89      conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
90          ProtobufCoprocessorService.class.getName());
91      util.startMiniCluster(2);
92      HBaseAdmin admin = new HBaseAdmin(conf);
93      HTableDescriptor desc = new HTableDescriptor(TEST_TABLE);
94      desc.addFamily(new HColumnDescriptor(TEST_FAMILY));
95      admin.createTable(desc, new byte[][]{ROWS[rowSeperator1], ROWS[rowSeperator2]});
96      util.waitUntilAllRegionsAssigned(TEST_TABLE);
97      admin.close();
98  
99      HTable table = new HTable(conf, TEST_TABLE);
100     for (int i = 0; i < ROWSIZE; i++) {
101       Put put = new Put(ROWS[i]);
102       put.add(TEST_FAMILY, TEST_QUALIFIER, Bytes.toBytes(i));
103       table.put(put);
104     }
105     table.close();
106   }
107 
108   @AfterClass
109   public static void tearDownAfterClass() throws Exception {
110     util.shutdownMiniCluster();
111   }
112 
113   private Map<byte [], Long> sum(final HTable table, final byte [] family,
114       final byte [] qualifier, final byte [] start, final byte [] end)
115   throws ServiceException, Throwable {
116     return table.coprocessorService(ColumnAggregationProtos.ColumnAggregationService.class,
117         start, end,
118       new Batch.Call<ColumnAggregationProtos.ColumnAggregationService, Long>() {
119         @Override
120         public Long call(ColumnAggregationProtos.ColumnAggregationService instance)
121         throws IOException {
122           BlockingRpcCallback<ColumnAggregationProtos.SumResponse> rpcCallback =
123               new BlockingRpcCallback<ColumnAggregationProtos.SumResponse>();
124           ColumnAggregationProtos.SumRequest.Builder builder =
125             ColumnAggregationProtos.SumRequest.newBuilder();
126           builder.setFamily(ByteString.copyFrom(family));
127           if (qualifier != null && qualifier.length > 0) {
128             builder.setQualifier(ByteString.copyFrom(qualifier));
129           }
130           instance.sum(null, builder.build(), rpcCallback);
131           return rpcCallback.get().getSum();
132         }
133       });
134   }
135 
136   @Test
137   public void testAggregation() throws Throwable {
138     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
139     Map<byte[], Long> results = sum(table, TEST_FAMILY, TEST_QUALIFIER,
140       ROWS[0], ROWS[ROWS.length-1]);
141     int sumResult = 0;
142     int expectedResult = 0;
143     for (Map.Entry<byte[], Long> e : results.entrySet()) {
144       LOG.info("Got value "+e.getValue()+" for region "+Bytes.toStringBinary(e.getKey()));
145       sumResult += e.getValue();
146     }
147     for (int i = 0; i < ROWSIZE; i++) {
148       expectedResult += i;
149     }
150     assertEquals("Invalid result", expectedResult, sumResult);
151 
152     results.clear();
153 
154     // scan: for region 2 and region 3
155     results = sum(table, TEST_FAMILY, TEST_QUALIFIER,
156       ROWS[rowSeperator1], ROWS[ROWS.length-1]);
157     sumResult = 0;
158     expectedResult = 0;
159     for (Map.Entry<byte[], Long> e : results.entrySet()) {
160       LOG.info("Got value "+e.getValue()+" for region "+Bytes.toStringBinary(e.getKey()));
161       sumResult += e.getValue();
162     }
163     for (int i = rowSeperator1; i < ROWSIZE; i++) {
164       expectedResult += i;
165     }
166     assertEquals("Invalid result", expectedResult, sumResult);
167     table.close();
168   }
169 
170   @Test
171   public void testCoprocessorService() throws Throwable {
172     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
173     NavigableMap<HRegionInfo,ServerName> regions = table.getRegionLocations();
174 
175     final TestProtos.EchoRequestProto request =
176         TestProtos.EchoRequestProto.newBuilder().setMessage("hello").build();
177     final Map<byte[], String> results = Collections.synchronizedMap(
178         new TreeMap<byte[], String>(Bytes.BYTES_COMPARATOR));
179     try {
180       // scan: for all regions
181       final RpcController controller = new ServerRpcController();
182       table.coprocessorService(TestRpcServiceProtos.TestProtobufRpcProto.class,
183           ROWS[0], ROWS[ROWS.length - 1],
184           new Batch.Call<TestRpcServiceProtos.TestProtobufRpcProto, TestProtos.EchoResponseProto>() {
185             public TestProtos.EchoResponseProto call(TestRpcServiceProtos.TestProtobufRpcProto instance)
186                 throws IOException {
187               LOG.debug("Default response is " + TestProtos.EchoRequestProto.getDefaultInstance());
188               BlockingRpcCallback<TestProtos.EchoResponseProto> callback = new BlockingRpcCallback<TestProtos.EchoResponseProto>();
189               instance.echo(controller, request, callback);
190               TestProtos.EchoResponseProto response = callback.get();
191               LOG.debug("Batch.Call returning result " + response);
192               return response;
193             }
194           },
195           new Batch.Callback<TestProtos.EchoResponseProto>() {
196             public void update(byte[] region, byte[] row, TestProtos.EchoResponseProto result) {
197               assertNotNull(result);
198               assertEquals("hello", result.getMessage());
199               results.put(region, result.getMessage());
200             }
201           }
202       );
203       for (Map.Entry<byte[], String> e : results.entrySet()) {
204         LOG.info("Got value "+e.getValue()+" for region "+Bytes.toStringBinary(e.getKey()));
205       }
206       assertEquals(3, results.size());
207       for (HRegionInfo info : regions.navigableKeySet()) {
208         LOG.info("Region info is "+info.getRegionNameAsString());
209         assertTrue(results.containsKey(info.getRegionName()));
210       }
211       results.clear();
212 
213       // scan: for region 2 and region 3
214       table.coprocessorService(TestRpcServiceProtos.TestProtobufRpcProto.class,
215           ROWS[rowSeperator1], ROWS[ROWS.length - 1],
216           new Batch.Call<TestRpcServiceProtos.TestProtobufRpcProto, TestProtos.EchoResponseProto>() {
217             public TestProtos.EchoResponseProto call(TestRpcServiceProtos.TestProtobufRpcProto instance)
218                 throws IOException {
219               LOG.debug("Default response is " + TestProtos.EchoRequestProto.getDefaultInstance());
220               BlockingRpcCallback<TestProtos.EchoResponseProto> callback = new BlockingRpcCallback<TestProtos.EchoResponseProto>();
221               instance.echo(controller, request, callback);
222               TestProtos.EchoResponseProto response = callback.get();
223               LOG.debug("Batch.Call returning result " + response);
224               return response;
225             }
226           },
227           new Batch.Callback<TestProtos.EchoResponseProto>() {
228             public void update(byte[] region, byte[] row, TestProtos.EchoResponseProto result) {
229               assertNotNull(result);
230               assertEquals("hello", result.getMessage());
231               results.put(region, result.getMessage());
232             }
233           }
234       );
235       for (Map.Entry<byte[], String> e : results.entrySet()) {
236         LOG.info("Got value "+e.getValue()+" for region "+Bytes.toStringBinary(e.getKey()));
237       }
238       assertEquals(2, results.size());
239     } finally {
240       table.close();
241     }
242   }
243 
244   @Test
245   public void testCoprocessorServiceNullResponse() throws Throwable {
246     HTable table = new HTable(util.getConfiguration(), TEST_TABLE);
247     NavigableMap<HRegionInfo,ServerName> regions = table.getRegionLocations();
248 
249     final TestProtos.EchoRequestProto request =
250         TestProtos.EchoRequestProto.newBuilder().setMessage("hello").build();
251     try {
252       // scan: for all regions
253       final RpcController controller = new ServerRpcController();
254       // test that null results are supported
255       Map<byte[], String> results = table.coprocessorService(TestRpcServiceProtos.TestProtobufRpcProto.class,
256           ROWS[0], ROWS[ROWS.length - 1],
257           new Batch.Call<TestRpcServiceProtos.TestProtobufRpcProto, String>() {
258             public String call(TestRpcServiceProtos.TestProtobufRpcProto instance)
259                 throws IOException {
260               BlockingRpcCallback<TestProtos.EchoResponseProto> callback = new BlockingRpcCallback<TestProtos.EchoResponseProto>();
261               instance.echo(controller, request, callback);
262               TestProtos.EchoResponseProto response = callback.get();
263               LOG.debug("Batch.Call got result " + response);
264               return null;
265             }
266           }
267       );
268       for (Map.Entry<byte[], String> e : results.entrySet()) {
269         LOG.info("Got value "+e.getValue()+" for region "+Bytes.toStringBinary(e.getKey()));
270       }
271       assertEquals(3, results.size());
272       for (HRegionInfo info : regions.navigableKeySet()) {
273         LOG.info("Region info is "+info.getRegionNameAsString());
274         assertTrue(results.containsKey(info.getRegionName()));
275         assertNull(results.get(info.getRegionName()));
276       }
277     } finally {
278       table.close();
279     }
280   }
281 
282   @Test
283   public void testMasterCoprocessorService() throws Throwable {
284     HBaseAdmin admin = util.getHBaseAdmin();
285     final TestProtos.EchoRequestProto request =
286         TestProtos.EchoRequestProto.newBuilder().setMessage("hello").build();
287     TestRpcServiceProtos.TestProtobufRpcProto.BlockingInterface service =
288         TestRpcServiceProtos.TestProtobufRpcProto.newBlockingStub(admin.coprocessorService());
289     assertEquals("hello", service.echo(null, request).getMessage());
290     admin.close();
291   }
292 
293   @Test
294   public void testCoprocessorError() throws Exception {
295     Configuration configuration = new Configuration(util.getConfiguration());
296     // Make it not retry forever
297     configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
298     HTable table = new HTable(configuration, TEST_TABLE);
299 
300     try {
301       CoprocessorRpcChannel protocol = table.coprocessorService(ROWS[0]);
302 
303       TestRpcServiceProtos.TestProtobufRpcProto.BlockingInterface service =
304           TestRpcServiceProtos.TestProtobufRpcProto.newBlockingStub(protocol);
305 
306       service.error(null, TestProtos.EmptyRequestProto.getDefaultInstance());
307       fail("Should have thrown an exception");
308     } catch (ServiceException e) {
309     } finally {
310       table.close();
311     }
312   }
313 
314   @Test
315   public void testMasterCoprocessorError() throws Throwable {
316     HBaseAdmin admin = util.getHBaseAdmin();
317     TestRpcServiceProtos.TestProtobufRpcProto.BlockingInterface service =
318         TestRpcServiceProtos.TestProtobufRpcProto.newBlockingStub(admin.coprocessorService());
319     try {
320       service.error(null, TestProtos.EmptyRequestProto.getDefaultInstance());
321       fail("Should have thrown an exception");
322     } catch (ServiceException e) {
323     } finally {
324       admin.close();
325     }
326   }
327 
328   private static byte[][] makeN(byte[] base, int n) {
329     byte[][] ret = new byte[n][];
330     for (int i = 0; i < n; i++) {
331       ret[i] = Bytes.add(base, Bytes.toBytes(String.format("%02d", i)));
332     }
333     return ret;
334   }
335 
336 }
337