View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.Iterator;
25  import java.util.List;
26  import java.util.concurrent.ExecutorService;
27  import java.util.concurrent.Executors;
28  
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.Cell;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.KeyValue;
34  import org.apache.hadoop.hbase.KeyValue.Type;
35  import org.apache.hadoop.hbase.TableName;
36  import org.apache.hadoop.hbase.client.ClientSmallScanner.SmallScannerCallableFactory;
37  import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
38  import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
39  import org.apache.hadoop.hbase.testclassification.SmallTests;
40  import org.junit.After;
41  import org.junit.Before;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  import org.mockito.Mockito;
45  import org.mockito.invocation.InvocationOnMock;
46  import org.mockito.stubbing.Answer;
47  
48  /**
49   * Test the ClientSmallReversedScanner.
50   */
51  @Category(SmallTests.class)
52  public class TestClientSmallReversedScanner {
53  
54    Scan scan;
55    ExecutorService pool;
56    Configuration conf;
57  
58    HConnection clusterConn;
59    RpcRetryingCallerFactory rpcFactory;
60    RpcControllerFactory controllerFactory;
61    RpcRetryingCaller<Result[]> caller;
62  
63    @Before
64    @SuppressWarnings("unchecked")
65    public void setup() throws IOException {
66      clusterConn = Mockito.mock(HConnection.class);
67      rpcFactory = Mockito.mock(RpcRetryingCallerFactory.class);
68      controllerFactory = Mockito.mock(RpcControllerFactory.class);
69      pool = Executors.newSingleThreadExecutor();
70      scan = new Scan();
71      conf = new Configuration();
72      Mockito.when(clusterConn.getConfiguration()).thenReturn(conf);
73      // Mock out the RpcCaller
74      caller = Mockito.mock(RpcRetryingCaller.class);
75      // Return the mock from the factory
76      Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
77    }
78  
79    @After
80    public void teardown() {
81      if (null != pool) {
82        pool.shutdownNow();
83      }
84    }
85  
86    /**
87     * Create a simple Answer which returns true the first time, and false every time after.
88     */
89    private Answer<Boolean> createTrueThenFalseAnswer() {
90      return new Answer<Boolean>() {
91        boolean first = true;
92  
93        @Override
94        public Boolean answer(InvocationOnMock invocation) {
95          if (first) {
96            first = false;
97            return true;
98          }
99          return false;
100       }
101     };
102   }
103 
104   private SmallScannerCallableFactory getFactory(
105       final RegionServerCallable<Result[]> callableWithReplicas) {
106     return new SmallScannerCallableFactory() {
107       @Override
108       public RegionServerCallable<Result[]> getCallable(final Scan sc, HConnection connection,
109           TableName table, ScanMetrics scanMetrics, byte[] localStartKey, final int cacheNum,
110           final RpcControllerFactory rpcControllerFactory) {
111         return callableWithReplicas;
112       }
113     };
114   }
115 
116   @Test
117   public void testContextPresent() throws Exception {
118     final KeyValue kv1 = new KeyValue("row1".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
119         Type.Maximum), kv2 = new KeyValue("row2".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
120         Type.Maximum), kv3 = new KeyValue("row3".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
121         Type.Maximum);
122 
123     @SuppressWarnings("unchecked")
124     RegionServerCallable<Result[]> callableWithReplicas = Mockito
125         .mock(RegionServerCallable.class);
126 
127     // Mock out the RpcCaller
128     @SuppressWarnings("unchecked")
129     RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
130     // Return the mock from the factory
131     Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
132 
133     // Intentionally leave a "default" caching size in the Scan. No matter the value, we
134     // should continue based on the server context
135 
136     SmallScannerCallableFactory factory = getFactory(callableWithReplicas);
137 
138     ClientSmallReversedScanner csrs = new ClientSmallReversedScanner(conf, scan,
139         TableName.valueOf("table"), clusterConn);
140 
141     try {
142       csrs.setRpcRetryingCaller(caller);
143       csrs.setRpcControllerFactory(controllerFactory);
144       csrs.setScannerCallableFactory(factory);
145 
146       // Return some data the first time, less the second, and none after that
147       Mockito.when(
148           caller.callWithRetries(callableWithReplicas,
149               HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD)).thenAnswer(
150           new Answer<Result[]>() {
151             int count = 0;
152 
153             @Override
154             public Result[] answer(InvocationOnMock invocation) {
155               Result[] results;
156               if (0 == count) {
157                 results = new Result[] {Result.create(new Cell[] {kv3}),
158                     Result.create(new Cell[] {kv2})};
159               } else if (1 == count) {
160                 results = new Result[] {Result.create(new Cell[] {kv1})};
161               } else {
162                 results = new Result[0];
163               }
164               count++;
165               return results;
166             }
167           });
168 
169       // Pass back the context always
170       Mockito.when(callableWithReplicas.hasMoreResultsContext()).thenReturn(true);
171       // Only have more results the first time
172       Mockito.when(callableWithReplicas.getServerHasMoreResults()).thenAnswer(
173           createTrueThenFalseAnswer());
174 
175       // A mocked HRegionInfo so ClientSmallScanner#nextScanner(...) works right
176       HRegionInfo regionInfo = Mockito.mock(HRegionInfo.class);
177       Mockito.when(callableWithReplicas.getHRegionInfo()).thenReturn(regionInfo);
178       // Trigger the "no more data" branch for #nextScanner(...)
179       Mockito.when(regionInfo.getEndKey()).thenReturn(HConstants.EMPTY_BYTE_ARRAY);
180 
181       csrs.loadCache();
182 
183       List<Result> results = csrs.cache;
184       Iterator<Result> iter = results.iterator();
185       assertEquals(3, results.size());
186       for (int i = 3; i >= 1 && iter.hasNext(); i--) {
187         Result result = iter.next();
188         byte[] row = result.getRow();
189         assertEquals("row" + i, new String(row, "UTF-8"));
190         assertEquals(1, result.getMap().size());
191       }
192       assertTrue(csrs.closed);
193     } finally {
194       csrs.close();
195     }
196   }
197 
198   @Test
199   public void testNoContextFewerRecords() throws Exception {
200     final KeyValue kv1 = new KeyValue("row1".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
201         Type.Maximum), kv2 = new KeyValue("row2".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
202         Type.Maximum), kv3 = new KeyValue("row3".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
203         Type.Maximum);
204 
205     @SuppressWarnings("unchecked")
206     RegionServerCallable<Result[]> callableWithReplicas = Mockito
207         .mock(RegionServerCallable.class);
208 
209     // While the server returns 2 records per batch, we expect more records.
210     scan.setCaching(2);
211 
212     SmallScannerCallableFactory factory = getFactory(callableWithReplicas);
213 
214     ClientSmallReversedScanner csrs = new ClientSmallReversedScanner(conf, scan,
215         TableName.valueOf("table"), clusterConn);
216 
217     try {
218       csrs.setRpcRetryingCaller(caller);
219       csrs.setRpcControllerFactory(controllerFactory);
220       csrs.setScannerCallableFactory(factory);
221 
222       // Return some data the first time, less the second, and none after that
223       Mockito.when(
224           caller.callWithRetries(callableWithReplicas,
225               HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD)).thenAnswer(
226           new Answer<Result[]>() {
227             int count = 0;
228 
229             @Override
230             public Result[] answer(InvocationOnMock invocation) {
231               Result[] results;
232               if (0 == count) {
233                 results = new Result[] {Result.create(new Cell[] {kv3}),
234                     Result.create(new Cell[] {kv2})};
235               } else if (1 == count) {
236                 // Return fewer records than expected (2)
237                 results = new Result[] {Result.create(new Cell[] {kv1})};
238               } else {
239                 throw new RuntimeException("Should not fetch a third batch from the server");
240               }
241               count++;
242               return results;
243             }
244           });
245 
246       // Server doesn't return the context
247       Mockito.when(callableWithReplicas.hasMoreResultsContext()).thenReturn(false);
248       // getServerHasMoreResults shouldn't be called when hasMoreResultsContext returns false
249       Mockito.when(callableWithReplicas.getServerHasMoreResults())
250           .thenThrow(new RuntimeException("Should not be called"));
251 
252       // A mocked HRegionInfo so ClientSmallScanner#nextScanner(...) works right
253       HRegionInfo regionInfo = Mockito.mock(HRegionInfo.class);
254       Mockito.when(callableWithReplicas.getHRegionInfo()).thenReturn(regionInfo);
255       // Trigger the "no more data" branch for #nextScanner(...)
256       Mockito.when(regionInfo.getEndKey()).thenReturn(HConstants.EMPTY_BYTE_ARRAY);
257 
258       csrs.loadCache();
259 
260       List<Result> results = csrs.cache;
261       Iterator<Result> iter = results.iterator();
262       assertEquals(2, results.size());
263       for (int i = 3; i >= 2 && iter.hasNext(); i--) {
264         Result result = iter.next();
265         byte[] row = result.getRow();
266         assertEquals("row" + i, new String(row, "UTF-8"));
267         assertEquals(1, result.getMap().size());
268       }
269 
270       // "consume" the Results
271       results.clear();
272 
273       csrs.loadCache();
274 
275       assertEquals(1, results.size());
276       Result result = results.get(0);
277       assertEquals("row1", new String(result.getRow(), "UTF-8"));
278       assertEquals(1, result.getMap().size());
279 
280       assertTrue(csrs.closed);
281     } finally {
282       csrs.close();
283     }
284   }
285 
286   @Test
287   public void testNoContextNoRecords() throws Exception {
288     @SuppressWarnings("unchecked")
289     RegionServerCallable<Result[]> callableWithReplicas = Mockito
290         .mock(RegionServerCallable.class);
291 
292     // While the server return 2 records per RPC, we expect there to be more records.
293     scan.setCaching(2);
294 
295     SmallScannerCallableFactory factory = getFactory(callableWithReplicas);
296 
297     ClientSmallReversedScanner csrs = new ClientSmallReversedScanner(conf, scan,
298         TableName.valueOf("table"), clusterConn);
299 
300     try {
301       csrs.setRpcRetryingCaller(caller);
302       csrs.setRpcControllerFactory(controllerFactory);
303       csrs.setScannerCallableFactory(factory);
304 
305       // Return some data the first time, less the second, and none after that
306       Mockito.when(
307           caller.callWithRetries(callableWithReplicas,
308               HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD)).thenReturn(new Result[0]);
309 
310       // Server doesn't return the context
311       Mockito.when(callableWithReplicas.hasMoreResultsContext()).thenReturn(false);
312       // Only have more results the first time
313       Mockito.when(callableWithReplicas.getServerHasMoreResults())
314           .thenThrow(new RuntimeException("Should not be called"));
315 
316       // A mocked HRegionInfo so ClientSmallScanner#nextScanner(...) works right
317       HRegionInfo regionInfo = Mockito.mock(HRegionInfo.class);
318       Mockito.when(callableWithReplicas.getHRegionInfo()).thenReturn(regionInfo);
319       // Trigger the "no more data" branch for #nextScanner(...)
320       Mockito.when(regionInfo.getEndKey()).thenReturn(HConstants.EMPTY_BYTE_ARRAY);
321 
322       csrs.loadCache();
323 
324       assertEquals(0, csrs.cache.size());
325       assertTrue(csrs.closed);
326     } finally {
327       csrs.close();
328     }
329   }
330 
331   @Test
332   public void testContextNoRecords() throws Exception {
333     @SuppressWarnings("unchecked")
334     RegionServerCallable<Result[]> callableWithReplicas = Mockito
335         .mock(RegionServerCallable.class);
336 
337     SmallScannerCallableFactory factory = getFactory(callableWithReplicas);
338 
339     ClientSmallReversedScanner csrs = new ClientSmallReversedScanner(conf, scan,
340         TableName.valueOf("table"), clusterConn);
341 
342     try {
343       csrs.setRpcRetryingCaller(caller);
344       csrs.setRpcControllerFactory(controllerFactory);
345       csrs.setScannerCallableFactory(factory);
346 
347       // Return some data the first time, less the second, and none after that
348       Mockito.when(
349           caller.callWithRetries(callableWithReplicas,
350               HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD)).thenReturn(new Result[0]);
351 
352       // Server doesn't return the context
353       Mockito.when(callableWithReplicas.hasMoreResultsContext()).thenReturn(true);
354       // Only have more results the first time
355       Mockito.when(callableWithReplicas.getServerHasMoreResults())
356           .thenReturn(false);
357 
358       // A mocked HRegionInfo so ClientSmallScanner#nextScanner(...) works right
359       HRegionInfo regionInfo = Mockito.mock(HRegionInfo.class);
360       Mockito.when(callableWithReplicas.getHRegionInfo()).thenReturn(regionInfo);
361       // Trigger the "no more data" branch for #nextScanner(...)
362       Mockito.when(regionInfo.getEndKey()).thenReturn(HConstants.EMPTY_BYTE_ARRAY);
363 
364       csrs.loadCache();
365 
366       assertEquals(0, csrs.cache.size());
367       assertTrue(csrs.closed);
368     } finally {
369       csrs.close();
370     }
371   }
372 }