View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.client;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertNotNull;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.io.IOException;
26  import java.util.concurrent.ExecutorService;
27  import java.util.concurrent.Executors;
28  
29  import org.apache.hadoop.conf.Configuration;
30  import org.apache.hadoop.hbase.Cell;
31  import org.apache.hadoop.hbase.CellScanner;
32  import org.apache.hadoop.hbase.KeyValue;
33  import org.apache.hadoop.hbase.KeyValue.Type;
34  import org.apache.hadoop.hbase.TableName;
35  import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
36  import org.apache.hadoop.hbase.testclassification.SmallTests;
37  import org.junit.After;
38  import org.junit.Before;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  import org.mockito.InOrder;
42  import org.mockito.Mockito;
43  import org.mockito.invocation.InvocationOnMock;
44  import org.mockito.stubbing.Answer;
45  
46  /**
47   * Test the ClientScanner.
48   */
49  @Category(SmallTests.class)
50  public class TestClientScanner {
51  
52    Scan scan;
53    ExecutorService pool;
54    Configuration conf;
55  
56    ClusterConnection clusterConn;
57    RpcRetryingCallerFactory rpcFactory;
58    RpcControllerFactory controllerFactory;
59  
60    @Before
61    @SuppressWarnings("deprecation")
62    public void setup() throws IOException {
63      clusterConn = Mockito.mock(ClusterConnection.class);
64      rpcFactory = Mockito.mock(RpcRetryingCallerFactory.class);
65      controllerFactory = Mockito.mock(RpcControllerFactory.class);
66      pool = Executors.newSingleThreadExecutor();
67      scan = new Scan();
68      conf = new Configuration();
69      Mockito.when(clusterConn.getConfiguration()).thenReturn(conf);
70    }
71  
72    @After
73    public void teardown() {
74      if (null != pool) {
75        pool.shutdownNow();
76      }
77    }
78  
79    private static class MockClientScanner extends ClientScanner {
80  
81      private boolean rpcFinished = false;
82      private boolean rpcFinishedFired = false;
83  
84      public MockClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
85          ClusterConnection connection, RpcRetryingCallerFactory rpcFactory,
86          RpcControllerFactory controllerFactory, ExecutorService pool, int primaryOperationTimeout)
87          throws IOException {
88        super(conf, scan, tableName, connection, rpcFactory, controllerFactory, pool,
89            primaryOperationTimeout);
90      }
91  
92      @Override
93      protected boolean nextScanner(int nbRows, final boolean done) throws IOException {
94        if (!rpcFinished) {
95          return super.nextScanner(nbRows, done);
96        }
97  
98        // Enforce that we don't short-circuit more than once
99        if (rpcFinishedFired) {
100         throw new RuntimeException("Expected nextScanner to only be called once after " +
101             " short-circuit was triggered.");
102       }
103       rpcFinishedFired = true;
104       return false;
105     }
106 
107     @Override
108     protected ScannerCallableWithReplicas getScannerCallable(byte [] localStartKey,
109         int nbRows) {
110       scan.setStartRow(localStartKey);
111       ScannerCallable s =
112           new ScannerCallable(getConnection(), getTable(), scan, this.scanMetrics,
113               this.rpcControllerFactory);
114       s.setCaching(nbRows);
115       ScannerCallableWithReplicas sr = new ScannerCallableWithReplicas(getTable(), getConnection(),
116        s, pool, primaryOperationTimeout, scan,
117        getRetries(), scannerTimeout, caching, conf, caller);
118       return sr;
119     }
120 
121     public void setRpcFinished(boolean rpcFinished) {
122       this.rpcFinished = rpcFinished;
123     }
124   }
125 
126   @Test
127   @SuppressWarnings("unchecked")
128   public void testNoResultsHint() throws IOException {
129     final Result[] results = new Result[1];
130     KeyValue kv1 = new KeyValue("row".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
131         Type.Maximum);
132     results[0] = Result.create(new Cell[] {kv1});
133 
134     RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
135 
136     Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
137     Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class),
138       Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {
139         private int count = 0;
140         @Override
141         public Result[] answer(InvocationOnMock invocation) throws Throwable {
142             ScannerCallableWithReplicas callable = invocation.getArgumentAt(0,
143                 ScannerCallableWithReplicas.class);
144           switch (count) {
145             case 0: // initialize
146             case 2: // close
147               count++;
148               return null;
149             case 1:
150               count++;
151               callable.setHasMoreResultsContext(false);
152               return results;
153             default:
154               throw new RuntimeException("Expected only 2 invocations");
155           }
156         }
157     });
158 
159     // Set a much larger cache and buffer size than we'll provide
160     scan.setCaching(100);
161     scan.setMaxResultSize(1000*1000);
162 
163     try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf("table"),
164         clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
165 
166       scanner.setRpcFinished(true);
167 
168       InOrder inOrder = Mockito.inOrder(caller);
169 
170       scanner.loadCache();
171 
172       // One more call due to initializeScannerInConstruction()
173       inOrder.verify(caller, Mockito.times(2)).callWithoutRetries(
174           Mockito.any(RetryingCallable.class), Mockito.anyInt());
175 
176       assertEquals(1, scanner.cache.size());
177       Result r = scanner.cache.poll();
178       assertNotNull(r);
179       CellScanner cs = r.cellScanner();
180       assertTrue(cs.advance());
181       assertEquals(kv1, cs.current());
182       assertFalse(cs.advance());
183     }
184   }
185 
186   @Test
187   @SuppressWarnings("unchecked")
188   public void testSizeLimit() throws IOException {
189     final Result[] results = new Result[1];
190     KeyValue kv1 = new KeyValue("row".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
191         Type.Maximum);
192     results[0] = Result.create(new Cell[] {kv1});
193 
194     RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
195 
196     Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
197     Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class),
198       Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {
199         private int count = 0;
200         @Override
201         public Result[] answer(InvocationOnMock invocation) throws Throwable {
202           ScannerCallableWithReplicas callable = invocation.getArgumentAt(0,
203               ScannerCallableWithReplicas.class);
204           switch (count) {
205             case 0: // initialize
206             case 2: // close
207               count++;
208               return null;
209             case 1:
210               count++;
211               callable.setHasMoreResultsContext(true);
212               callable.setServerHasMoreResults(false);
213               return results;
214             default:
215               throw new RuntimeException("Expected only 2 invocations");
216           }
217         }
218     });
219 
220     Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
221 
222     // Set a much larger cache
223     scan.setCaching(100);
224     // The single key-value will exit the loop
225     scan.setMaxResultSize(1);
226 
227     try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf("table"),
228         clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
229 
230       // Due to initializeScannerInConstruction()
231       Mockito.verify(caller).callWithoutRetries(Mockito.any(RetryingCallable.class),
232           Mockito.anyInt());
233 
234       InOrder inOrder = Mockito.inOrder(caller);
235 
236       scanner.loadCache();
237 
238       inOrder.verify(caller, Mockito.times(2)).callWithoutRetries(
239           Mockito.any(RetryingCallable.class), Mockito.anyInt());
240 
241       assertEquals(1, scanner.cache.size());
242       Result r = scanner.cache.poll();
243       assertNotNull(r);
244       CellScanner cs = r.cellScanner();
245       assertTrue(cs.advance());
246       assertEquals(kv1, cs.current());
247       assertFalse(cs.advance());
248     }
249   }
250 
251   @Test
252   @SuppressWarnings("unchecked")
253   public void testCacheLimit() throws IOException {
254     KeyValue kv1 = new KeyValue("row1".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
255         Type.Maximum), kv2 = new KeyValue("row2".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
256         Type.Maximum), kv3 = new KeyValue("row3".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
257         Type.Maximum);
258     final Result[] results = new Result[] {Result.create(new Cell[] {kv1}),
259         Result.create(new Cell[] {kv2}), Result.create(new Cell[] {kv3})};
260 
261     RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
262 
263     Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
264     Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class),
265       Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {
266         private int count = 0;
267         @Override
268         public Result[] answer(InvocationOnMock invocation) throws Throwable {
269           ScannerCallableWithReplicas callable = invocation.getArgumentAt(0,
270               ScannerCallableWithReplicas.class);
271           switch (count) {
272             case 0: // initialize
273             case 2: // close
274               count++;
275               return null;
276             case 1:
277               count++;
278               callable.setHasMoreResultsContext(true);
279               callable.setServerHasMoreResults(false);
280               return results;
281             default:
282               throw new RuntimeException("Expected only 2 invocations");
283           }
284         }
285     });
286 
287     Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
288 
289     // Set a small cache
290     scan.setCaching(1);
291     // Set a very large size
292     scan.setMaxResultSize(1000*1000);
293 
294     try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf("table"),
295         clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
296 
297       // Due to initializeScannerInConstruction()
298       Mockito.verify(caller).callWithoutRetries(Mockito.any(RetryingCallable.class),
299           Mockito.anyInt());
300 
301       InOrder inOrder = Mockito.inOrder(caller);
302 
303       scanner.loadCache();
304 
305       // Ensures that possiblyNextScanner isn't called at the end which would trigger
306       // another call to callWithoutRetries
307       inOrder.verify(caller, Mockito.times(2)).callWithoutRetries(
308           Mockito.any(RetryingCallable.class), Mockito.anyInt());
309 
310       assertEquals(3, scanner.cache.size());
311       Result r = scanner.cache.poll();
312       assertNotNull(r);
313       CellScanner cs = r.cellScanner();
314       assertTrue(cs.advance());
315       assertEquals(kv1, cs.current());
316       assertFalse(cs.advance());
317 
318       r = scanner.cache.poll();
319       assertNotNull(r);
320       cs = r.cellScanner();
321       assertTrue(cs.advance());
322       assertEquals(kv2, cs.current());
323       assertFalse(cs.advance());
324 
325       r = scanner.cache.poll();
326       assertNotNull(r);
327       cs = r.cellScanner();
328       assertTrue(cs.advance());
329       assertEquals(kv3, cs.current());
330       assertFalse(cs.advance());
331     }
332   }
333 
334   @Test
335   @SuppressWarnings("unchecked")
336   public void testNoMoreResults() throws IOException {
337     final Result[] results = new Result[1];
338     KeyValue kv1 = new KeyValue("row".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
339         Type.Maximum);
340     results[0] = Result.create(new Cell[] {kv1});
341 
342     RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
343 
344     Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
345     Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class),
346       Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {
347         private int count = 0;
348         @Override
349         public Result[] answer(InvocationOnMock invocation) throws Throwable {
350           ScannerCallableWithReplicas callable = invocation.getArgumentAt(0,
351               ScannerCallableWithReplicas.class);
352           switch (count) {
353             case 0: // initialize
354             case 2: // close
355               count++;
356               return null;
357             case 1:
358               count++;
359               callable.setHasMoreResultsContext(true);
360               callable.setServerHasMoreResults(false);
361               return results;
362             default:
363               throw new RuntimeException("Expected only 2 invocations");
364           }
365         }
366     });
367 
368     Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
369 
370     // Set a much larger cache and buffer size than we'll provide
371     scan.setCaching(100);
372     scan.setMaxResultSize(1000*1000);
373 
374     try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf("table"),
375         clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
376 
377       // Due to initializeScannerInConstruction()
378       Mockito.verify(caller).callWithoutRetries(Mockito.any(RetryingCallable.class),
379           Mockito.anyInt());
380 
381       scanner.setRpcFinished(true);
382 
383       InOrder inOrder = Mockito.inOrder(caller);
384 
385       scanner.loadCache();
386 
387       inOrder.verify(caller, Mockito.times(2)).callWithoutRetries(
388           Mockito.any(RetryingCallable.class), Mockito.anyInt());
389 
390       assertEquals(1, scanner.cache.size());
391       Result r = scanner.cache.poll();
392       assertNotNull(r);
393       CellScanner cs = r.cellScanner();
394       assertTrue(cs.advance());
395       assertEquals(kv1, cs.current());
396       assertFalse(cs.advance());
397     }
398   }
399 
400   @Test
401   @SuppressWarnings("unchecked")
402   public void testMoreResults() throws IOException {
403     final Result[] results1 = new Result[1];
404     KeyValue kv1 = new KeyValue("row".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
405         Type.Maximum);
406     results1[0] = Result.create(new Cell[] {kv1});
407 
408     final Result[] results2 = new Result[1];
409     KeyValue kv2 = new KeyValue("row2".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
410         Type.Maximum);
411     results2[0] = Result.create(new Cell[] {kv2});
412 
413 
414     RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
415 
416     Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
417     Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class),
418         Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {
419           private int count = 0;
420           @Override
421           public Result[] answer(InvocationOnMock invocation) throws Throwable {
422             ScannerCallableWithReplicas callable = invocation.getArgumentAt(0,
423                 ScannerCallableWithReplicas.class);
424             switch (count) {
425               case 0: // initialize
426               case 3: // close
427                 count++;
428                 return null;
429               case 1:
430                 count++;
431                 callable.setHasMoreResultsContext(true);
432                 callable.setServerHasMoreResults(true);
433                 return results1;
434               case 2:
435                 count++;
436                 // The server reports back false WRT more results
437                 callable.setHasMoreResultsContext(true);
438                 callable.setServerHasMoreResults(false);
439                 return results2;
440               default:
441                 throw new RuntimeException("Expected only 2 invocations");
442             }
443           }
444       });
445 
446     // Set a much larger cache and buffer size than we'll provide
447     scan.setCaching(100);
448     scan.setMaxResultSize(1000*1000);
449 
450     try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf("table"),
451         clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
452 
453       // Due to initializeScannerInConstruction()
454       Mockito.verify(caller).callWithoutRetries(Mockito.any(RetryingCallable.class),
455           Mockito.anyInt());
456 
457       InOrder inOrder = Mockito.inOrder(caller);
458 
459       scanner.loadCache();
460 
461       inOrder.verify(caller, Mockito.times(2)).callWithoutRetries(
462           Mockito.any(RetryingCallable.class), Mockito.anyInt());
463 
464       assertEquals(1, scanner.cache.size());
465       Result r = scanner.cache.poll();
466       assertNotNull(r);
467       CellScanner cs = r.cellScanner();
468       assertTrue(cs.advance());
469       assertEquals(kv1, cs.current());
470       assertFalse(cs.advance());
471 
472       scanner.setRpcFinished(true);
473 
474       inOrder = Mockito.inOrder(caller);
475 
476       scanner.loadCache();
477 
478       inOrder.verify(caller, Mockito.times(3)).callWithoutRetries(
479           Mockito.any(RetryingCallable.class), Mockito.anyInt());
480 
481       r = scanner.cache.poll();
482       assertNotNull(r);
483       cs = r.cellScanner();
484       assertTrue(cs.advance());
485       assertEquals(kv2, cs.current());
486       assertFalse(cs.advance());
487     }
488   }
489 }