1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.client;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertNotNull;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.util.concurrent.ExecutorService;
27 import java.util.concurrent.Executors;
28
29 import org.apache.hadoop.conf.Configuration;
30 import org.apache.hadoop.hbase.Cell;
31 import org.apache.hadoop.hbase.CellScanner;
32 import org.apache.hadoop.hbase.KeyValue;
33 import org.apache.hadoop.hbase.KeyValue.Type;
34 import org.apache.hadoop.hbase.TableName;
35 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
36 import org.apache.hadoop.hbase.testclassification.SmallTests;
37 import org.junit.After;
38 import org.junit.Before;
39 import org.junit.Test;
40 import org.junit.experimental.categories.Category;
41 import org.mockito.InOrder;
42 import org.mockito.Mockito;
43 import org.mockito.invocation.InvocationOnMock;
44 import org.mockito.stubbing.Answer;
45
46
47
48
49 @Category(SmallTests.class)
50 public class TestClientScanner {
51
52 Scan scan;
53 ExecutorService pool;
54 Configuration conf;
55
56 ClusterConnection clusterConn;
57 RpcRetryingCallerFactory rpcFactory;
58 RpcControllerFactory controllerFactory;
59
60 @Before
61 @SuppressWarnings("deprecation")
62 public void setup() throws IOException {
63 clusterConn = Mockito.mock(ClusterConnection.class);
64 rpcFactory = Mockito.mock(RpcRetryingCallerFactory.class);
65 controllerFactory = Mockito.mock(RpcControllerFactory.class);
66 pool = Executors.newSingleThreadExecutor();
67 scan = new Scan();
68 conf = new Configuration();
69 Mockito.when(clusterConn.getConfiguration()).thenReturn(conf);
70 }
71
72 @After
73 public void teardown() {
74 if (null != pool) {
75 pool.shutdownNow();
76 }
77 }
78
79 private static class MockClientScanner extends ClientScanner {
80
81 private boolean rpcFinished = false;
82 private boolean rpcFinishedFired = false;
83
84 public MockClientScanner(final Configuration conf, final Scan scan, final TableName tableName,
85 ClusterConnection connection, RpcRetryingCallerFactory rpcFactory,
86 RpcControllerFactory controllerFactory, ExecutorService pool, int primaryOperationTimeout)
87 throws IOException {
88 super(conf, scan, tableName, connection, rpcFactory, controllerFactory, pool,
89 primaryOperationTimeout);
90 }
91
92 @Override
93 protected boolean nextScanner(int nbRows, final boolean done) throws IOException {
94 if (!rpcFinished) {
95 return super.nextScanner(nbRows, done);
96 }
97
98
99 if (rpcFinishedFired) {
100 throw new RuntimeException("Expected nextScanner to only be called once after " +
101 " short-circuit was triggered.");
102 }
103 rpcFinishedFired = true;
104 return false;
105 }
106
107 @Override
108 protected ScannerCallableWithReplicas getScannerCallable(byte [] localStartKey,
109 int nbRows) {
110 scan.setStartRow(localStartKey);
111 ScannerCallable s =
112 new ScannerCallable(getConnection(), getTable(), scan, this.scanMetrics,
113 this.rpcControllerFactory);
114 s.setCaching(nbRows);
115 ScannerCallableWithReplicas sr = new ScannerCallableWithReplicas(getTable(), getConnection(),
116 s, pool, primaryOperationTimeout, scan,
117 getRetries(), scannerTimeout, caching, conf, caller);
118 return sr;
119 }
120
121 public void setRpcFinished(boolean rpcFinished) {
122 this.rpcFinished = rpcFinished;
123 }
124 }
125
126 @Test
127 @SuppressWarnings("unchecked")
128 public void testNoResultsHint() throws IOException {
129 final Result[] results = new Result[1];
130 KeyValue kv1 = new KeyValue("row".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
131 Type.Maximum);
132 results[0] = Result.create(new Cell[] {kv1});
133
134 RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
135
136 Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
137 Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class),
138 Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {
139 private int count = 0;
140 @Override
141 public Result[] answer(InvocationOnMock invocation) throws Throwable {
142 ScannerCallableWithReplicas callable = invocation.getArgumentAt(0,
143 ScannerCallableWithReplicas.class);
144 switch (count) {
145 case 0:
146 case 2:
147 count++;
148 return null;
149 case 1:
150 count++;
151 callable.setHasMoreResultsContext(false);
152 return results;
153 default:
154 throw new RuntimeException("Expected only 2 invocations");
155 }
156 }
157 });
158
159
160 scan.setCaching(100);
161 scan.setMaxResultSize(1000*1000);
162
163 try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf("table"),
164 clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
165
166 scanner.setRpcFinished(true);
167
168 InOrder inOrder = Mockito.inOrder(caller);
169
170 scanner.loadCache();
171
172
173 inOrder.verify(caller, Mockito.times(2)).callWithoutRetries(
174 Mockito.any(RetryingCallable.class), Mockito.anyInt());
175
176 assertEquals(1, scanner.cache.size());
177 Result r = scanner.cache.poll();
178 assertNotNull(r);
179 CellScanner cs = r.cellScanner();
180 assertTrue(cs.advance());
181 assertEquals(kv1, cs.current());
182 assertFalse(cs.advance());
183 }
184 }
185
186 @Test
187 @SuppressWarnings("unchecked")
188 public void testSizeLimit() throws IOException {
189 final Result[] results = new Result[1];
190 KeyValue kv1 = new KeyValue("row".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
191 Type.Maximum);
192 results[0] = Result.create(new Cell[] {kv1});
193
194 RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
195
196 Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
197 Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class),
198 Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {
199 private int count = 0;
200 @Override
201 public Result[] answer(InvocationOnMock invocation) throws Throwable {
202 ScannerCallableWithReplicas callable = invocation.getArgumentAt(0,
203 ScannerCallableWithReplicas.class);
204 switch (count) {
205 case 0:
206 case 2:
207 count++;
208 return null;
209 case 1:
210 count++;
211 callable.setHasMoreResultsContext(true);
212 callable.setServerHasMoreResults(false);
213 return results;
214 default:
215 throw new RuntimeException("Expected only 2 invocations");
216 }
217 }
218 });
219
220 Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
221
222
223 scan.setCaching(100);
224
225 scan.setMaxResultSize(1);
226
227 try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf("table"),
228 clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
229
230
231 Mockito.verify(caller).callWithoutRetries(Mockito.any(RetryingCallable.class),
232 Mockito.anyInt());
233
234 InOrder inOrder = Mockito.inOrder(caller);
235
236 scanner.loadCache();
237
238 inOrder.verify(caller, Mockito.times(2)).callWithoutRetries(
239 Mockito.any(RetryingCallable.class), Mockito.anyInt());
240
241 assertEquals(1, scanner.cache.size());
242 Result r = scanner.cache.poll();
243 assertNotNull(r);
244 CellScanner cs = r.cellScanner();
245 assertTrue(cs.advance());
246 assertEquals(kv1, cs.current());
247 assertFalse(cs.advance());
248 }
249 }
250
251 @Test
252 @SuppressWarnings("unchecked")
253 public void testCacheLimit() throws IOException {
254 KeyValue kv1 = new KeyValue("row1".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
255 Type.Maximum), kv2 = new KeyValue("row2".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
256 Type.Maximum), kv3 = new KeyValue("row3".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
257 Type.Maximum);
258 final Result[] results = new Result[] {Result.create(new Cell[] {kv1}),
259 Result.create(new Cell[] {kv2}), Result.create(new Cell[] {kv3})};
260
261 RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
262
263 Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
264 Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class),
265 Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {
266 private int count = 0;
267 @Override
268 public Result[] answer(InvocationOnMock invocation) throws Throwable {
269 ScannerCallableWithReplicas callable = invocation.getArgumentAt(0,
270 ScannerCallableWithReplicas.class);
271 switch (count) {
272 case 0:
273 case 2:
274 count++;
275 return null;
276 case 1:
277 count++;
278 callable.setHasMoreResultsContext(true);
279 callable.setServerHasMoreResults(false);
280 return results;
281 default:
282 throw new RuntimeException("Expected only 2 invocations");
283 }
284 }
285 });
286
287 Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
288
289
290 scan.setCaching(1);
291
292 scan.setMaxResultSize(1000*1000);
293
294 try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf("table"),
295 clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
296
297
298 Mockito.verify(caller).callWithoutRetries(Mockito.any(RetryingCallable.class),
299 Mockito.anyInt());
300
301 InOrder inOrder = Mockito.inOrder(caller);
302
303 scanner.loadCache();
304
305
306
307 inOrder.verify(caller, Mockito.times(2)).callWithoutRetries(
308 Mockito.any(RetryingCallable.class), Mockito.anyInt());
309
310 assertEquals(3, scanner.cache.size());
311 Result r = scanner.cache.poll();
312 assertNotNull(r);
313 CellScanner cs = r.cellScanner();
314 assertTrue(cs.advance());
315 assertEquals(kv1, cs.current());
316 assertFalse(cs.advance());
317
318 r = scanner.cache.poll();
319 assertNotNull(r);
320 cs = r.cellScanner();
321 assertTrue(cs.advance());
322 assertEquals(kv2, cs.current());
323 assertFalse(cs.advance());
324
325 r = scanner.cache.poll();
326 assertNotNull(r);
327 cs = r.cellScanner();
328 assertTrue(cs.advance());
329 assertEquals(kv3, cs.current());
330 assertFalse(cs.advance());
331 }
332 }
333
334 @Test
335 @SuppressWarnings("unchecked")
336 public void testNoMoreResults() throws IOException {
337 final Result[] results = new Result[1];
338 KeyValue kv1 = new KeyValue("row".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
339 Type.Maximum);
340 results[0] = Result.create(new Cell[] {kv1});
341
342 RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
343
344 Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
345 Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class),
346 Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {
347 private int count = 0;
348 @Override
349 public Result[] answer(InvocationOnMock invocation) throws Throwable {
350 ScannerCallableWithReplicas callable = invocation.getArgumentAt(0,
351 ScannerCallableWithReplicas.class);
352 switch (count) {
353 case 0:
354 case 2:
355 count++;
356 return null;
357 case 1:
358 count++;
359 callable.setHasMoreResultsContext(true);
360 callable.setServerHasMoreResults(false);
361 return results;
362 default:
363 throw new RuntimeException("Expected only 2 invocations");
364 }
365 }
366 });
367
368 Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
369
370
371 scan.setCaching(100);
372 scan.setMaxResultSize(1000*1000);
373
374 try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf("table"),
375 clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
376
377
378 Mockito.verify(caller).callWithoutRetries(Mockito.any(RetryingCallable.class),
379 Mockito.anyInt());
380
381 scanner.setRpcFinished(true);
382
383 InOrder inOrder = Mockito.inOrder(caller);
384
385 scanner.loadCache();
386
387 inOrder.verify(caller, Mockito.times(2)).callWithoutRetries(
388 Mockito.any(RetryingCallable.class), Mockito.anyInt());
389
390 assertEquals(1, scanner.cache.size());
391 Result r = scanner.cache.poll();
392 assertNotNull(r);
393 CellScanner cs = r.cellScanner();
394 assertTrue(cs.advance());
395 assertEquals(kv1, cs.current());
396 assertFalse(cs.advance());
397 }
398 }
399
400 @Test
401 @SuppressWarnings("unchecked")
402 public void testMoreResults() throws IOException {
403 final Result[] results1 = new Result[1];
404 KeyValue kv1 = new KeyValue("row".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
405 Type.Maximum);
406 results1[0] = Result.create(new Cell[] {kv1});
407
408 final Result[] results2 = new Result[1];
409 KeyValue kv2 = new KeyValue("row2".getBytes(), "cf".getBytes(), "cq".getBytes(), 1,
410 Type.Maximum);
411 results2[0] = Result.create(new Cell[] {kv2});
412
413
414 RpcRetryingCaller<Result[]> caller = Mockito.mock(RpcRetryingCaller.class);
415
416 Mockito.when(rpcFactory.<Result[]> newCaller()).thenReturn(caller);
417 Mockito.when(caller.callWithoutRetries(Mockito.any(RetryingCallable.class),
418 Mockito.anyInt())).thenAnswer(new Answer<Result[]>() {
419 private int count = 0;
420 @Override
421 public Result[] answer(InvocationOnMock invocation) throws Throwable {
422 ScannerCallableWithReplicas callable = invocation.getArgumentAt(0,
423 ScannerCallableWithReplicas.class);
424 switch (count) {
425 case 0:
426 case 3:
427 count++;
428 return null;
429 case 1:
430 count++;
431 callable.setHasMoreResultsContext(true);
432 callable.setServerHasMoreResults(true);
433 return results1;
434 case 2:
435 count++;
436
437 callable.setHasMoreResultsContext(true);
438 callable.setServerHasMoreResults(false);
439 return results2;
440 default:
441 throw new RuntimeException("Expected only 2 invocations");
442 }
443 }
444 });
445
446
447 scan.setCaching(100);
448 scan.setMaxResultSize(1000*1000);
449
450 try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf("table"),
451 clusterConn, rpcFactory, controllerFactory, pool, Integer.MAX_VALUE)) {
452
453
454 Mockito.verify(caller).callWithoutRetries(Mockito.any(RetryingCallable.class),
455 Mockito.anyInt());
456
457 InOrder inOrder = Mockito.inOrder(caller);
458
459 scanner.loadCache();
460
461 inOrder.verify(caller, Mockito.times(2)).callWithoutRetries(
462 Mockito.any(RetryingCallable.class), Mockito.anyInt());
463
464 assertEquals(1, scanner.cache.size());
465 Result r = scanner.cache.poll();
466 assertNotNull(r);
467 CellScanner cs = r.cellScanner();
468 assertTrue(cs.advance());
469 assertEquals(kv1, cs.current());
470 assertFalse(cs.advance());
471
472 scanner.setRpcFinished(true);
473
474 inOrder = Mockito.inOrder(caller);
475
476 scanner.loadCache();
477
478 inOrder.verify(caller, Mockito.times(3)).callWithoutRetries(
479 Mockito.any(RetryingCallable.class), Mockito.anyInt());
480
481 r = scanner.cache.poll();
482 assertNotNull(r);
483 cs = r.cellScanner();
484 assertTrue(cs.advance());
485 assertEquals(kv2, cs.current());
486 assertFalse(cs.advance());
487 }
488 }
489 }