1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.hbase.TableName;
25 import org.apache.hadoop.hbase.HConstants;
26 import org.apache.hadoop.hbase.HRegionInfo;
27 import org.apache.hadoop.hbase.HRegionLocation;
28 import org.apache.hadoop.hbase.MediumTests;
29 import org.apache.hadoop.hbase.ServerName;
30 import org.apache.hadoop.hbase.util.Threads;
31 import org.junit.Assert;
32 import org.junit.Test;
33 import org.junit.experimental.categories.Category;
34 import org.mockito.Mockito;
35
36 import java.io.IOException;
37 import java.io.InterruptedIOException;
38 import java.util.ArrayList;
39 import java.util.Arrays;
40 import java.util.List;
41 import java.util.Map;
42 import java.util.concurrent.ExecutorService;
43 import java.util.concurrent.SynchronousQueue;
44 import java.util.concurrent.ThreadPoolExecutor;
45 import java.util.concurrent.TimeUnit;
46 import java.util.concurrent.atomic.AtomicBoolean;
47 import java.util.concurrent.atomic.AtomicInteger;
48
49 @Category(MediumTests.class)
50 public class TestAsyncProcess {
51 private static final TableName DUMMY_TABLE =
52 TableName.valueOf("DUMMY_TABLE");
53 private static final byte[] DUMMY_BYTES_1 = "DUMMY_BYTES_1".getBytes();
54 private static final byte[] DUMMY_BYTES_2 = "DUMMY_BYTES_2".getBytes();
55 private static final byte[] FAILS = "FAILS".getBytes();
56 private static final Configuration conf = new Configuration();
57
58 private static ServerName sn = new ServerName("localhost:10,1254");
59 private static HRegionInfo hri1 = new HRegionInfo(DUMMY_TABLE, DUMMY_BYTES_1, DUMMY_BYTES_2);
60 private static HRegionInfo hri2 =
61 new HRegionInfo(DUMMY_TABLE, DUMMY_BYTES_2, HConstants.EMPTY_END_ROW);
62 private static HRegionLocation loc1 = new HRegionLocation(hri1, sn);
63 private static HRegionLocation loc2 = new HRegionLocation(hri2, sn);
64
65 private static final String success = "success";
66 private static Exception failure = new Exception("failure");
67
68 static class MyAsyncProcess<Res> extends AsyncProcess<Res> {
69 public MyAsyncProcess(HConnection hc, AsyncProcessCallback<Res> callback, Configuration conf) {
70 super(hc, DUMMY_TABLE, new ThreadPoolExecutor(1, 10, 60, TimeUnit.SECONDS,
71 new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("test-TestAsyncProcess")),
72 callback, conf, new RpcRetryingCallerFactory(conf));
73 }
74
75 @Override
76 protected RpcRetryingCaller<MultiResponse> createCaller(MultiServerCallable<Row> callable) {
77 final MultiResponse mr = createMultiResponse(callable.getLocation(), callable.getMulti());
78 return new RpcRetryingCaller<MultiResponse>(conf) {
79 @Override
80 public MultiResponse callWithoutRetries( RetryingCallable<MultiResponse> callable)
81 throws IOException, RuntimeException {
82 return mr;
83 }
84 };
85 }
86 }
87
88 static MultiResponse createMultiResponse(final HRegionLocation loc,
89 final MultiAction<Row> multi) {
90 final MultiResponse mr = new MultiResponse();
91 for (Map.Entry<byte[], List<Action<Row>>> entry : multi.actions.entrySet()) {
92 for (Action a : entry.getValue()) {
93 if (Arrays.equals(FAILS, a.getAction().getRow())) {
94 mr.add(loc.getRegionInfo().getRegionName(), a.getOriginalIndex(), failure);
95 } else {
96 mr.add(loc.getRegionInfo().getRegionName(), a.getOriginalIndex(), success);
97 }
98 }
99 }
100 return mr;
101 }
102
103
104
105
106 static class MyConnectionImpl extends HConnectionManager.HConnectionImplementation {
107 MyAsyncProcess<?> ap;
108 final static Configuration c = new Configuration();
109
110 static {
111 c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
112 }
113
114 protected MyConnectionImpl() {
115 super(c);
116 }
117
118 protected MyConnectionImpl(Configuration conf) {
119 super(conf);
120 }
121
122 @Override
123 protected <R> AsyncProcess createAsyncProcess(TableName tableName,
124 ExecutorService pool,
125 AsyncProcess.AsyncProcessCallback<R> callback,
126 Configuration conf) {
127 ap = new MyAsyncProcess<R>(this, callback, conf);
128 return ap;
129 }
130
131 @Override
132 public HRegionLocation locateRegion(final TableName tableName,
133 final byte[] row) {
134 return loc1;
135 }
136
137 }
138
139 @Test
140 public void testSubmit() throws Exception {
141 HConnection hc = createHConnection();
142 AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
143
144 List<Put> puts = new ArrayList<Put>();
145 puts.add(createPut(true, true));
146
147 ap.submit(puts, false);
148 Assert.assertTrue(puts.isEmpty());
149 }
150
151 @Test
152 public void testSubmitWithCB() throws Exception {
153 HConnection hc = createHConnection();
154 MyCB mcb = new MyCB();
155 AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
156
157 List<Put> puts = new ArrayList<Put>();
158 puts.add(createPut(true, true));
159
160 ap.submit(puts, false);
161 Assert.assertTrue(puts.isEmpty());
162
163 while (!(mcb.successCalled.get() == 1) && !ap.hasError()) {
164 Thread.sleep(1);
165 }
166 Assert.assertEquals(mcb.successCalled.get(), 1);
167 }
168
169 @Test
170 public void testSubmitBusyRegion() throws Exception {
171 HConnection hc = createHConnection();
172 AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
173
174 List<Put> puts = new ArrayList<Put>();
175 puts.add(createPut(true, true));
176
177 ap.incTaskCounters(hri1.getEncodedName());
178 ap.submit(puts, false);
179 Assert.assertEquals(puts.size(), 1);
180
181 ap.decTaskCounters(hri1.getEncodedName());
182 ap.submit(puts, false);
183 Assert.assertTrue(puts.isEmpty());
184 }
185
186 @Test
187 public void testFail() throws Exception {
188 HConnection hc = createHConnection();
189 MyCB mcb = new MyCB();
190 AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
191
192 List<Put> puts = new ArrayList<Put>();
193 Put p = createPut(true, false);
194 puts.add(p);
195
196 ap.submit(puts, false);
197 Assert.assertTrue(puts.isEmpty());
198
199 while (!ap.hasError()) {
200 Thread.sleep(1);
201 }
202
203 Assert.assertEquals(0, mcb.successCalled.get());
204 Assert.assertEquals(2, mcb.retriableFailure.get());
205 Assert.assertEquals(1, mcb.failureCalled.get());
206
207 Assert.assertEquals(1, ap.getErrors().exceptions.size());
208 Assert.assertTrue("was: " + ap.getErrors().exceptions.get(0),
209 failure.equals(ap.getErrors().exceptions.get(0)));
210 Assert.assertTrue("was: " + ap.getErrors().exceptions.get(0),
211 failure.equals(ap.getErrors().exceptions.get(0)));
212
213 Assert.assertEquals(1, ap.getFailedOperations().size());
214 Assert.assertTrue("was: " + ap.getFailedOperations().get(0),
215 p.equals(ap.getFailedOperations().get(0)));
216 }
217
218 @Test
219 public void testWaitForNextTaskDone() throws IOException {
220 HConnection hc = createHConnection();
221 MyCB mcb = new MyCB();
222 final AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
223 ap.tasksSent.incrementAndGet();
224
225 final AtomicBoolean checkPoint = new AtomicBoolean(false);
226 final AtomicBoolean checkPoint2 = new AtomicBoolean(false);
227
228 Thread t = new Thread(){
229 @Override
230 public void run(){
231 Threads.sleep(1000);
232 Assert.assertFalse(checkPoint.get());
233 ap.tasksDone.incrementAndGet();
234 checkPoint2.set(true);
235 }
236 };
237
238 t.start();
239 ap.waitForNextTaskDone(0);
240 checkPoint.set(true);
241 while (!checkPoint2.get()){
242 Threads.sleep(1);
243 }
244 }
245
246 @Test
247 public void testSubmitTrue() throws IOException {
248 HConnection hc = createHConnection();
249 MyCB mcb = new MyCB();
250 final AsyncProcess<Object> ap = new MyAsyncProcess<Object>(hc, mcb, conf);
251 ap.tasksSent.incrementAndGet();
252 final AtomicInteger ai = new AtomicInteger(1);
253 ap.taskCounterPerRegion.put(hri1.getEncodedName(), ai);
254
255 final AtomicBoolean checkPoint = new AtomicBoolean(false);
256 final AtomicBoolean checkPoint2 = new AtomicBoolean(false);
257
258 Thread t = new Thread(){
259 @Override
260 public void run(){
261 Threads.sleep(1000);
262 Assert.assertFalse(checkPoint.get());
263 ai.decrementAndGet();
264 ap.tasksDone.incrementAndGet();
265 checkPoint2.set(true);
266 }
267 };
268
269 List<Put> puts = new ArrayList<Put>();
270 Put p = createPut(true, true);
271 puts.add(p);
272
273 ap.submit(puts, false);
274 Assert.assertFalse(puts.isEmpty());
275
276 t.start();
277
278 ap.submit(puts, true);
279 Assert.assertTrue(puts.isEmpty());
280
281 checkPoint.set(true);
282 while (!checkPoint2.get()){
283 Threads.sleep(1);
284 }
285 }
286
287 @Test
288 public void testFailAndSuccess() throws Exception {
289 HConnection hc = createHConnection();
290 MyCB mcb = new MyCB();
291 AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
292
293 List<Put> puts = new ArrayList<Put>();
294 puts.add(createPut(true, false));
295 puts.add(createPut(true, true));
296 puts.add(createPut(true, true));
297
298 ap.submit(puts, false);
299 Assert.assertTrue(puts.isEmpty());
300
301 while (!ap.hasError()) {
302 Thread.sleep(1);
303 }
304 Assert.assertEquals(mcb.successCalled.get(), 2);
305 Assert.assertEquals(mcb.retriableFailure.get(), 2);
306 Assert.assertEquals(mcb.failureCalled.get(), 1);
307
308 Assert.assertEquals(1, ap.getErrors().actions.size());
309
310
311 puts.add(createPut(true, true));
312 ap.submit(puts, false);
313 Assert.assertTrue(puts.isEmpty());
314
315 while (mcb.successCalled.get() != 3) {
316 Thread.sleep(1);
317 }
318 Assert.assertEquals(mcb.retriableFailure.get(), 2);
319 Assert.assertEquals(mcb.failureCalled.get(), 1);
320
321 ap.clearErrors();
322 Assert.assertTrue(ap.getErrors().actions.isEmpty());
323 }
324
325 @Test
326 public void testFlush() throws Exception {
327 HConnection hc = createHConnection();
328 MyCB mcb = new MyCB();
329 AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
330
331 List<Put> puts = new ArrayList<Put>();
332 puts.add(createPut(true, false));
333 puts.add(createPut(true, true));
334 puts.add(createPut(true, true));
335
336 ap.submit(puts, false);
337 ap.waitUntilDone();
338
339 Assert.assertEquals(mcb.successCalled.get(), 2);
340 Assert.assertEquals(mcb.retriableFailure.get(), 2);
341 Assert.assertEquals(mcb.failureCalled.get(), 1);
342
343 Assert.assertEquals(1, ap.getFailedOperations().size());
344 }
345
346 @Test
347 public void testMaxTask() throws Exception {
348 HConnection hc = createHConnection();
349 final AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
350
351 for (int i = 0; i < 1000; i++) {
352 ap.incTaskCounters("dummy");
353 }
354
355 final Thread myThread = Thread.currentThread();
356
357 Thread t = new Thread() {
358 public void run() {
359 Threads.sleep(2000);
360 myThread.interrupt();
361 }
362 };
363
364 List<Put> puts = new ArrayList<Put>();
365 puts.add(createPut(true, true));
366
367 t.start();
368
369 try {
370 ap.submit(puts, false);
371 Assert.fail("We should have been interrupted.");
372 } catch (InterruptedIOException expected) {
373 }
374
375 final long sleepTime = 2000;
376
377 Thread t2 = new Thread() {
378 public void run() {
379 Threads.sleep(sleepTime);
380 while (ap.tasksDone.get() > 0) {
381 ap.decTaskCounters("dummy");
382 }
383 }
384 };
385 t2.start();
386
387 long start = System.currentTimeMillis();
388 ap.submit(new ArrayList<Row>(), false);
389 long end = System.currentTimeMillis();
390
391
392 Assert.assertTrue(start + 100L + sleepTime > end);
393 }
394
395
396 private class MyCB implements AsyncProcess.AsyncProcessCallback<Object> {
397 private AtomicInteger successCalled = new AtomicInteger(0);
398 private AtomicInteger failureCalled = new AtomicInteger(0);
399 private AtomicInteger retriableFailure = new AtomicInteger(0);
400
401
402 @Override
403 public void success(int originalIndex, byte[] region, Row row, Object o) {
404 successCalled.incrementAndGet();
405 }
406
407 @Override
408 public boolean failure(int originalIndex, byte[] region, Row row, Throwable t) {
409 failureCalled.incrementAndGet();
410 return true;
411 }
412
413 @Override
414 public boolean retriableFailure(int originalIndex, Row row, byte[] region,
415 Throwable exception) {
416
417 return (retriableFailure.incrementAndGet() < 2);
418 }
419 }
420
421
422 private static HConnection createHConnection() throws IOException {
423 HConnection hc = Mockito.mock(HConnection.class);
424
425 Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
426 Mockito.eq(DUMMY_BYTES_1), Mockito.anyBoolean())).thenReturn(loc1);
427 Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
428 Mockito.eq(DUMMY_BYTES_1))).thenReturn(loc1);
429
430 Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
431 Mockito.eq(DUMMY_BYTES_2), Mockito.anyBoolean())).thenReturn(loc2);
432 Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
433 Mockito.eq(DUMMY_BYTES_2))).thenReturn(loc2);
434
435 Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
436 Mockito.eq(FAILS), Mockito.anyBoolean())).thenReturn(loc2);
437 Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
438 Mockito.eq(FAILS))).thenReturn(loc2);
439
440 return hc;
441 }
442
443 @Test
444 public void testHTablePutSuccess() throws Exception {
445 HTable ht = Mockito.mock(HTable.class);
446 HConnection hc = createHConnection();
447 ht.ap = new MyAsyncProcess<Object>(hc, null, conf);
448
449 Put put = createPut(true, true);
450
451 Assert.assertEquals(0, ht.getWriteBufferSize());
452 ht.put(put);
453 Assert.assertEquals(0, ht.getWriteBufferSize());
454 }
455
456 private void doHTableFailedPut(boolean bufferOn) throws Exception {
457 HTable ht = new HTable();
458 HConnection hc = createHConnection();
459 MyCB mcb = new MyCB();
460 ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
461 ht.setAutoFlush(true, true);
462 if (bufferOn) {
463 ht.setWriteBufferSize(1024L * 1024L);
464 } else {
465 ht.setWriteBufferSize(0L);
466 }
467
468 Put put = createPut(true, false);
469
470 Assert.assertEquals(0L, ht.currentWriteBufferSize);
471 try {
472 ht.put(put);
473 if (bufferOn) {
474 ht.flushCommits();
475 }
476 Assert.fail();
477 } catch (RetriesExhaustedException expected) {
478 }
479 Assert.assertEquals(0L, ht.currentWriteBufferSize);
480 Assert.assertEquals(0, mcb.successCalled.get());
481 Assert.assertEquals(2, mcb.retriableFailure.get());
482 Assert.assertEquals(1, mcb.failureCalled.get());
483
484
485 ht.close();
486 }
487
488 @Test
489 public void testHTableFailedPutWithBuffer() throws Exception {
490 doHTableFailedPut(true);
491 }
492
493 @Test
494 public void doHTableFailedPutWithoutBuffer() throws Exception {
495 doHTableFailedPut(false);
496 }
497
498 @Test
499 public void testHTableFailedPutAndNewPut() throws Exception {
500 HTable ht = new HTable();
501 HConnection hc = createHConnection();
502 MyCB mcb = new MyCB();
503 ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
504 ht.setAutoFlush(false, true);
505 ht.setWriteBufferSize(0);
506
507 Put p = createPut(true, false);
508 ht.put(p);
509
510 ht.ap.waitUntilDone();
511
512
513
514
515
516
517 p = createPut(true, true);
518 Assert.assertEquals(0, ht.writeAsyncBuffer.size());
519 try {
520 ht.put(p);
521 Assert.fail();
522 } catch (RetriesExhaustedException expected) {
523 }
524 Assert.assertEquals("the put should not been inserted.", 0, ht.writeAsyncBuffer.size());
525 }
526
527
528 @Test
529 public void testWithNoClearOnFail() throws IOException {
530 HTable ht = new HTable();
531 HConnection hc = createHConnection();
532 MyCB mcb = new MyCB();
533 ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
534 ht.setAutoFlush(false, false);
535
536 Put p = createPut(true, false);
537 ht.put(p);
538 Assert.assertEquals(0, ht.writeAsyncBuffer.size());
539 try {
540 ht.flushCommits();
541 } catch (RetriesExhaustedWithDetailsException expected) {
542 }
543 Assert.assertEquals(1, ht.writeAsyncBuffer.size());
544
545 try {
546 ht.close();
547 } catch (RetriesExhaustedWithDetailsException expected) {
548 }
549 Assert.assertEquals(1, ht.writeAsyncBuffer.size());
550 }
551
552 @Test
553 public void testBatch() throws IOException, InterruptedException {
554 HTable ht = new HTable();
555 ht.connection = new MyConnectionImpl();
556
557 List<Put> puts = new ArrayList<Put>();
558 puts.add(createPut(true, true));
559 puts.add(createPut(true, true));
560 puts.add(createPut(true, true));
561 puts.add(createPut(true, true));
562 puts.add(createPut(true, false));
563 puts.add(createPut(true, true));
564 puts.add(createPut(true, false));
565
566 Object[] res = new Object[puts.size()];
567 try {
568 ht.processBatch(puts, res);
569 Assert.fail();
570 } catch (RetriesExhaustedException expected) {
571 }
572
573 Assert.assertEquals(res[0], success);
574 Assert.assertEquals(res[1], success);
575 Assert.assertEquals(res[2], success);
576 Assert.assertEquals(res[3], success);
577 Assert.assertEquals(res[4], failure);
578 Assert.assertEquals(res[5], success);
579 Assert.assertEquals(res[6], failure);
580 }
581
582 @Test
583 public void testErrorsServers() throws InterruptedIOException,
584 RetriesExhaustedWithDetailsException {
585 HTable ht = new HTable();
586 Configuration configuration = new Configuration(conf);
587 configuration.setBoolean(HConnectionManager.RETRIES_BY_SERVER_KEY, true);
588 configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 20);
589
590 MyConnectionImpl mci = new MyConnectionImpl(configuration);
591 ht.connection = mci;
592 ht.ap = new MyAsyncProcess<Object>(mci, null, configuration);
593
594
595 Assert.assertTrue(ht.ap.useServerTrackerForRetries);
596 Assert.assertNotNull(ht.ap.createServerErrorTracker());
597 Assert.assertTrue(ht.ap.serverTrackerTimeout > 10000);
598 ht.ap.serverTrackerTimeout = 1;
599
600
601 Put p = createPut(true, false);
602 ht.setAutoFlush(false);
603 ht.put(p);
604
605 long start = System.currentTimeMillis();
606 try {
607 ht.flushCommits();
608 Assert.fail();
609 } catch (RetriesExhaustedWithDetailsException expected) {
610 }
611
612 Assert.assertTrue((System.currentTimeMillis() - start) < 10000);
613 }
614
615
616
617
618
619
620
621 private Put createPut(boolean reg1, boolean success) {
622 Put p;
623 if (!success) {
624 p = new Put(FAILS);
625 } else if (reg1) {
626 p = new Put(DUMMY_BYTES_1);
627 } else {
628 p = new Put(DUMMY_BYTES_2);
629 }
630
631 p.add(DUMMY_BYTES_1, DUMMY_BYTES_1, DUMMY_BYTES_1);
632
633 return p;
634 }
635 }