View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.TableName;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.HRegionInfo;
27  import org.apache.hadoop.hbase.HRegionLocation;
28  import org.apache.hadoop.hbase.MediumTests;
29  import org.apache.hadoop.hbase.ServerName;
30  import org.apache.hadoop.hbase.util.Threads;
31  import org.junit.Assert;
32  import org.junit.Test;
33  import org.junit.experimental.categories.Category;
34  import org.mockito.Mockito;
35  
36  import java.io.IOException;
37  import java.io.InterruptedIOException;
38  import java.util.ArrayList;
39  import java.util.Arrays;
40  import java.util.List;
41  import java.util.Map;
42  import java.util.concurrent.ExecutorService;
43  import java.util.concurrent.SynchronousQueue;
44  import java.util.concurrent.ThreadPoolExecutor;
45  import java.util.concurrent.TimeUnit;
46  import java.util.concurrent.atomic.AtomicBoolean;
47  import java.util.concurrent.atomic.AtomicInteger;
48  
49  @Category(MediumTests.class)
50  public class TestAsyncProcess {
51    private static final TableName DUMMY_TABLE =
52        TableName.valueOf("DUMMY_TABLE");
53    private static final byte[] DUMMY_BYTES_1 = "DUMMY_BYTES_1".getBytes();
54    private static final byte[] DUMMY_BYTES_2 = "DUMMY_BYTES_2".getBytes();
55    private static final byte[] FAILS = "FAILS".getBytes();
56    private static final Configuration conf = new Configuration();
57  
58    private static ServerName sn = new ServerName("localhost:10,1254");
59    private static HRegionInfo hri1 = new HRegionInfo(DUMMY_TABLE, DUMMY_BYTES_1, DUMMY_BYTES_2);
60    private static HRegionInfo hri2 =
61        new HRegionInfo(DUMMY_TABLE, DUMMY_BYTES_2, HConstants.EMPTY_END_ROW);
62    private static HRegionLocation loc1 = new HRegionLocation(hri1, sn);
63    private static HRegionLocation loc2 = new HRegionLocation(hri2, sn);
64  
65    private static final String success = "success";
66    private static Exception failure = new Exception("failure");
67  
68    static class MyAsyncProcess<Res> extends AsyncProcess<Res> {
69      public MyAsyncProcess(HConnection hc, AsyncProcessCallback<Res> callback, Configuration conf) {
70        super(hc, DUMMY_TABLE, new ThreadPoolExecutor(1, 10, 60, TimeUnit.SECONDS,
71          new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("test-TestAsyncProcess")),
72            callback, conf, new RpcRetryingCallerFactory(conf));
73      }
74  
75      @Override
76      protected RpcRetryingCaller<MultiResponse> createCaller(MultiServerCallable<Row> callable) {
77        final MultiResponse mr = createMultiResponse(callable.getLocation(), callable.getMulti());
78        return new RpcRetryingCaller<MultiResponse>(conf) {
79          @Override
80          public MultiResponse callWithoutRetries( RetryingCallable<MultiResponse> callable)
81          throws IOException, RuntimeException {
82            return mr;
83          }
84        };
85      }
86    }
87  
88    static MultiResponse createMultiResponse(final HRegionLocation loc,
89        final MultiAction<Row> multi) {
90      final MultiResponse mr = new MultiResponse();
91      for (Map.Entry<byte[], List<Action<Row>>> entry : multi.actions.entrySet()) {
92        for (Action a : entry.getValue()) {
93          if (Arrays.equals(FAILS, a.getAction().getRow())) {
94            mr.add(loc.getRegionInfo().getRegionName(), a.getOriginalIndex(), failure);
95          } else {
96            mr.add(loc.getRegionInfo().getRegionName(), a.getOriginalIndex(), success);
97          }
98        }
99      }
100     return mr;
101   }
102 
103   /**
104    * Returns our async process.
105    */
106   static class MyConnectionImpl extends HConnectionManager.HConnectionImplementation {
107     MyAsyncProcess<?> ap;
108     final static Configuration c = new Configuration();
109 
110     static {
111       c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
112     }
113 
114     protected MyConnectionImpl() {
115       super(c);
116     }
117 
118     protected MyConnectionImpl(Configuration conf) {
119       super(conf);
120     }
121 
122     @Override
123     protected <R> AsyncProcess createAsyncProcess(TableName tableName,
124                                                   ExecutorService pool,
125                                                   AsyncProcess.AsyncProcessCallback<R> callback,
126                                                   Configuration conf) {
127       ap = new MyAsyncProcess<R>(this, callback, conf);
128       return ap;
129     }
130 
131     @Override
132     public HRegionLocation locateRegion(final TableName tableName,
133                                         final byte[] row) {
134       return loc1;
135     }
136 
137   }
138 
139   @Test
140   public void testSubmit() throws Exception {
141     HConnection hc = createHConnection();
142     AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
143 
144     List<Put> puts = new ArrayList<Put>();
145     puts.add(createPut(true, true));
146 
147     ap.submit(puts, false);
148     Assert.assertTrue(puts.isEmpty());
149   }
150 
151   @Test
152   public void testSubmitWithCB() throws Exception {
153     HConnection hc = createHConnection();
154     MyCB mcb = new MyCB();
155     AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
156 
157     List<Put> puts = new ArrayList<Put>();
158     puts.add(createPut(true, true));
159 
160     ap.submit(puts, false);
161     Assert.assertTrue(puts.isEmpty());
162 
163     while (!(mcb.successCalled.get() == 1) && !ap.hasError()) {
164       Thread.sleep(1);
165     }
166     Assert.assertEquals(mcb.successCalled.get(), 1);
167   }
168 
169   @Test
170   public void testSubmitBusyRegion() throws Exception {
171     HConnection hc = createHConnection();
172     AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
173 
174     List<Put> puts = new ArrayList<Put>();
175     puts.add(createPut(true, true));
176 
177     ap.incTaskCounters(hri1.getEncodedName());
178     ap.submit(puts, false);
179     Assert.assertEquals(puts.size(), 1);
180 
181     ap.decTaskCounters(hri1.getEncodedName());
182     ap.submit(puts, false);
183     Assert.assertTrue(puts.isEmpty());
184   }
185 
186   @Test
187   public void testFail() throws Exception {
188     HConnection hc = createHConnection();
189     MyCB mcb = new MyCB();
190     AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
191 
192     List<Put> puts = new ArrayList<Put>();
193     Put p = createPut(true, false);
194     puts.add(p);
195 
196     ap.submit(puts, false);
197     Assert.assertTrue(puts.isEmpty());
198 
199     while (!ap.hasError()) {
200       Thread.sleep(1);
201     }
202 
203     Assert.assertEquals(0, mcb.successCalled.get());
204     Assert.assertEquals(2, mcb.retriableFailure.get());
205     Assert.assertEquals(1, mcb.failureCalled.get());
206 
207     Assert.assertEquals(1, ap.getErrors().exceptions.size());
208     Assert.assertTrue("was: " + ap.getErrors().exceptions.get(0),
209         failure.equals(ap.getErrors().exceptions.get(0)));
210     Assert.assertTrue("was: " + ap.getErrors().exceptions.get(0),
211         failure.equals(ap.getErrors().exceptions.get(0)));
212 
213     Assert.assertEquals(1, ap.getFailedOperations().size());
214     Assert.assertTrue("was: " + ap.getFailedOperations().get(0),
215         p.equals(ap.getFailedOperations().get(0)));
216   }
217 
218   @Test
219   public void testWaitForNextTaskDone() throws IOException {
220     HConnection hc = createHConnection();
221     MyCB mcb = new MyCB();
222     final AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
223     ap.tasksSent.incrementAndGet();
224 
225     final AtomicBoolean checkPoint = new AtomicBoolean(false);
226     final AtomicBoolean checkPoint2 = new AtomicBoolean(false);
227 
228     Thread t = new Thread(){
229       @Override
230       public void run(){
231         Threads.sleep(1000);
232         Assert.assertFalse(checkPoint.get());
233         ap.tasksDone.incrementAndGet();
234         checkPoint2.set(true);
235       }
236     };
237 
238     t.start();
239     ap.waitForNextTaskDone(0);
240     checkPoint.set(true);
241     while (!checkPoint2.get()){
242       Threads.sleep(1);
243     }
244   }
245 
246   @Test
247   public void testSubmitTrue() throws IOException {
248     HConnection hc = createHConnection();
249     MyCB mcb = new MyCB();
250     final AsyncProcess<Object> ap = new MyAsyncProcess<Object>(hc, mcb, conf);
251     ap.tasksSent.incrementAndGet();
252     final AtomicInteger ai = new AtomicInteger(1);
253     ap.taskCounterPerRegion.put(hri1.getEncodedName(), ai);
254 
255     final AtomicBoolean checkPoint = new AtomicBoolean(false);
256     final AtomicBoolean checkPoint2 = new AtomicBoolean(false);
257 
258     Thread t = new Thread(){
259       @Override
260       public void run(){
261         Threads.sleep(1000);
262         Assert.assertFalse(checkPoint.get());
263         ai.decrementAndGet();
264         ap.tasksDone.incrementAndGet();
265         checkPoint2.set(true);
266       }
267     };
268 
269     List<Put> puts = new ArrayList<Put>();
270     Put p = createPut(true, true);
271     puts.add(p);
272 
273     ap.submit(puts, false);
274     Assert.assertFalse(puts.isEmpty());
275 
276     t.start();
277 
278     ap.submit(puts, true);
279     Assert.assertTrue(puts.isEmpty());
280 
281     checkPoint.set(true);
282     while (!checkPoint2.get()){
283       Threads.sleep(1);
284     }
285   }
286 
287   @Test
288   public void testFailAndSuccess() throws Exception {
289     HConnection hc = createHConnection();
290     MyCB mcb = new MyCB();
291     AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
292 
293     List<Put> puts = new ArrayList<Put>();
294     puts.add(createPut(true, false));
295     puts.add(createPut(true, true));
296     puts.add(createPut(true, true));
297 
298     ap.submit(puts, false);
299     Assert.assertTrue(puts.isEmpty());
300 
301     while (!ap.hasError()) {
302       Thread.sleep(1);
303     }
304     Assert.assertEquals(mcb.successCalled.get(), 2);
305     Assert.assertEquals(mcb.retriableFailure.get(), 2);
306     Assert.assertEquals(mcb.failureCalled.get(), 1);
307 
308     Assert.assertEquals(1, ap.getErrors().actions.size());
309 
310 
311     puts.add(createPut(true, true));
312     ap.submit(puts, false);
313     Assert.assertTrue(puts.isEmpty());
314 
315     while (mcb.successCalled.get() != 3) {
316       Thread.sleep(1);
317     }
318     Assert.assertEquals(mcb.retriableFailure.get(), 2);
319     Assert.assertEquals(mcb.failureCalled.get(), 1);
320 
321     ap.clearErrors();
322     Assert.assertTrue(ap.getErrors().actions.isEmpty());
323   }
324 
325   @Test
326   public void testFlush() throws Exception {
327     HConnection hc = createHConnection();
328     MyCB mcb = new MyCB();
329     AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
330 
331     List<Put> puts = new ArrayList<Put>();
332     puts.add(createPut(true, false));
333     puts.add(createPut(true, true));
334     puts.add(createPut(true, true));
335 
336     ap.submit(puts, false);
337     ap.waitUntilDone();
338 
339     Assert.assertEquals(mcb.successCalled.get(), 2);
340     Assert.assertEquals(mcb.retriableFailure.get(), 2);
341     Assert.assertEquals(mcb.failureCalled.get(), 1);
342 
343     Assert.assertEquals(1, ap.getFailedOperations().size());
344   }
345 
346   @Test
347   public void testMaxTask() throws Exception {
348     HConnection hc = createHConnection();
349     final AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
350 
351     for (int i = 0; i < 1000; i++) {
352       ap.incTaskCounters("dummy");
353     }
354 
355     final Thread myThread = Thread.currentThread();
356 
357     Thread t = new Thread() {
358       public void run() {
359         Threads.sleep(2000);
360         myThread.interrupt();
361       }
362     };
363 
364     List<Put> puts = new ArrayList<Put>();
365     puts.add(createPut(true, true));
366 
367     t.start();
368 
369     try {
370       ap.submit(puts, false);
371       Assert.fail("We should have been interrupted.");
372     } catch (InterruptedIOException expected) {
373     }
374 
375     final long sleepTime = 2000;
376 
377     Thread t2 = new Thread() {
378       public void run() {
379         Threads.sleep(sleepTime);
380         while (ap.tasksDone.get() > 0) {
381           ap.decTaskCounters("dummy");
382         }
383       }
384     };
385     t2.start();
386 
387     long start = System.currentTimeMillis();
388     ap.submit(new ArrayList<Row>(), false);
389     long end = System.currentTimeMillis();
390 
391     //Adds 100 to secure us against approximate timing.
392     Assert.assertTrue(start + 100L + sleepTime > end);
393   }
394 
395 
396   private class MyCB implements AsyncProcess.AsyncProcessCallback<Object> {
397     private AtomicInteger successCalled = new AtomicInteger(0);
398     private AtomicInteger failureCalled = new AtomicInteger(0);
399     private AtomicInteger retriableFailure = new AtomicInteger(0);
400 
401 
402     @Override
403     public void success(int originalIndex, byte[] region, Row row, Object o) {
404       successCalled.incrementAndGet();
405     }
406 
407     @Override
408     public boolean failure(int originalIndex, byte[] region, Row row, Throwable t) {
409       failureCalled.incrementAndGet();
410       return true;
411     }
412 
413     @Override
414     public boolean retriableFailure(int originalIndex, Row row, byte[] region,
415                                     Throwable exception) {
416       // We retry once only.
417       return (retriableFailure.incrementAndGet() < 2);
418     }
419   }
420 
421 
422   private static HConnection createHConnection() throws IOException {
423     HConnection hc = Mockito.mock(HConnection.class);
424 
425     Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
426         Mockito.eq(DUMMY_BYTES_1), Mockito.anyBoolean())).thenReturn(loc1);
427     Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
428         Mockito.eq(DUMMY_BYTES_1))).thenReturn(loc1);
429 
430     Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
431         Mockito.eq(DUMMY_BYTES_2), Mockito.anyBoolean())).thenReturn(loc2);
432     Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
433         Mockito.eq(DUMMY_BYTES_2))).thenReturn(loc2);
434 
435     Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
436         Mockito.eq(FAILS), Mockito.anyBoolean())).thenReturn(loc2);
437     Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
438         Mockito.eq(FAILS))).thenReturn(loc2);
439 
440     return hc;
441   }
442 
443   @Test
444   public void testHTablePutSuccess() throws Exception {
445     HTable ht = Mockito.mock(HTable.class);
446     HConnection hc = createHConnection();
447     ht.ap = new MyAsyncProcess<Object>(hc, null, conf);
448 
449     Put put = createPut(true, true);
450 
451     Assert.assertEquals(0, ht.getWriteBufferSize());
452     ht.put(put);
453     Assert.assertEquals(0, ht.getWriteBufferSize());
454   }
455 
456   private void doHTableFailedPut(boolean bufferOn) throws Exception {
457     HTable ht = new HTable();
458     HConnection hc = createHConnection();
459     MyCB mcb = new MyCB(); // This allows to have some hints on what's going on.
460     ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
461     ht.setAutoFlush(true, true);
462     if (bufferOn) {
463       ht.setWriteBufferSize(1024L * 1024L);
464     } else {
465       ht.setWriteBufferSize(0L);
466     }
467 
468     Put put = createPut(true, false);
469 
470     Assert.assertEquals(0L, ht.currentWriteBufferSize);
471     try {
472       ht.put(put);
473       if (bufferOn) {
474         ht.flushCommits();
475       }
476       Assert.fail();
477     } catch (RetriesExhaustedException expected) {
478     }
479     Assert.assertEquals(0L, ht.currentWriteBufferSize);
480     Assert.assertEquals(0, mcb.successCalled.get());
481     Assert.assertEquals(2, mcb.retriableFailure.get());
482     Assert.assertEquals(1, mcb.failureCalled.get());
483 
484     // This should not raise any exception, puts have been 'received' before by the catch.
485     ht.close();
486   }
487 
488   @Test
489   public void testHTableFailedPutWithBuffer() throws Exception {
490     doHTableFailedPut(true);
491   }
492 
493   @Test
494   public void doHTableFailedPutWithoutBuffer() throws Exception {
495     doHTableFailedPut(false);
496   }
497 
498   @Test
499   public void testHTableFailedPutAndNewPut() throws Exception {
500     HTable ht = new HTable();
501     HConnection hc = createHConnection();
502     MyCB mcb = new MyCB(); // This allows to have some hints on what's going on.
503     ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
504     ht.setAutoFlush(false, true);
505     ht.setWriteBufferSize(0);
506 
507     Put p = createPut(true, false);
508     ht.put(p);
509 
510     ht.ap.waitUntilDone(); // Let's do all the retries.
511 
512     // We're testing that we're behaving as we were behaving in 0.94: sending exceptions in the
513     //  doPut if it fails.
514     // This said, it's not a very easy going behavior. For example, when we insert a list of
515     //  puts, we may raise an exception in the middle of the list. It's then up to the caller to
516     //  manage what was inserted, what was tried but failed, and what was not even tried.
517     p = createPut(true, true);
518     Assert.assertEquals(0, ht.writeAsyncBuffer.size());
519     try {
520       ht.put(p);
521       Assert.fail();
522     } catch (RetriesExhaustedException expected) {
523     }
524     Assert.assertEquals("the put should not been inserted.", 0, ht.writeAsyncBuffer.size());
525   }
526 
527 
528   @Test
529   public void testWithNoClearOnFail() throws IOException {
530     HTable ht = new HTable();
531     HConnection hc = createHConnection();
532     MyCB mcb = new MyCB();
533     ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
534     ht.setAutoFlush(false, false);
535 
536     Put p = createPut(true, false);
537     ht.put(p);
538     Assert.assertEquals(0, ht.writeAsyncBuffer.size());
539     try {
540       ht.flushCommits();
541     } catch (RetriesExhaustedWithDetailsException expected) {
542     }
543     Assert.assertEquals(1, ht.writeAsyncBuffer.size());
544 
545     try {
546       ht.close();
547     } catch (RetriesExhaustedWithDetailsException expected) {
548     }
549     Assert.assertEquals(1, ht.writeAsyncBuffer.size());
550   }
551 
552   @Test
553   public void testBatch() throws IOException, InterruptedException {
554     HTable ht = new HTable();
555     ht.connection = new MyConnectionImpl();
556 
557     List<Put> puts = new ArrayList<Put>();
558     puts.add(createPut(true, true));
559     puts.add(createPut(true, true));
560     puts.add(createPut(true, true));
561     puts.add(createPut(true, true));
562     puts.add(createPut(true, false)); // <=== the bad apple, position 4
563     puts.add(createPut(true, true));
564     puts.add(createPut(true, false)); // <=== another bad apple, position 6
565 
566     Object[] res = new Object[puts.size()];
567     try {
568       ht.processBatch(puts, res);
569       Assert.fail();
570     } catch (RetriesExhaustedException expected) {
571     }
572 
573     Assert.assertEquals(res[0], success);
574     Assert.assertEquals(res[1], success);
575     Assert.assertEquals(res[2], success);
576     Assert.assertEquals(res[3], success);
577     Assert.assertEquals(res[4], failure);
578     Assert.assertEquals(res[5], success);
579     Assert.assertEquals(res[6], failure);
580   }
581 
582   @Test
583   public void testErrorsServers() throws InterruptedIOException,
584       RetriesExhaustedWithDetailsException {
585     HTable ht = new HTable();
586     Configuration configuration = new Configuration(conf);
587     configuration.setBoolean(HConnectionManager.RETRIES_BY_SERVER_KEY, true);
588     configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 20);
589 
590     MyConnectionImpl mci = new MyConnectionImpl(configuration);
591     ht.connection = mci;
592     ht.ap = new MyAsyncProcess<Object>(mci, null, configuration);
593 
594 
595     Assert.assertTrue(ht.ap.useServerTrackerForRetries);
596     Assert.assertNotNull(ht.ap.createServerErrorTracker());
597     Assert.assertTrue(ht.ap.serverTrackerTimeout > 10000);
598     ht.ap.serverTrackerTimeout = 1;
599 
600 
601     Put p = createPut(true, false);
602     ht.setAutoFlush(false);
603     ht.put(p);
604 
605     long start = System.currentTimeMillis();
606     try {
607       ht.flushCommits();
608       Assert.fail();
609     } catch (RetriesExhaustedWithDetailsException expected) {
610     }
611     // Checking that the ErrorsServers came into play and made us stop immediately
612     Assert.assertTrue((System.currentTimeMillis() - start) < 10000);
613   }
614 
615 
616   /**
617    * @param reg1    if true, creates a put on region 1, region 2 otherwise
618    * @param success if true, the put will succeed.
619    * @return a put
620    */
621   private Put createPut(boolean reg1, boolean success) {
622     Put p;
623     if (!success) {
624       p = new Put(FAILS);
625     } else if (reg1) {
626       p = new Put(DUMMY_BYTES_1);
627     } else {
628       p = new Put(DUMMY_BYTES_2);
629     }
630 
631     p.add(DUMMY_BYTES_1, DUMMY_BYTES_1, DUMMY_BYTES_1);
632 
633     return p;
634   }
635 }