View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.hbase.TableName;
25  import org.apache.hadoop.hbase.HConstants;
26  import org.apache.hadoop.hbase.HRegionInfo;
27  import org.apache.hadoop.hbase.HRegionLocation;
28  import org.apache.hadoop.hbase.MediumTests;
29  import org.apache.hadoop.hbase.ServerName;
30  import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
31  import org.apache.hadoop.hbase.util.Bytes;
32  import org.apache.hadoop.hbase.util.Threads;
33  import org.junit.Assert;
34  import org.junit.Test;
35  import org.junit.experimental.categories.Category;
36  import org.mockito.Mockito;
37  
38  import java.io.IOException;
39  import java.io.InterruptedIOException;
40  import java.util.ArrayList;
41  import java.util.Arrays;
42  import java.util.List;
43  import java.util.Map;
44  import java.util.concurrent.ExecutorService;
45  import java.util.concurrent.SynchronousQueue;
46  import java.util.concurrent.ThreadFactory;
47  import java.util.concurrent.ThreadPoolExecutor;
48  import java.util.concurrent.TimeUnit;
49  import java.util.concurrent.atomic.AtomicBoolean;
50  import java.util.concurrent.atomic.AtomicInteger;
51  
52  @Category(MediumTests.class)
53  public class TestAsyncProcess {
54    private static final TableName DUMMY_TABLE =
55        TableName.valueOf("DUMMY_TABLE");
56    private static final byte[] DUMMY_BYTES_1 = "DUMMY_BYTES_1".getBytes();
57    private static final byte[] DUMMY_BYTES_2 = "DUMMY_BYTES_2".getBytes();
58    private static final byte[] DUMMY_BYTES_3 = "DUMMY_BYTES_3".getBytes();
59    private static final byte[] FAILS = "FAILS".getBytes();
60    private static final Configuration conf = new Configuration();
61  
62    private static ServerName sn = ServerName.valueOf("localhost:10,1254");
63    private static ServerName sn2 = ServerName.valueOf("localhost:140,12540");
64    private static HRegionInfo hri1 =
65        new HRegionInfo(DUMMY_TABLE, DUMMY_BYTES_1, DUMMY_BYTES_2, false, 1);
66    private static HRegionInfo hri2 =
67        new HRegionInfo(DUMMY_TABLE, DUMMY_BYTES_2, HConstants.EMPTY_END_ROW, false, 2);
68    private static HRegionInfo hri3 =
69        new HRegionInfo(DUMMY_TABLE, DUMMY_BYTES_3, HConstants.EMPTY_END_ROW, false, 3);
70    private static HRegionLocation loc1 = new HRegionLocation(hri1, sn);
71    private static HRegionLocation loc2 = new HRegionLocation(hri2, sn);
72    private static HRegionLocation loc3 = new HRegionLocation(hri3, sn2);
73  
74    private static final String success = "success";
75    private static Exception failure = new Exception("failure");
76  
77    static class MyAsyncProcess<Res> extends AsyncProcess<Res> {
78      final AtomicInteger nbMultiResponse = new AtomicInteger();
79      final AtomicInteger nbActions = new AtomicInteger();
80  
81      static class CountingThreadFactory implements ThreadFactory {
82        final AtomicInteger nbThreads;
83        ThreadFactory realFactory =  Threads.newDaemonThreadFactory("test-TestAsyncProcess");
84        @Override
85        public Thread newThread(Runnable r) {
86          nbThreads.incrementAndGet();
87          return realFactory.newThread(r);
88        }
89  
90        CountingThreadFactory(AtomicInteger nbThreads){
91          this.nbThreads = nbThreads;
92        }
93      }
94  
95      public MyAsyncProcess(HConnection hc, AsyncProcessCallback<Res> callback, Configuration conf) {
96        this(hc, callback, conf, new AtomicInteger());
97      }
98  
99        public MyAsyncProcess(HConnection hc, AsyncProcessCallback<Res> callback, Configuration conf,
100                           AtomicInteger nbThreads) {
101       super(hc, DUMMY_TABLE, new ThreadPoolExecutor(1, 20, 60, TimeUnit.SECONDS,
102         new SynchronousQueue<Runnable>(), new CountingThreadFactory(nbThreads)),
103           callback, conf, new RpcRetryingCallerFactory(conf), new RpcControllerFactory(conf));
104     }
105 
106     @Override
107     protected RpcRetryingCaller<MultiResponse> createCaller(MultiServerCallable<Row> callable) {
108       final MultiResponse mr = createMultiResponse(callable.getLocation(), callable.getMulti(),
109           nbMultiResponse, nbActions);
110       return new RpcRetryingCaller<MultiResponse>(100, 10) {
111         @Override
112         public MultiResponse callWithoutRetries( RetryingCallable<MultiResponse> callable)
113         throws IOException, RuntimeException {
114           try {
115             // sleep one second in order for threadpool to start another thread instead of reusing
116             // existing one. 
117             Thread.sleep(1000);
118           } catch (InterruptedException e) {
119             // ignore error
120           }
121           return mr;
122         }
123       };
124     }
125   }
126 
127   static MultiResponse createMultiResponse(final HRegionLocation loc,
128       final MultiAction<Row> multi, AtomicInteger nbMultiResponse, AtomicInteger nbActions) {
129     final MultiResponse mr = new MultiResponse();
130     nbMultiResponse.incrementAndGet();
131     for (Map.Entry<byte[], List<Action<Row>>> entry : multi.actions.entrySet()) {
132       for (Action a : entry.getValue()) {
133         nbActions.incrementAndGet();
134         if (Arrays.equals(FAILS, a.getAction().getRow())) {
135           mr.add(loc.getRegionInfo().getRegionName(), a.getOriginalIndex(), failure);
136         } else {
137           mr.add(loc.getRegionInfo().getRegionName(), a.getOriginalIndex(), success);
138         }
139       }
140     }
141     return mr;
142   }
143   /**
144    * Returns our async process.
145    */
146   static class MyConnectionImpl extends HConnectionManager.HConnectionImplementation {
147     MyAsyncProcess<?> ap;
148     final AtomicInteger nbThreads = new AtomicInteger(0);
149     final static Configuration c = new Configuration();
150 
151     static {
152       c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
153     }
154 
155     protected MyConnectionImpl() {
156       super(c);
157     }
158 
159     protected MyConnectionImpl(Configuration conf) {
160       super(conf);
161     }
162 
163     @Override
164     protected <R> AsyncProcess createAsyncProcess(TableName tableName,
165                                                   ExecutorService pool,
166                                                   AsyncProcess.AsyncProcessCallback<R> callback,
167                                                   Configuration confn ) {
168       ap = new MyAsyncProcess<R>(this, callback, c, nbThreads);
169       return ap;
170     }
171 
172     @Override
173     public HRegionLocation locateRegion(final TableName tableName,
174                                         final byte[] row) {
175       return loc1;
176     }
177   }
178 
179   /**
180    * Returns our async process.
181    */
182   static class MyConnectionImpl2 extends MyConnectionImpl {
183     List<HRegionLocation> hrl;
184     final boolean usedRegions[];
185 
186     protected MyConnectionImpl2(List<HRegionLocation> hrl) {
187       super(c);
188       this.hrl = hrl;
189       this.usedRegions = new boolean[hrl.size()];
190     }
191 
192     @Override
193     public HRegionLocation locateRegion(final TableName tableName,
194                                         final byte[] row) {
195       int i = 0;
196       for (HRegionLocation hr:hrl){
197         if (Arrays.equals(row, hr.getRegionInfo().getStartKey())){
198             usedRegions[i] = true;
199           return hr;
200         }
201         i++;
202       }
203       return null;
204     }
205   }
206 
207   @Test
208   public void testSubmit() throws Exception {
209     HConnection hc = createHConnection();
210     AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
211 
212     List<Put> puts = new ArrayList<Put>();
213     puts.add(createPut(1, true));
214 
215     ap.submit(puts, false);
216     Assert.assertTrue(puts.isEmpty());
217   }
218 
219   @Test
220   public void testSubmitWithCB() throws Exception {
221     HConnection hc = createHConnection();
222     MyCB mcb = new MyCB();
223     AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
224 
225     List<Put> puts = new ArrayList<Put>();
226     puts.add(createPut(1, true));
227 
228     ap.submit(puts, false);
229     Assert.assertTrue(puts.isEmpty());
230 
231     while (!(mcb.successCalled.get() == 1) && !ap.hasError()) {
232       Thread.sleep(1);
233     }
234     Assert.assertEquals(mcb.successCalled.get(), 1);
235   }
236 
237   @Test
238   public void testSubmitBusyRegion() throws Exception {
239     HConnection hc = createHConnection();
240     AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
241 
242     List<Put> puts = new ArrayList<Put>();
243     puts.add(createPut(1, true));
244 
245     ap.incTaskCounters(Arrays.asList(hri1.getRegionName()), sn);
246     ap.submit(puts, false);
247     Assert.assertEquals(puts.size(), 1);
248 
249     ap.decTaskCounters(Arrays.asList(hri1.getRegionName()), sn);
250     ap.submit(puts, false);
251     Assert.assertTrue(puts.isEmpty());
252   }
253 
254 
255   @Test
256   public void testSubmitBusyRegionServer() throws Exception {
257     HConnection hc = createHConnection();
258     AsyncProcess<Object> ap = new MyAsyncProcess<Object>(hc, null, conf);
259 
260     ap.taskCounterPerServer.put(sn2, new AtomicInteger(ap.maxConcurrentTasksPerServer));
261 
262     List<Put> puts = new ArrayList<Put>();
263     puts.add(createPut(1, true));
264     puts.add(createPut(3, true)); // <== this one won't be taken, the rs is busy
265     puts.add(createPut(1, true)); // <== this one will make it, the region is already in
266     puts.add(createPut(2, true)); // <== new region, but the rs is ok
267 
268     ap.submit(puts, false);
269     Assert.assertEquals(" puts=" + puts, 1, puts.size());
270 
271     ap.taskCounterPerServer.put(sn2, new AtomicInteger(ap.maxConcurrentTasksPerServer - 1));
272     ap.submit(puts, false);
273     Assert.assertTrue(puts.isEmpty());
274   }
275 
276   @Test
277   public void testFail() throws Exception {
278     HConnection hc = createHConnection();
279     MyCB mcb = new MyCB();
280     AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
281 
282     List<Put> puts = new ArrayList<Put>();
283     Put p = createPut(1, false);
284     puts.add(p);
285 
286     ap.submit(puts, false);
287     Assert.assertTrue(puts.isEmpty());
288 
289     while (!ap.hasError()) {
290       Thread.sleep(1);
291     }
292 
293     Assert.assertEquals(0, mcb.successCalled.get());
294     Assert.assertEquals(2, mcb.retriableFailure.get());
295     Assert.assertEquals(1, mcb.failureCalled.get());
296 
297     Assert.assertEquals(1, ap.getErrors().exceptions.size());
298     Assert.assertTrue("was: " + ap.getErrors().exceptions.get(0),
299         failure.equals(ap.getErrors().exceptions.get(0)));
300     Assert.assertTrue("was: " + ap.getErrors().exceptions.get(0),
301         failure.equals(ap.getErrors().exceptions.get(0)));
302 
303     Assert.assertEquals(1, ap.getFailedOperations().size());
304     Assert.assertTrue("was: " + ap.getFailedOperations().get(0),
305         p.equals(ap.getFailedOperations().get(0)));
306   }
307 
308   @Test
309   public void testWaitForNextTaskDone() throws IOException {
310     HConnection hc = createHConnection();
311     MyCB mcb = new MyCB();
312     final AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
313     ap.tasksSent.incrementAndGet();
314 
315     final AtomicBoolean checkPoint = new AtomicBoolean(false);
316     final AtomicBoolean checkPoint2 = new AtomicBoolean(false);
317 
318     Thread t = new Thread(){
319       @Override
320       public void run(){
321         Threads.sleep(1000);
322         Assert.assertFalse(checkPoint.get());
323         ap.tasksDone.incrementAndGet();
324         checkPoint2.set(true);
325       }
326     };
327 
328     t.start();
329     ap.waitForNextTaskDone(0);
330     checkPoint.set(true);
331     while (!checkPoint2.get()){
332       Threads.sleep(1);
333     }
334   }
335 
336   @Test
337   public void testSubmitTrue() throws IOException {
338     HConnection hc = createHConnection();
339     MyCB mcb = new MyCB();
340     final AsyncProcess<Object> ap = new MyAsyncProcess<Object>(hc, mcb, conf);
341     ap.tasksSent.incrementAndGet();
342     final AtomicInteger ai = new AtomicInteger(1);
343     ap.taskCounterPerRegion.put(hri1.getRegionName(), ai);
344 
345     final AtomicBoolean checkPoint = new AtomicBoolean(false);
346     final AtomicBoolean checkPoint2 = new AtomicBoolean(false);
347 
348     Thread t = new Thread(){
349       @Override
350       public void run(){
351         Threads.sleep(1000);
352         Assert.assertFalse(checkPoint.get());
353         ai.decrementAndGet();
354         ap.tasksDone.incrementAndGet();
355         checkPoint2.set(true);
356       }
357     };
358 
359     List<Put> puts = new ArrayList<Put>();
360     Put p = createPut(1, true);
361     puts.add(p);
362 
363     ap.submit(puts, false);
364     Assert.assertFalse(puts.isEmpty());
365 
366     t.start();
367 
368     ap.submit(puts, true);
369     Assert.assertTrue(puts.isEmpty());
370 
371     checkPoint.set(true);
372     while (!checkPoint2.get()){
373       Threads.sleep(1);
374     }
375   }
376 
377   @Test
378   public void testFailAndSuccess() throws Exception {
379     HConnection hc = createHConnection();
380     MyCB mcb = new MyCB();
381     AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
382 
383     List<Put> puts = new ArrayList<Put>();
384     puts.add(createPut(1, false));
385     puts.add(createPut(1, true));
386     puts.add(createPut(1, true));
387 
388     ap.submit(puts, false);
389     Assert.assertTrue(puts.isEmpty());
390 
391     while (!ap.hasError()) {
392       Thread.sleep(1);
393     }
394     ap.waitUntilDone();
395  
396     Assert.assertEquals(mcb.successCalled.get(), 2);
397     Assert.assertEquals(mcb.retriableFailure.get(), 2);
398     Assert.assertEquals(mcb.failureCalled.get(), 1);
399 
400     Assert.assertEquals(1, ap.getErrors().actions.size());
401 
402 
403     puts.add(createPut(1, true));
404     ap.submit(puts, false);
405     Assert.assertTrue(puts.isEmpty());
406 
407     while (mcb.successCalled.get() != 3) {
408       Thread.sleep(1);
409     }
410     Assert.assertEquals(mcb.retriableFailure.get(), 2);
411     Assert.assertEquals(mcb.failureCalled.get(), 1);
412 
413     ap.clearErrors();
414     Assert.assertTrue(ap.getErrors().actions.isEmpty());
415   }
416 
417   @Test
418   public void testFlush() throws Exception {
419     HConnection hc = createHConnection();
420     MyCB mcb = new MyCB();
421     AsyncProcess ap = new MyAsyncProcess<Object>(hc, mcb, conf);
422 
423     List<Put> puts = new ArrayList<Put>();
424     puts.add(createPut(1, false));
425     puts.add(createPut(1, true));
426     puts.add(createPut(1, true));
427 
428     ap.submit(puts, false);
429     ap.waitUntilDone();
430 
431     Assert.assertEquals(mcb.successCalled.get(), 2);
432     Assert.assertEquals(mcb.retriableFailure.get(), 2);
433     Assert.assertEquals(mcb.failureCalled.get(), 1);
434 
435     Assert.assertEquals(1, ap.getFailedOperations().size());
436   }
437 
438   @Test
439   public void testMaxTask() throws Exception {
440     HConnection hc = createHConnection();
441     final AsyncProcess ap = new MyAsyncProcess<Object>(hc, null, conf);
442 
443     for (int i = 0; i < 1000; i++) {
444       ap.incTaskCounters(Arrays.asList("dummy".getBytes()), sn);
445     }
446 
447     final Thread myThread = Thread.currentThread();
448 
449     Thread t = new Thread() {
450       public void run() {
451         Threads.sleep(2000);
452         myThread.interrupt();
453       }
454     };
455 
456     List<Put> puts = new ArrayList<Put>();
457     puts.add(createPut(1, true));
458 
459     t.start();
460 
461     try {
462       ap.submit(puts, false);
463       Assert.fail("We should have been interrupted.");
464     } catch (InterruptedIOException expected) {
465     }
466 
467     final long sleepTime = 2000;
468 
469     Thread t2 = new Thread() {
470       public void run() {
471         Threads.sleep(sleepTime);
472         while (ap.tasksDone.get() > 0) {
473           ap.decTaskCounters(Arrays.asList("dummy".getBytes()), sn);
474         }
475       }
476     };
477     t2.start();
478 
479     long start = System.currentTimeMillis();
480     ap.submit(new ArrayList<Row>(), false);
481     long end = System.currentTimeMillis();
482 
483     //Adds 100 to secure us against approximate timing.
484     Assert.assertTrue(start + 100L + sleepTime > end);
485   }
486 
487 
488   private class MyCB implements AsyncProcess.AsyncProcessCallback<Object> {
489     private final AtomicInteger successCalled = new AtomicInteger(0);
490     private final AtomicInteger failureCalled = new AtomicInteger(0);
491     private final AtomicInteger retriableFailure = new AtomicInteger(0);
492 
493 
494     @Override
495     public void success(int originalIndex, byte[] region, Row row, Object o) {
496       successCalled.incrementAndGet();
497     }
498 
499     @Override
500     public boolean failure(int originalIndex, byte[] region, Row row, Throwable t) {
501       failureCalled.incrementAndGet();
502       return true;
503     }
504 
505     @Override
506     public boolean retriableFailure(int originalIndex, Row row, byte[] region,
507                                     Throwable exception) {
508       // We retry once only.
509       return (retriableFailure.incrementAndGet() < 2);
510     }
511   }
512 
513 
514   private static HConnection createHConnection() throws IOException {
515     HConnection hc = Mockito.mock(HConnection.class);
516 
517     Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
518         Mockito.eq(DUMMY_BYTES_1), Mockito.anyBoolean())).thenReturn(loc1);
519     Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
520         Mockito.eq(DUMMY_BYTES_1))).thenReturn(loc1);
521 
522     Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
523         Mockito.eq(DUMMY_BYTES_2), Mockito.anyBoolean())).thenReturn(loc2);
524     Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
525         Mockito.eq(DUMMY_BYTES_2))).thenReturn(loc2);
526 
527     Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
528         Mockito.eq(DUMMY_BYTES_3), Mockito.anyBoolean())).thenReturn(loc2);
529     Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
530         Mockito.eq(DUMMY_BYTES_3))).thenReturn(loc3);
531 
532     Mockito.when(hc.getRegionLocation(Mockito.eq(DUMMY_TABLE),
533         Mockito.eq(FAILS), Mockito.anyBoolean())).thenReturn(loc2);
534     Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE),
535         Mockito.eq(FAILS))).thenReturn(loc2);
536 
537     return hc;
538   }
539 
540   @Test
541   public void testHTablePutSuccess() throws Exception {
542     HTable ht = Mockito.mock(HTable.class);
543     HConnection hc = createHConnection();
544     ht.ap = new MyAsyncProcess<Object>(hc, null, conf);
545 
546     Put put = createPut(1, true);
547 
548     Assert.assertEquals(0, ht.getWriteBufferSize());
549     ht.put(put);
550     Assert.assertEquals(0, ht.getWriteBufferSize());
551   }
552 
553   private void doHTableFailedPut(boolean bufferOn) throws Exception {
554     HTable ht = new HTable();
555     HConnection hc = createHConnection();
556     MyCB mcb = new MyCB(); // This allows to have some hints on what's going on.
557     ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
558     ht.setAutoFlush(true, true);
559     if (bufferOn) {
560       ht.setWriteBufferSize(1024L * 1024L);
561     } else {
562       ht.setWriteBufferSize(0L);
563     }
564 
565     Put put = createPut(1, false);
566 
567     Assert.assertEquals(0L, ht.currentWriteBufferSize);
568     try {
569       ht.put(put);
570       if (bufferOn) {
571         ht.flushCommits();
572       }
573       Assert.fail();
574     } catch (RetriesExhaustedException expected) {
575     }
576     Assert.assertEquals(0L, ht.currentWriteBufferSize);
577     Assert.assertEquals(0, mcb.successCalled.get());
578     Assert.assertEquals(2, mcb.retriableFailure.get());
579     Assert.assertEquals(1, mcb.failureCalled.get());
580 
581     // This should not raise any exception, puts have been 'received' before by the catch.
582     ht.close();
583   }
584 
585   @Test
586   public void testHTableFailedPutWithBuffer() throws Exception {
587     doHTableFailedPut(true);
588   }
589 
590   @Test
591   public void doHTableFailedPutWithoutBuffer() throws Exception {
592     doHTableFailedPut(false);
593   }
594 
595   @Test
596   public void testHTableFailedPutAndNewPut() throws Exception {
597     HTable ht = new HTable();
598     HConnection hc = createHConnection();
599     MyCB mcb = new MyCB(); // This allows to have some hints on what's going on.
600     ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
601     ht.setAutoFlush(false, true);
602     ht.setWriteBufferSize(0);
603 
604     Put p = createPut(1, false);
605     ht.put(p);
606 
607     ht.ap.waitUntilDone(); // Let's do all the retries.
608 
609     // We're testing that we're behaving as we were behaving in 0.94: sending exceptions in the
610     //  doPut if it fails.
611     // This said, it's not a very easy going behavior. For example, when we insert a list of
612     //  puts, we may raise an exception in the middle of the list. It's then up to the caller to
613     //  manage what was inserted, what was tried but failed, and what was not even tried.
614     p = createPut(1, true);
615     Assert.assertEquals(0, ht.writeAsyncBuffer.size());
616     try {
617       ht.put(p);
618       Assert.fail();
619     } catch (RetriesExhaustedException expected) {
620     }
621     Assert.assertEquals("the put should not been inserted.", 0, ht.writeAsyncBuffer.size());
622   }
623 
624 
625   @Test
626   public void testWithNoClearOnFail() throws IOException {
627     HTable ht = new HTable();
628     HConnection hc = createHConnection();
629     MyCB mcb = new MyCB();
630     ht.ap = new MyAsyncProcess<Object>(hc, mcb, conf);
631     ht.setAutoFlush(false, false);
632 
633     Put p = createPut(1, false);
634     ht.put(p);
635     Assert.assertEquals(0, ht.writeAsyncBuffer.size());
636     try {
637       ht.flushCommits();
638     } catch (RetriesExhaustedWithDetailsException expected) {
639     }
640     Assert.assertEquals(1, ht.writeAsyncBuffer.size());
641 
642     try {
643       ht.close();
644     } catch (RetriesExhaustedWithDetailsException expected) {
645     }
646     Assert.assertEquals(1, ht.writeAsyncBuffer.size());
647   }
648 
649   @Test
650   public void testBatch() throws IOException, InterruptedException {
651     HTable ht = new HTable();
652     ht.connection = new MyConnectionImpl();
653 
654     List<Put> puts = new ArrayList<Put>();
655     puts.add(createPut(1, true));
656     puts.add(createPut(1, true));
657     puts.add(createPut(1, true));
658     puts.add(createPut(1, true));
659     puts.add(createPut(1, false)); // <=== the bad apple, position 4
660     puts.add(createPut(1, true));
661     puts.add(createPut(1, false)); // <=== another bad apple, position 6
662 
663     Object[] res = new Object[puts.size()];
664     try {
665       ht.processBatch(puts, res);
666       Assert.fail();
667     } catch (RetriesExhaustedException expected) {
668     }
669 
670     Assert.assertEquals(res[0], success);
671     Assert.assertEquals(res[1], success);
672     Assert.assertEquals(res[2], success);
673     Assert.assertEquals(res[3], success);
674     Assert.assertEquals(res[4], failure);
675     Assert.assertEquals(res[5], success);
676     Assert.assertEquals(res[6], failure);
677   }
678 
679   @Test
680   public void testErrorsServers() throws IOException {
681     HTable ht = new HTable();
682     Configuration configuration = new Configuration(conf);
683     configuration.setBoolean(HConnectionManager.RETRIES_BY_SERVER_KEY, true);
684     configuration.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 3);
685     // set default writeBufferSize
686     ht.setWriteBufferSize(configuration.getLong("hbase.client.write.buffer", 2097152));
687 
688     MyConnectionImpl mci = new MyConnectionImpl(configuration);
689     ht.connection = mci;
690     ht.ap = new MyAsyncProcess<Object>(mci, null, configuration);
691 
692 
693     Assert.assertNotNull(ht.ap.createServerErrorTracker());
694     Assert.assertTrue(ht.ap.serverTrackerTimeout > 200);
695     ht.ap.serverTrackerTimeout = 1;
696 
697     Put p = createPut(1, false);
698     ht.setAutoFlush(false, false);
699     ht.put(p);
700 
701     try {
702       ht.flushCommits();
703       Assert.fail();
704     } catch (RetriesExhaustedWithDetailsException expected) {
705     }
706     // Checking that the ErrorsServers came into play and didn't make us stop immediately
707     Assert.assertEquals(ht.ap.tasksSent.get(), 3);
708   }
709 
710   /**
711    * This test simulates multiple regions on 2 servers. We should have 2 multi requests and
712    *  2 threads: 1 per server, this whatever the number of regions.
713    */
714   @Test
715   public void testThreadCreation() throws Exception {
716     final int NB_REGS = 100;
717     List<HRegionLocation> hrls = new ArrayList<HRegionLocation>(NB_REGS);
718     List<Get> gets = new ArrayList<Get>(NB_REGS);
719     for (int i = 0; i < NB_REGS; i++) {
720       HRegionInfo hri = new HRegionInfo(
721           DUMMY_TABLE, Bytes.toBytes(i * 10L), Bytes.toBytes(i * 10L + 9L), false, i);
722       HRegionLocation hrl = new HRegionLocation(hri, i % 2 == 0 ? sn : sn2);
723       hrls.add(hrl);
724 
725       Get get = new Get(Bytes.toBytes(i * 10L));
726       gets.add(get);
727     }
728 
729     HTable ht = new HTable();
730     MyConnectionImpl2 con = new MyConnectionImpl2(hrls);
731     ht.connection = con;
732 
733     ht.batch(gets);
734 
735     Assert.assertEquals(con.ap.nbActions.get(), NB_REGS);
736     Assert.assertEquals("1 multi response per server", 2, con.ap.nbMultiResponse.get());
737     Assert.assertEquals("1 thread per server", 2, con.nbThreads.get());
738 
739     int nbReg = 0;
740     for (int i =0; i<NB_REGS; i++){
741       if (con.usedRegions[i]) nbReg++;
742     }
743     Assert.assertEquals("nbReg=" + nbReg, nbReg, NB_REGS);
744   }
745 
746 
747   /**
748    * @param regCnt  the region: 1 to 3.
749    * @param success if true, the put will succeed.
750    * @return a put
751    */
752   private Put createPut(int regCnt, boolean success) {
753     Put p;
754     if (!success) {
755       p = new Put(FAILS);
756     } else switch (regCnt){
757       case 1 :
758         p = new Put(DUMMY_BYTES_1);
759         break;
760       case 2:
761         p = new Put(DUMMY_BYTES_2);
762         break;
763       case 3:
764         p = new Put(DUMMY_BYTES_3);
765         break;
766       default:
767         throw new IllegalArgumentException("unknown " + regCnt);
768     }
769 
770     p.add(DUMMY_BYTES_1, DUMMY_BYTES_1, DUMMY_BYTES_1);
771 
772     return p;
773   }
774 }