View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.Collection;
27  import java.util.HashMap;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.concurrent.ConcurrentHashMap;
32  import java.util.concurrent.ConcurrentMap;
33  import java.util.concurrent.ConcurrentSkipListMap;
34  import java.util.concurrent.ExecutorService;
35  import java.util.concurrent.RejectedExecutionException;
36  import java.util.concurrent.atomic.AtomicBoolean;
37  import java.util.concurrent.atomic.AtomicInteger;
38  import java.util.concurrent.atomic.AtomicLong;
39  
40  import org.apache.commons.logging.Log;
41  import org.apache.commons.logging.LogFactory;
42  import org.apache.hadoop.conf.Configuration;
43  import org.apache.hadoop.hbase.DoNotRetryIOException;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HRegionLocation;
47  import org.apache.hadoop.hbase.ServerName;
48  import org.apache.hadoop.hbase.TableName;
49  import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
52  import org.apache.hadoop.hbase.util.Pair;
53  import org.cloudera.htrace.Trace;
54  
55  import com.google.common.base.Preconditions;
56  
57  /**
58   * This class  allows a continuous flow of requests. It's written to be compatible with a
59   * synchronous caller such as HTable.
60   * <p>
61   * The caller sends a buffer of operation, by calling submit. This class extract from this list
62   * the operations it can send, i.e. the operations that are on region that are not considered
63   * as busy. The process is asynchronous, i.e. it returns immediately when if has finished to
64   * iterate on the list. If, and only if, the maximum number of current task is reached, the call
65   * to submit will block.
66   * </p>
67   * <p>
68   * The class manages internally the retries.
69   * </p>
70   * <p>
71   * The class includes an error marker: it allows to know if an operation has failed or not, and
72   * to get the exception details, i.e. the full list of throwables for each attempt. This marker
73   * is here to help the backward compatibility in HTable. In most (new) cases, it should be
74   * managed by the callbacks.
75   * </p>
76   * <p>
77   * A callback is available, in order to: <list>
78   * <li>Get the result of the operation (failure or success)</li>
79   * <li>When an operation fails but could be retried, allows or not to retry</li>
80   * <li>When an operation fails for good (can't be retried or already retried the maximum number
81   * time), register the error or not.
82   * </list>
83   * <p>
84   * This class is not thread safe externally; only one thread should submit operations at a time.
85   * Internally, the class is thread safe enough to manage simultaneously new submission and results
86   * arising from older operations.
87   * </p>
88   * <p>
89   * Internally, this class works with {@link Row}, this mean it could be theoretically used for
90   * gets as well.
91   * </p>
92   */
93  class AsyncProcess<CResult> {
94    private static final Log LOG = LogFactory.getLog(AsyncProcess.class);
95    protected static final AtomicLong COUNTER = new AtomicLong();
96    protected final long id;
97    private final int startLogErrorsCnt;
98    protected final HConnection hConnection;
99    protected final TableName tableName;
100   protected final ExecutorService pool;
101   protected final AsyncProcessCallback<CResult> callback;
102   protected final BatchErrors errors = new BatchErrors();
103   protected final AtomicBoolean hasError = new AtomicBoolean(false);
104   protected final AtomicLong tasksSent = new AtomicLong(0);
105   protected final AtomicLong tasksDone = new AtomicLong(0);
106   protected final AtomicLong retriesCnt = new AtomicLong(0);
107   protected final ConcurrentMap<byte[], AtomicInteger> taskCounterPerRegion =
108       new ConcurrentSkipListMap<byte[], AtomicInteger>(Bytes.BYTES_COMPARATOR);
109   protected final ConcurrentMap<ServerName, AtomicInteger> taskCounterPerServer =
110       new ConcurrentHashMap<ServerName, AtomicInteger>();
111 
112   /**
113    * The number of tasks simultaneously executed on the cluster.
114    */
115   protected final int maxTotalConcurrentTasks;
116 
117   /**
118    * The number of tasks we run in parallel on a single region.
119    * With 1 (the default) , we ensure that the ordering of the queries is respected: we don't start
120    * a set of operations on a region before the previous one is done. As well, this limits
121    * the pressure we put on the region server.
122    */
123   protected final int maxConcurrentTasksPerRegion;
124 
125   /**
126    * The number of task simultaneously executed on a single region server.
127    */
128   protected final int maxConcurrentTasksPerServer;
129   protected final long pause;
130   protected int numTries;
131   protected int serverTrackerTimeout;
132   protected RpcRetryingCallerFactory rpcCallerFactory;
133   private RpcControllerFactory rpcFactory;
134 
135 
136   /**
137    * This interface allows to keep the interface of the previous synchronous interface, that uses
138    * an array of object to return the result.
139    * <p/>
140    * This interface allows the caller to specify the behavior on errors: <list>
141    * <li>If we have not yet reach the maximum number of retries, the user can nevertheless
142    * specify if this specific operation should be retried or not.
143    * </li>
144    * <li>If an operation fails (i.e. is not retried or fails after all retries), the user can
145    * specify is we should mark this AsyncProcess as in error or not.
146    * </li>
147    * </list>
148    */
149   interface AsyncProcessCallback<CResult> {
150 
151     /**
152      * Called on success. originalIndex holds the index in the action list.
153      */
154     void success(int originalIndex, byte[] region, Row row, CResult result);
155 
156     /**
157      * called on failure, if we don't retry (i.e. called once per failed operation).
158      *
159      * @return true if we should store the error and tag this async process as being in error.
160      *         false if the failure of this operation can be safely ignored, and does not require
161      *         the current process to be stopped without proceeding with the other operations in
162      *         the queue.
163      */
164     boolean failure(int originalIndex, byte[] region, Row row, Throwable t);
165 
166     /**
167      * Called on a failure we plan to retry. This allows the user to stop retrying. Will be
168      * called multiple times for a single action if it fails multiple times.
169      *
170      * @return false if we should retry, true otherwise.
171      */
172     boolean retriableFailure(int originalIndex, Row row, byte[] region, Throwable exception);
173   }
174 
175   private static class BatchErrors {
176     private final List<Throwable> throwables = new ArrayList<Throwable>();
177     private final List<Row> actions = new ArrayList<Row>();
178     private final List<String> addresses = new ArrayList<String>();
179 
180     public synchronized void add(Throwable ex, Row row, HRegionLocation location) {
181       if (row == null){
182         throw new IllegalArgumentException("row cannot be null. location=" + location);
183       }
184 
185       throwables.add(ex);
186       actions.add(row);
187       addresses.add(location != null ? location.getServerName().toString() : "null location");
188     }
189 
190     private synchronized RetriesExhaustedWithDetailsException makeException() {
191       return new RetriesExhaustedWithDetailsException(
192           new ArrayList<Throwable>(throwables),
193           new ArrayList<Row>(actions), new ArrayList<String>(addresses));
194     }
195 
196     public synchronized void clear() {
197       throwables.clear();
198       actions.clear();
199       addresses.clear();
200     }
201   }
202 
203   public AsyncProcess(HConnection hc, TableName tableName, ExecutorService pool,
204       AsyncProcessCallback<CResult> callback, Configuration conf,
205       RpcRetryingCallerFactory rpcCaller, RpcControllerFactory rpcFactory) {
206     if (hc == null){
207       throw new IllegalArgumentException("HConnection cannot be null.");
208     }
209 
210     this.hConnection = hc;
211     this.tableName = tableName;
212     this.pool = pool;
213     this.callback = callback;
214 
215     this.id = COUNTER.incrementAndGet();
216 
217     this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
218         HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
219     this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
220         HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
221 
222     this.maxTotalConcurrentTasks = conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
223       HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS);
224     this.maxConcurrentTasksPerServer = conf.getInt(HConstants.HBASE_CLIENT_MAX_PERSERVER_TASKS,
225           HConstants.DEFAULT_HBASE_CLIENT_MAX_PERSERVER_TASKS);
226     this.maxConcurrentTasksPerRegion = conf.getInt(HConstants.HBASE_CLIENT_MAX_PERREGION_TASKS,
227           HConstants.DEFAULT_HBASE_CLIENT_MAX_PERREGION_TASKS);
228 
229     // A few failure is fine: region moved, then is not opened, then is overloaded. We try
230     //  to have an acceptable heuristic for the number of errors we don't log.
231     //  9 was chosen because we wait for 1s at this stage.
232     this.startLogErrorsCnt = conf.getInt("hbase.client.start.log.errors.counter", 9);
233 
234     if (this.maxTotalConcurrentTasks <= 0) {
235       throw new IllegalArgumentException("maxTotalConcurrentTasks=" + maxTotalConcurrentTasks);
236     }
237     if (this.maxConcurrentTasksPerServer <= 0) {
238       throw new IllegalArgumentException("maxConcurrentTasksPerServer=" +
239           maxConcurrentTasksPerServer);
240     }
241     if (this.maxConcurrentTasksPerRegion <= 0) {
242       throw new IllegalArgumentException("maxConcurrentTasksPerRegion=" +
243           maxConcurrentTasksPerRegion);
244     }
245 
246     // Server tracker allows us to do faster, and yet useful (hopefully), retries.
247     // However, if we are too useful, we might fail very quickly due to retry count limit.
248     // To avoid this, we are going to cheat for now (see HBASE-7659), and calculate maximum
249     // retry time if normal retries were used. Then we will retry until this time runs out.
250     // If we keep hitting one server, the net effect will be the incremental backoff, and
251     // essentially the same number of retries as planned. If we have to do faster retries,
252     // we will do more retries in aggregate, but the user will be none the wiser.
253     this.serverTrackerTimeout = 0;
254     for (int i = 0; i < this.numTries; ++i) {
255       serverTrackerTimeout += ConnectionUtils.getPauseTime(this.pause, i);
256     }
257 
258     this.rpcCallerFactory = rpcCaller;
259     Preconditions.checkNotNull(rpcFactory);
260     this.rpcFactory = rpcFactory;
261   }
262 
263   /**
264    * Extract from the rows list what we can submit. The rows we can not submit are kept in the
265    * list.
266    *
267    * @param rows - the submitted row. Modified by the method: we remove the rows we took.
268    * @param atLeastOne true if we should submit at least a subset.
269    */
270   public void submit(List<? extends Row> rows, boolean atLeastOne) throws InterruptedIOException {
271     if (rows.isEmpty()) {
272       return;
273     }
274 
275     // This looks like we are keying by region but HRegionLocation has a comparator that compares
276     // on the server portion only (hostname + port) so this Map collects regions by server.
277     Map<HRegionLocation, MultiAction<Row>> actionsByServer =
278       new HashMap<HRegionLocation, MultiAction<Row>>();
279     List<Action<Row>> retainedActions = new ArrayList<Action<Row>>(rows.size());
280 
281     long currentTaskCnt = tasksDone.get();
282     boolean alreadyLooped = false;
283 
284     NonceGenerator ng = this.hConnection.getNonceGenerator();
285     do {
286       if (alreadyLooped){
287         // if, for whatever reason, we looped, we want to be sure that something has changed.
288         waitForNextTaskDone(currentTaskCnt);
289         currentTaskCnt = tasksDone.get();
290       } else {
291         alreadyLooped = true;
292       }
293 
294       // Wait until there is at least one slot for a new task.
295       waitForMaximumCurrentTasks(maxTotalConcurrentTasks - 1);
296 
297       // Remember the previous decisions about regions or region servers we put in the
298       //  final multi.
299       Map<Long, Boolean> regionIncluded = new HashMap<Long, Boolean>();
300       Map<ServerName, Boolean> serverIncluded = new HashMap<ServerName, Boolean>();
301 
302       int posInList = -1;
303       Iterator<? extends Row> it = rows.iterator();
304       while (it.hasNext()) {
305         Row r = it.next();
306         HRegionLocation loc = findDestLocation(r, posInList);
307 
308         if (loc == null) { // loc is null if there is an error such as meta not available.
309           it.remove();
310         } else if (canTakeOperation(loc, regionIncluded, serverIncluded)) {
311           Action<Row> action = new Action<Row>(r, ++posInList);
312           setNonce(ng, r, action);
313           retainedActions.add(action);
314           addAction(loc, action, actionsByServer, ng);
315           it.remove();
316         }
317       }
318     } while (retainedActions.isEmpty() && atLeastOne && !hasError());
319 
320     HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
321     sendMultiAction(retainedActions, actionsByServer, 1, errorsByServer);
322   }
323 
324   /**
325    * Group the actions per region server.
326    *
327    * @param loc - the destination. Must not be null.
328    * @param action - the action to add to the multiaction
329    * @param actionsByServer the multiaction per server
330    * @param ng Nonce generator, or null if no nonces are needed.
331    */
332   private void addAction(HRegionLocation loc, Action<Row> action, Map<HRegionLocation,
333       MultiAction<Row>> actionsByServer, NonceGenerator ng) {
334     final byte[] regionName = loc.getRegionInfo().getRegionName();
335     MultiAction<Row> multiAction = actionsByServer.get(loc);
336     if (multiAction == null) {
337       multiAction = new MultiAction<Row>();
338       actionsByServer.put(loc, multiAction);
339     }
340     if (action.hasNonce() && !multiAction.hasNonceGroup()) {
341       // TODO: this code executes for every (re)try, and calls getNonceGroup again
342       //       for the same action. It must return the same value across calls.
343       multiAction.setNonceGroup(ng.getNonceGroup());
344     }
345 
346     multiAction.add(regionName, action);
347   }
348 
349   /**
350    * Find the destination.
351    *
352    * @param row          the row
353    * @param posInList    the position in the list
354    * @return the destination. Null if we couldn't find it.
355    */
356   private HRegionLocation findDestLocation(Row row, int posInList) {
357     if (row == null) throw new IllegalArgumentException("#" + id + ", row cannot be null");
358     HRegionLocation loc = null;
359     IOException locationException = null;
360     try {
361       loc = hConnection.locateRegion(this.tableName, row.getRow());
362       if (loc == null) {
363         locationException = new IOException("#" + id + ", no location found, aborting submit for" +
364             " tableName=" + tableName +
365             " rowkey=" + Arrays.toString(row.getRow()));
366       }
367     } catch (IOException e) {
368       locationException = e;
369     }
370     if (locationException != null) {
371       // There are multiple retries in locateRegion already. No need to add new.
372       // We can't continue with this row, hence it's the last retry.
373       manageError(posInList, row, false, locationException, null);
374       return null;
375     }
376 
377     return loc;
378   }
379 
380   /**
381    * Check if we should send new operations to this region or region server.
382    * We're taking into account the past decision; if we have already accepted
383    * operation on a given region, we accept all operations for this region.
384    *
385    * @param loc; the region and the server name we want to use.
386    * @return true if this region is considered as busy.
387    */
388   protected boolean canTakeOperation(HRegionLocation loc,
389                                      Map<Long, Boolean> regionsIncluded,
390                                      Map<ServerName, Boolean> serversIncluded) {
391     long regionId = loc.getRegionInfo().getRegionId();
392     Boolean regionPrevious = regionsIncluded.get(regionId);
393 
394     if (regionPrevious != null) {
395       // We already know what to do with this region.
396       return regionPrevious;
397     }
398 
399     Boolean serverPrevious = serversIncluded.get(loc.getServerName());
400     if (Boolean.FALSE.equals(serverPrevious)) {
401       // It's a new region, on a region server that we have already excluded.
402       regionsIncluded.put(regionId, Boolean.FALSE);
403       return false;
404     }
405 
406     AtomicInteger regionCnt = taskCounterPerRegion.get(loc.getRegionInfo().getRegionName());
407     if (regionCnt != null && regionCnt.get() >= maxConcurrentTasksPerRegion) {
408       // Too many tasks on this region already.
409       regionsIncluded.put(regionId, Boolean.FALSE);
410       return false;
411     }
412 
413     if (serverPrevious == null) {
414       // The region is ok, but we need to decide for this region server.
415       int newServers = 0; // number of servers we're going to contact so far
416       for (Map.Entry<ServerName, Boolean> kv : serversIncluded.entrySet()) {
417         if (kv.getValue()) {
418           newServers++;
419         }
420       }
421 
422       // Do we have too many total tasks already?
423       boolean ok = (newServers + getCurrentTasksCount()) < maxTotalConcurrentTasks;
424 
425       if (ok) {
426         // If the total is fine, is it ok for this individual server?
427         AtomicInteger serverCnt = taskCounterPerServer.get(loc.getServerName());
428         ok = (serverCnt == null || serverCnt.get() < maxConcurrentTasksPerServer);
429       }
430 
431       if (!ok) {
432         regionsIncluded.put(regionId, Boolean.FALSE);
433         serversIncluded.put(loc.getServerName(), Boolean.FALSE);
434         return false;
435       }
436 
437       serversIncluded.put(loc.getServerName(), Boolean.TRUE);
438     } else {
439       assert serverPrevious.equals(Boolean.TRUE);
440     }
441 
442     regionsIncluded.put(regionId, Boolean.TRUE);
443 
444     return true;
445   }
446 
447   /**
448    * Submit immediately the list of rows, whatever the server status. Kept for backward
449    * compatibility: it allows to be used with the batch interface that return an array of objects.
450    *
451    * @param rows the list of rows.
452    */
453   public void submitAll(List<? extends Row> rows) {
454     List<Action<Row>> actions = new ArrayList<Action<Row>>(rows.size());
455 
456     // The position will be used by the processBatch to match the object array returned.
457     int posInList = -1;
458     NonceGenerator ng = this.hConnection.getNonceGenerator();
459     for (Row r : rows) {
460       posInList++;
461       if (r instanceof Put) {
462         Put put = (Put) r;
463         if (put.isEmpty()) {
464           throw new IllegalArgumentException("No columns to insert for #" + (posInList+1)+ " item");
465         }
466       }
467       Action<Row> action = new Action<Row>(r, posInList);
468       setNonce(ng, r, action);
469       actions.add(action);
470     }
471     HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
472     submit(actions, actions, 1, errorsByServer);
473   }
474 
475   private void setNonce(NonceGenerator ng, Row r, Action<Row> action) {
476     if (!(r instanceof Append) && !(r instanceof Increment)) return;
477     action.setNonce(ng.newNonce()); // Action handles NO_NONCE, so it's ok if ng is disabled.
478   }
479 
480 
481   /**
482    * Group a list of actions per region servers, and send them. The created MultiActions are
483    * added to the inProgress list. Does not take into account the region/server load.
484    *
485    * @param initialActions - the full list of the actions in progress
486    * @param currentActions - the list of row to submit
487    * @param numAttempt - the current numAttempt (first attempt is 1)
488    */
489   private void submit(List<Action<Row>> initialActions,
490                       List<Action<Row>> currentActions, int numAttempt,
491                       final HConnectionManager.ServerErrorTracker errorsByServer) {
492 
493     if (numAttempt > 1){
494       retriesCnt.incrementAndGet();
495     }
496 
497     // group per location => regions server
498     final Map<HRegionLocation, MultiAction<Row>> actionsByServer =
499         new HashMap<HRegionLocation, MultiAction<Row>>();
500 
501     NonceGenerator ng = this.hConnection.getNonceGenerator();
502     for (Action<Row> action : currentActions) {
503       HRegionLocation loc = findDestLocation(action.getAction(), action.getOriginalIndex());
504       if (loc != null) {
505         addAction(loc, action, actionsByServer, ng);
506       }
507     }
508 
509     if (!actionsByServer.isEmpty()) {
510       sendMultiAction(initialActions, actionsByServer, numAttempt, errorsByServer);
511     }
512   }
513 
514   /**
515    * Send a multi action structure to the servers, after a delay depending on the attempt
516    * number. Asynchronous.
517    *
518    * @param initialActions  the list of the actions, flat.
519    * @param actionsByServer the actions structured by regions
520    * @param numAttempt      the attempt number.
521    */
522   public void sendMultiAction(final List<Action<Row>> initialActions,
523                               Map<HRegionLocation, MultiAction<Row>> actionsByServer,
524                               final int numAttempt,
525                               final HConnectionManager.ServerErrorTracker errorsByServer) {
526     // Send the queries and add them to the inProgress list
527     // This iteration is by server (the HRegionLocation comparator is by server portion only).
528     for (Map.Entry<HRegionLocation, MultiAction<Row>> e : actionsByServer.entrySet()) {
529       final HRegionLocation loc = e.getKey();
530       final MultiAction<Row> multiAction = e.getValue();
531       incTaskCounters(multiAction.getRegions(), loc.getServerName());
532       Runnable runnable = Trace.wrap("AsyncProcess.sendMultiAction", new Runnable() {
533         @Override
534         public void run() {
535           MultiResponse res;
536           try {
537             MultiServerCallable<Row> callable = createCallable(loc, multiAction);
538             try {
539               res = createCaller(callable).callWithoutRetries(callable);
540             } catch (IOException e) {
541               // The service itself failed . It may be an error coming from the communication
542               //   layer, but, as well, a functional error raised by the server.
543               receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, e,
544                   errorsByServer);
545               return;
546             } catch (Throwable t) {
547               // This should not happen. Let's log & retry anyway.
548               LOG.error("#" + id + ", Caught throwable while calling. This is unexpected." +
549                   " Retrying. Server is " + loc.getServerName() + ", tableName=" + tableName, t);
550               receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, t,
551                   errorsByServer);
552               return;
553             }
554 
555             // Nominal case: we received an answer from the server, and it's not an exception.
556             receiveMultiAction(initialActions, multiAction, loc, res, numAttempt, errorsByServer);
557 
558           } finally {
559             decTaskCounters(multiAction.getRegions(), loc.getServerName());
560           }
561         }
562       });
563 
564       try {
565         this.pool.submit(runnable);
566       } catch (RejectedExecutionException ree) {
567         // This should never happen. But as the pool is provided by the end user, let's secure
568         //  this a little.
569         decTaskCounters(multiAction.getRegions(), loc.getServerName());
570         LOG.warn("#" + id + ", the task was rejected by the pool. This is unexpected." +
571             " Server is " + loc.getServerName(), ree);
572         // We're likely to fail again, but this will increment the attempt counter, so it will
573         //  finish.
574         receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, ree, errorsByServer);
575       }
576     }
577   }
578 
579   /**
580    * Create a callable. Isolated to be easily overridden in the tests.
581    */
582   protected MultiServerCallable<Row> createCallable(final HRegionLocation location,
583       final MultiAction<Row> multi) {
584     return new MultiServerCallable<Row>(hConnection, tableName, location, this.rpcFactory, multi);
585   }
586 
587   /**
588    * For tests.
589    * @param callable: used in tests.
590    * @return Returns a caller.
591    */
592   protected RpcRetryingCaller<MultiResponse> createCaller(MultiServerCallable<Row> callable) {
593     return rpcCallerFactory.<MultiResponse> newCaller();
594   }
595 
596   /**
597    * Check that we can retry acts accordingly: logs, set the error status, call the callbacks.
598    *
599    * @param originalIndex the position in the list sent
600    * @param row           the row
601    * @param canRetry      if false, we won't retry whatever the settings.
602    * @param throwable     the throwable, if any (can be null)
603    * @param location      the location, if any (can be null)
604    * @return true if the action can be retried, false otherwise.
605    */
606   private boolean manageError(int originalIndex, Row row, boolean canRetry,
607                               Throwable throwable, HRegionLocation location) {
608     if (canRetry && throwable != null && throwable instanceof DoNotRetryIOException) {
609       canRetry = false;
610     }
611 
612     byte[] region = null;
613     if (canRetry && callback != null) {
614       region = location == null ? null : location.getRegionInfo().getEncodedNameAsBytes();
615       canRetry = callback.retriableFailure(originalIndex, row, region, throwable);
616     }
617 
618     if (!canRetry) {
619       if (callback != null) {
620         if (region == null && location != null) {
621           region = location.getRegionInfo().getEncodedNameAsBytes();
622         }
623         callback.failure(originalIndex, region, row, throwable);
624       }
625       errors.add(throwable, row, location);
626       this.hasError.set(true);
627     }
628 
629     return canRetry;
630   }
631 
632   /**
633    * Resubmit all the actions from this multiaction after a failure.
634    *
635    * @param initialActions the full initial action list
636    * @param rsActions  the actions still to do from the initial list
637    * @param location   the destination
638    * @param numAttempt the number of attempts so far
639    * @param t the throwable (if any) that caused the resubmit
640    */
641   private void receiveGlobalFailure(List<Action<Row>> initialActions, MultiAction<Row> rsActions,
642                                     HRegionLocation location, int numAttempt, Throwable t,
643                                     HConnectionManager.ServerErrorTracker errorsByServer) {
644     // Do not use the exception for updating cache because it might be coming from
645     // any of the regions in the MultiAction.
646     hConnection.updateCachedLocations(tableName,
647       rsActions.actions.values().iterator().next().get(0).getAction().getRow(), null, location);
648     errorsByServer.reportServerError(location);
649 
650     List<Action<Row>> toReplay = new ArrayList<Action<Row>>(initialActions.size());
651     for (Map.Entry<byte[], List<Action<Row>>> e : rsActions.actions.entrySet()) {
652       for (Action<Row> action : e.getValue()) {
653         if (manageError(action.getOriginalIndex(), action.getAction(), true, t, location)) {
654           toReplay.add(action);
655         }
656       }
657     }
658 
659     logAndResubmit(initialActions, location, toReplay, numAttempt, rsActions.size(),
660         t, errorsByServer);
661   }
662 
663   /**
664    * Log as many info as possible, and, if there is something to replay, submit it again after
665    *  a back off sleep.
666    */
667   private void logAndResubmit(List<Action<Row>> initialActions, HRegionLocation oldLocation,
668                               List<Action<Row>> toReplay, int numAttempt, int failureCount,
669                               Throwable throwable,
670                               HConnectionManager.ServerErrorTracker errorsByServer) {
671     if (toReplay.isEmpty()) {
672       // it's either a success or a last failure
673       if (failureCount != 0) {
674         // We have a failure but nothing to retry. We're done, it's a final failure..
675         LOG.warn(createLog(numAttempt, failureCount, toReplay.size(),
676             oldLocation.getServerName(), throwable, -1, false,
677             errorsByServer.getStartTrackingTime()));
678       } else if (numAttempt > startLogErrorsCnt + 1) {
679         // The operation was successful, but needed several attempts. Let's log this.
680         LOG.info(createLog(numAttempt, failureCount, 0,
681             oldLocation.getServerName(), throwable, -1, false,
682             errorsByServer.getStartTrackingTime()));
683       }
684       return;
685     }
686 
687     // We have something to replay. We're going to sleep a little before.
688 
689     // We have two contradicting needs here:
690     //  1) We want to get the new location after having slept, as it may change.
691     //  2) We want to take into account the location when calculating the sleep time.
692     // It should be possible to have some heuristics to take the right decision. Short term,
693     //  we go for one.
694     long backOffTime = errorsByServer.calculateBackoffTime(oldLocation, pause);
695 
696     if (numAttempt > startLogErrorsCnt) {
697       // We use this value to have some logs when we have multiple failures, but not too many
698       //  logs, as errors are to be expected when a region moves, splits and so on
699       LOG.info(createLog(numAttempt, failureCount, toReplay.size(),
700           oldLocation.getServerName(), throwable, backOffTime, true,
701           errorsByServer.getStartTrackingTime()));
702     }
703 
704     try {
705       Thread.sleep(backOffTime);
706     } catch (InterruptedException e) {
707       LOG.warn("#" + id + ", not sent: " + toReplay.size() + " operations, " + oldLocation, e);
708       Thread.currentThread().interrupt();
709       return;
710     }
711 
712     submit(initialActions, toReplay, numAttempt + 1, errorsByServer);
713   }
714 
715   /**
716    * Called when we receive the result of a server query.
717    *
718    * @param initialActions - the whole action list
719    * @param multiAction    - the multiAction we sent
720    * @param location       - the location. It's used as a server name.
721    * @param responses      - the response, if any
722    * @param numAttempt     - the attempt
723    */
724   private void receiveMultiAction(List<Action<Row>> initialActions, MultiAction<Row> multiAction,
725                                   HRegionLocation location,
726                                   MultiResponse responses, int numAttempt,
727                                   HConnectionManager.ServerErrorTracker errorsByServer) {
728      assert responses != null;
729 
730     // Success or partial success
731     // Analyze detailed results. We can still have individual failures to be redo.
732     // two specific throwables are managed:
733     //  - DoNotRetryIOException: we continue to retry for other actions
734     //  - RegionMovedException: we update the cache with the new region location
735 
736     List<Action<Row>> toReplay = new ArrayList<Action<Row>>();
737     Throwable throwable = null;
738     int failureCount = 0;
739     boolean canRetry = true;
740 
741     for (Map.Entry<byte[], List<Pair<Integer, Object>>> resultsForRS :
742         responses.getResults().entrySet()) {
743 
744       boolean regionFailureRegistered = false;
745       for (Pair<Integer, Object> regionResult : resultsForRS.getValue()) {
746         Object result = regionResult.getSecond();
747 
748         // Failure: retry if it's make sense else update the errors lists
749         if (result == null || result instanceof Throwable) {
750           throwable = (Throwable) result;
751           Action<Row> correspondingAction = initialActions.get(regionResult.getFirst());
752           Row row = correspondingAction.getAction();
753           failureCount++;
754           if (!regionFailureRegistered) { // We're doing this once per location.
755             regionFailureRegistered= true;
756             // The location here is used as a server name.
757             hConnection.updateCachedLocations(this.tableName, row.getRow(), result, location);
758             if (failureCount == 1) {
759               errorsByServer.reportServerError(location);
760               canRetry = errorsByServer.canRetryMore(numAttempt);
761             }
762           }
763 
764           if (manageError(correspondingAction.getOriginalIndex(), row, canRetry,
765               throwable, location)) {
766             toReplay.add(correspondingAction);
767           }
768         } else { // success
769           if (callback != null) {
770             int index = regionResult.getFirst();
771             Action<Row> correspondingAction = initialActions.get(index);
772             Row row = correspondingAction.getAction();
773             //noinspection unchecked
774             this.callback.success(index, resultsForRS.getKey(), row, (CResult) result);
775           }
776         }
777       }
778     }
779 
780     // The failures global to a region. We will use for multiAction we sent previously to find the
781     //   actions to replay.
782 
783     for (Map.Entry<byte[], Throwable> throwableEntry : responses.getExceptions().entrySet()) {
784       throwable = throwableEntry.getValue();
785       byte[] region =throwableEntry.getKey();
786       List<Action<Row>> actions = multiAction.actions.get(region);
787       if (actions == null || actions.isEmpty()) {
788         throw new IllegalStateException("Wrong response for the region: " +
789             HRegionInfo.encodeRegionName(region));
790       }
791 
792       if (failureCount == 0) {
793         errorsByServer.reportServerError(location);
794         canRetry = errorsByServer.canRetryMore(numAttempt);
795       }
796       hConnection.updateCachedLocations(this.tableName, actions.get(0).getAction().getRow(),
797           throwable, location);
798       failureCount += actions.size();
799 
800       for (Action<Row> action : actions) {
801         Row row = action.getAction();
802         if (manageError(action.getOriginalIndex(), row, canRetry, throwable, location)) {
803           toReplay.add(action);
804         }
805       }
806     }
807 
808     logAndResubmit(initialActions, location, toReplay, numAttempt, failureCount,
809         throwable, errorsByServer);
810   }
811 
812   private String createLog(int numAttempt, int failureCount, int replaySize, ServerName sn,
813                            Throwable error, long backOffTime, boolean willRetry, String startTime){
814     StringBuilder sb = new StringBuilder();
815 
816     sb.append("#").append(id).append(", table=").append(tableName).
817         append(", attempt=").append(numAttempt).append("/").append(numTries).append(" ");
818 
819     if (failureCount > 0 || error != null){
820       sb.append("failed ").append(failureCount).append(" ops").append(", last exception: ").
821           append(error == null ? "null" : error);
822     } else {
823       sb.append("SUCCEEDED");
824     }
825 
826     sb.append(" on ").append(sn);
827 
828     sb.append(", tracking started ").append(startTime);
829 
830     if (willRetry) {
831       sb.append(", retrying after ").append(backOffTime).append(" ms").
832           append(", replay ").append(replaySize).append(" ops.");
833     } else if (failureCount > 0) {
834       sb.append(" - FAILED, NOT RETRYING ANYMORE");
835     }
836 
837     return sb.toString();
838   }
839 
840   /**
841    * Waits for another task to finish.
842    * @param currentNumberOfTask - the number of task finished when calling the method.
843    */
844   protected void waitForNextTaskDone(long currentNumberOfTask) throws InterruptedIOException {
845     while (currentNumberOfTask == tasksDone.get()) {
846       try {
847         synchronized (this.tasksDone) {
848           this.tasksDone.wait(100);
849         }
850       } catch (InterruptedException e) {
851         throw new InterruptedIOException("#" + id + ", interrupted." +
852             " currentNumberOfTask=" + currentNumberOfTask +
853             ",  tableName=" + tableName + ", tasksDone=" + tasksDone.get());
854       }
855     }
856   }
857 
858   /**
859    * Wait until the async does not have more than max tasks in progress.
860    */
861   private void waitForMaximumCurrentTasks(int max) throws InterruptedIOException {
862     long lastLog = EnvironmentEdgeManager.currentTimeMillis();
863     long currentTasksDone = this.tasksDone.get();
864 
865     while ((tasksSent.get() - currentTasksDone) > max) {
866       long now = EnvironmentEdgeManager.currentTimeMillis();
867       if (now > lastLog + 10000) {
868         lastLog = now;
869         LOG.info("#" + id + ", waiting for some tasks to finish. Expected max="
870             + max + ", tasksSent=" + tasksSent.get() + ", tasksDone=" + tasksDone.get() +
871             ", currentTasksDone=" + currentTasksDone + ", retries=" + retriesCnt.get() +
872             " hasError=" + hasError() + ", tableName=" + tableName);
873       }
874       waitForNextTaskDone(currentTasksDone);
875       currentTasksDone = this.tasksDone.get();
876     }
877   }
878 
879   private long getCurrentTasksCount(){
880     return  tasksSent.get() - tasksDone.get();
881   }
882 
883   /**
884    * Wait until all tasks are executed, successfully or not.
885    */
886   public void waitUntilDone() throws InterruptedIOException {
887     waitForMaximumCurrentTasks(0);
888   }
889 
890 
891   public boolean hasError() {
892     return hasError.get();
893   }
894 
895   public List<? extends Row> getFailedOperations() {
896     return errors.actions;
897   }
898 
899   /**
900    * Clean the errors stacks. Should be called only when there are no actions in progress.
901    */
902   public void clearErrors() {
903     errors.clear();
904     hasError.set(false);
905   }
906 
907   public RetriesExhaustedWithDetailsException getErrors() {
908     return errors.makeException();
909   }
910 
911   /**
912    * increment the tasks counters for a given set of regions. MT safe.
913    */
914   protected void incTaskCounters(Collection<byte[]> regions, ServerName sn) {
915     tasksSent.incrementAndGet();
916 
917     AtomicInteger serverCnt = taskCounterPerServer.get(sn);
918     if (serverCnt == null) {
919       taskCounterPerServer.putIfAbsent(sn, new AtomicInteger());
920       serverCnt = taskCounterPerServer.get(sn);
921     }
922     serverCnt.incrementAndGet();
923 
924     for (byte[] regBytes : regions) {
925       AtomicInteger regionCnt = taskCounterPerRegion.get(regBytes);
926       if (regionCnt == null) {
927         regionCnt = new AtomicInteger();
928         AtomicInteger oldCnt = taskCounterPerRegion.putIfAbsent(regBytes, regionCnt);
929         if (oldCnt != null) {
930           regionCnt = oldCnt;
931         }
932       }
933       regionCnt.incrementAndGet();
934     }
935   }
936 
937   /**
938    * Decrements the counters for a given region and the region server. MT Safe.
939    */
940   protected void decTaskCounters(Collection<byte[]> regions, ServerName sn) {
941     for (byte[] regBytes : regions) {
942       AtomicInteger regionCnt = taskCounterPerRegion.get(regBytes);
943       regionCnt.decrementAndGet();
944     }
945 
946     taskCounterPerServer.get(sn).decrementAndGet();
947 
948     tasksDone.incrementAndGet();
949     synchronized (tasksDone) {
950       tasksDone.notifyAll();
951     }
952   }
953 
954   /**
955    * Creates the server error tracker to use inside process.
956    * Currently, to preserve the main assumption about current retries, and to work well with
957    * the retry-limit-based calculation, the calculation is local per Process object.
958    * We may benefit from connection-wide tracking of server errors.
959    * @return ServerErrorTracker to use, null if there is no ServerErrorTracker on this connection
960    */
961   protected HConnectionManager.ServerErrorTracker createServerErrorTracker() {
962     return new HConnectionManager.ServerErrorTracker(this.serverTrackerTimeout, this.numTries);
963   }
964 }