View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.IOException;
23  import java.io.InterruptedIOException;
24  import java.util.ArrayList;
25  import java.util.Arrays;
26  import java.util.Collection;
27  import java.util.HashMap;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Map;
31  import java.util.concurrent.ConcurrentHashMap;
32  import java.util.concurrent.ConcurrentMap;
33  import java.util.concurrent.ConcurrentSkipListMap;
34  import java.util.concurrent.ExecutorService;
35  import java.util.concurrent.RejectedExecutionException;
36  import java.util.concurrent.atomic.AtomicBoolean;
37  import java.util.concurrent.atomic.AtomicInteger;
38  import java.util.concurrent.atomic.AtomicLong;
39  
40  import org.apache.commons.logging.Log;
41  import org.apache.commons.logging.LogFactory;
42  import org.apache.hadoop.conf.Configuration;
43  import org.apache.hadoop.hbase.DoNotRetryIOException;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HRegionLocation;
47  import org.apache.hadoop.hbase.ServerName;
48  import org.apache.hadoop.hbase.TableName;
49  import org.apache.hadoop.hbase.util.Bytes;
50  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
51  import org.apache.hadoop.hbase.util.Pair;
52  import org.cloudera.htrace.Trace;
53  
54  /**
55   * This class  allows a continuous flow of requests. It's written to be compatible with a
56   * synchronous caller such as HTable.
57   * <p>
58   * The caller sends a buffer of operation, by calling submit. This class extract from this list
59   * the operations it can send, i.e. the operations that are on region that are not considered
60   * as busy. The process is asynchronous, i.e. it returns immediately when if has finished to
61   * iterate on the list. If, and only if, the maximum number of current task is reached, the call
62   * to submit will block.
63   * </p>
64   * <p>
65   * The class manages internally the retries.
66   * </p>
67   * <p>
68   * The class includes an error marker: it allows to know if an operation has failed or not, and
69   * to get the exception details, i.e. the full list of throwables for each attempt. This marker
70   * is here to help the backward compatibility in HTable. In most (new) cases, it should be
71   * managed by the callbacks.
72   * </p>
73   * <p>
74   * A callback is available, in order to: <list>
75   * <li>Get the result of the operation (failure or success)</li>
76   * <li>When an operation fails but could be retried, allows or not to retry</li>
77   * <li>When an operation fails for good (can't be retried or already retried the maximum number
78   * time), register the error or not.
79   * </list>
80   * <p>
81   * This class is not thread safe externally; only one thread should submit operations at a time.
82   * Internally, the class is thread safe enough to manage simultaneously new submission and results
83   * arising from older operations.
84   * </p>
85   * <p>
86   * Internally, this class works with {@link Row}, this mean it could be theoretically used for
87   * gets as well.
88   * </p>
89   */
90  class AsyncProcess<CResult> {
91    private static final Log LOG = LogFactory.getLog(AsyncProcess.class);
92    protected static final AtomicLong COUNTER = new AtomicLong();
93    protected final long id;
94    private final int startLogErrorsCnt;
95    protected final HConnection hConnection;
96    protected final TableName tableName;
97    protected final ExecutorService pool;
98    protected final AsyncProcessCallback<CResult> callback;
99    protected final BatchErrors errors = new BatchErrors();
100   protected final AtomicBoolean hasError = new AtomicBoolean(false);
101   protected final AtomicLong tasksSent = new AtomicLong(0);
102   protected final AtomicLong tasksDone = new AtomicLong(0);
103   protected final AtomicLong retriesCnt = new AtomicLong(0);
104   protected final ConcurrentMap<byte[], AtomicInteger> taskCounterPerRegion =
105       new ConcurrentSkipListMap<byte[], AtomicInteger>(Bytes.BYTES_COMPARATOR);
106   protected final ConcurrentMap<ServerName, AtomicInteger> taskCounterPerServer =
107       new ConcurrentHashMap<ServerName, AtomicInteger>();
108 
109   /**
110    * The number of tasks simultaneously executed on the cluster.
111    */
112   protected final int maxTotalConcurrentTasks;
113 
114   /**
115    * The number of tasks we run in parallel on a single region.
116    * With 1 (the default) , we ensure that the ordering of the queries is respected: we don't start
117    * a set of operations on a region before the previous one is done. As well, this limits
118    * the pressure we put on the region server.
119    */
120   protected final int maxConcurrentTasksPerRegion;
121 
122   /**
123    * The number of task simultaneously executed on a single region server.
124    */
125   protected final int maxConcurrentTasksPerServer;
126   protected final long pause;
127   protected int numTries;
128   protected int serverTrackerTimeout;
129   protected RpcRetryingCallerFactory rpcCallerFactory;
130 
131 
132   /**
133    * This interface allows to keep the interface of the previous synchronous interface, that uses
134    * an array of object to return the result.
135    * <p/>
136    * This interface allows the caller to specify the behavior on errors: <list>
137    * <li>If we have not yet reach the maximum number of retries, the user can nevertheless
138    * specify if this specific operation should be retried or not.
139    * </li>
140    * <li>If an operation fails (i.e. is not retried or fails after all retries), the user can
141    * specify is we should mark this AsyncProcess as in error or not.
142    * </li>
143    * </list>
144    */
145   interface AsyncProcessCallback<CResult> {
146 
147     /**
148      * Called on success. originalIndex holds the index in the action list.
149      */
150     void success(int originalIndex, byte[] region, Row row, CResult result);
151 
152     /**
153      * called on failure, if we don't retry (i.e. called once per failed operation).
154      *
155      * @return true if we should store the error and tag this async process as being in error.
156      *         false if the failure of this operation can be safely ignored, and does not require
157      *         the current process to be stopped without proceeding with the other operations in
158      *         the queue.
159      */
160     boolean failure(int originalIndex, byte[] region, Row row, Throwable t);
161 
162     /**
163      * Called on a failure we plan to retry. This allows the user to stop retrying. Will be
164      * called multiple times for a single action if it fails multiple times.
165      *
166      * @return false if we should retry, true otherwise.
167      */
168     boolean retriableFailure(int originalIndex, Row row, byte[] region, Throwable exception);
169   }
170 
171   private static class BatchErrors {
172     private final List<Throwable> throwables = new ArrayList<Throwable>();
173     private final List<Row> actions = new ArrayList<Row>();
174     private final List<String> addresses = new ArrayList<String>();
175 
176     public synchronized void add(Throwable ex, Row row, HRegionLocation location) {
177       if (row == null){
178         throw new IllegalArgumentException("row cannot be null. location=" + location);
179       }
180 
181       throwables.add(ex);
182       actions.add(row);
183       addresses.add(location != null ? location.getServerName().toString() : "null location");
184     }
185 
186     private synchronized RetriesExhaustedWithDetailsException makeException() {
187       return new RetriesExhaustedWithDetailsException(
188           new ArrayList<Throwable>(throwables),
189           new ArrayList<Row>(actions), new ArrayList<String>(addresses));
190     }
191 
192     public synchronized void clear() {
193       throwables.clear();
194       actions.clear();
195       addresses.clear();
196     }
197   }
198 
199   public AsyncProcess(HConnection hc, TableName tableName, ExecutorService pool,
200       AsyncProcessCallback<CResult> callback, Configuration conf,
201       RpcRetryingCallerFactory rpcCaller) {
202     if (hc == null){
203       throw new IllegalArgumentException("HConnection cannot be null.");
204     }
205 
206     this.hConnection = hc;
207     this.tableName = tableName;
208     this.pool = pool;
209     this.callback = callback;
210 
211     this.id = COUNTER.incrementAndGet();
212 
213     this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
214         HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
215     this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
216         HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
217 
218     this.maxTotalConcurrentTasks = conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
219       HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS);
220     this.maxConcurrentTasksPerServer = conf.getInt(HConstants.HBASE_CLIENT_MAX_PERSERVER_TASKS,
221           HConstants.DEFAULT_HBASE_CLIENT_MAX_PERSERVER_TASKS);
222     this.maxConcurrentTasksPerRegion = conf.getInt(HConstants.HBASE_CLIENT_MAX_PERREGION_TASKS,
223           HConstants.DEFAULT_HBASE_CLIENT_MAX_PERREGION_TASKS);
224 
225     // A few failure is fine: region moved, then is not opened, then is overloaded. We try
226     //  to have an acceptable heuristic for the number of errors we don't log.
227     //  9 was chosen because we wait for 1s at this stage.
228     this.startLogErrorsCnt = conf.getInt("hbase.client.start.log.errors.counter", 9);
229 
230     if (this.maxTotalConcurrentTasks <= 0) {
231       throw new IllegalArgumentException("maxTotalConcurrentTasks=" + maxTotalConcurrentTasks);
232     }
233     if (this.maxConcurrentTasksPerServer <= 0) {
234       throw new IllegalArgumentException("maxConcurrentTasksPerServer=" +
235           maxConcurrentTasksPerServer);
236     }
237     if (this.maxConcurrentTasksPerRegion <= 0) {
238       throw new IllegalArgumentException("maxConcurrentTasksPerRegion=" +
239           maxConcurrentTasksPerRegion);
240     }
241 
242     // Server tracker allows us to do faster, and yet useful (hopefully), retries.
243     // However, if we are too useful, we might fail very quickly due to retry count limit.
244     // To avoid this, we are going to cheat for now (see HBASE-7659), and calculate maximum
245     // retry time if normal retries were used. Then we will retry until this time runs out.
246     // If we keep hitting one server, the net effect will be the incremental backoff, and
247     // essentially the same number of retries as planned. If we have to do faster retries,
248     // we will do more retries in aggregate, but the user will be none the wiser.
249     this.serverTrackerTimeout = 0;
250     for (int i = 0; i < this.numTries; ++i) {
251       serverTrackerTimeout += ConnectionUtils.getPauseTime(this.pause, i);
252     }
253 
254 
255     this.rpcCallerFactory = rpcCaller;
256   }
257 
258   /**
259    * Extract from the rows list what we can submit. The rows we can not submit are kept in the
260    * list.
261    *
262    * @param rows - the submitted row. Modified by the method: we remove the rows we took.
263    * @param atLeastOne true if we should submit at least a subset.
264    */
265   public void submit(List<? extends Row> rows, boolean atLeastOne) throws InterruptedIOException {
266     if (rows.isEmpty()) {
267       return;
268     }
269 
270     // This looks like we are keying by region but HRegionLocation has a comparator that compares
271     // on the server portion only (hostname + port) so this Map collects regions by server.
272     Map<HRegionLocation, MultiAction<Row>> actionsByServer =
273       new HashMap<HRegionLocation, MultiAction<Row>>();
274     List<Action<Row>> retainedActions = new ArrayList<Action<Row>>(rows.size());
275 
276     long currentTaskCnt = tasksDone.get();
277     boolean alreadyLooped = false;
278 
279     NonceGenerator ng = this.hConnection.getNonceGenerator();
280     do {
281       if (alreadyLooped){
282         // if, for whatever reason, we looped, we want to be sure that something has changed.
283         waitForNextTaskDone(currentTaskCnt);
284         currentTaskCnt = tasksDone.get();
285       } else {
286         alreadyLooped = true;
287       }
288 
289       // Wait until there is at least one slot for a new task.
290       waitForMaximumCurrentTasks(maxTotalConcurrentTasks - 1);
291 
292       // Remember the previous decisions about regions or region servers we put in the
293       //  final multi.
294       Map<Long, Boolean> regionIncluded = new HashMap<Long, Boolean>();
295       Map<ServerName, Boolean> serverIncluded = new HashMap<ServerName, Boolean>();
296 
297       int posInList = -1;
298       Iterator<? extends Row> it = rows.iterator();
299       while (it.hasNext()) {
300         Row r = it.next();
301         HRegionLocation loc = findDestLocation(r, posInList);
302 
303         if (loc == null) { // loc is null if there is an error such as meta not available.
304           it.remove();
305         } else if (canTakeOperation(loc, regionIncluded, serverIncluded)) {
306           Action<Row> action = new Action<Row>(r, ++posInList);
307           setNonce(ng, r, action);
308           retainedActions.add(action);
309           addAction(loc, action, actionsByServer, ng);
310           it.remove();
311         }
312       }
313     } while (retainedActions.isEmpty() && atLeastOne && !hasError());
314 
315     HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
316     sendMultiAction(retainedActions, actionsByServer, 1, errorsByServer);
317   }
318 
319   /**
320    * Group the actions per region server.
321    *
322    * @param loc - the destination. Must not be null.
323    * @param action - the action to add to the multiaction
324    * @param actionsByServer the multiaction per server
325    * @param ng Nonce generator, or null if no nonces are needed.
326    */
327   private void addAction(HRegionLocation loc, Action<Row> action, Map<HRegionLocation,
328       MultiAction<Row>> actionsByServer, NonceGenerator ng) {
329     final byte[] regionName = loc.getRegionInfo().getRegionName();
330     MultiAction<Row> multiAction = actionsByServer.get(loc);
331     if (multiAction == null) {
332       multiAction = new MultiAction<Row>();
333       actionsByServer.put(loc, multiAction);
334     }
335     if (action.hasNonce() && !multiAction.hasNonceGroup()) {
336       // TODO: this code executes for every (re)try, and calls getNonceGroup again
337       //       for the same action. It must return the same value across calls.
338       multiAction.setNonceGroup(ng.getNonceGroup());
339     }
340 
341     multiAction.add(regionName, action);
342   }
343 
344   /**
345    * Find the destination.
346    *
347    * @param row          the row
348    * @param posInList    the position in the list
349    * @return the destination. Null if we couldn't find it.
350    */
351   private HRegionLocation findDestLocation(Row row, int posInList) {
352     if (row == null) throw new IllegalArgumentException("#" + id + ", row cannot be null");
353     HRegionLocation loc = null;
354     IOException locationException = null;
355     try {
356       loc = hConnection.locateRegion(this.tableName, row.getRow());
357       if (loc == null) {
358         locationException = new IOException("#" + id + ", no location found, aborting submit for" +
359             " tableName=" + tableName +
360             " rowkey=" + Arrays.toString(row.getRow()));
361       }
362     } catch (IOException e) {
363       locationException = e;
364     }
365     if (locationException != null) {
366       // There are multiple retries in locateRegion already. No need to add new.
367       // We can't continue with this row, hence it's the last retry.
368       manageError(posInList, row, false, locationException, null);
369       return null;
370     }
371 
372     return loc;
373   }
374 
375   /**
376    * Check if we should send new operations to this region or region server.
377    * We're taking into account the past decision; if we have already accepted
378    * operation on a given region, we accept all operations for this region.
379    *
380    * @param loc; the region and the server name we want to use.
381    * @return true if this region is considered as busy.
382    */
383   protected boolean canTakeOperation(HRegionLocation loc,
384                                      Map<Long, Boolean> regionsIncluded,
385                                      Map<ServerName, Boolean> serversIncluded) {
386     long regionId = loc.getRegionInfo().getRegionId();
387     Boolean regionPrevious = regionsIncluded.get(regionId);
388 
389     if (regionPrevious != null) {
390       // We already know what to do with this region.
391       return regionPrevious;
392     }
393 
394     Boolean serverPrevious = serversIncluded.get(loc.getServerName());
395     if (Boolean.FALSE.equals(serverPrevious)) {
396       // It's a new region, on a region server that we have already excluded.
397       regionsIncluded.put(regionId, Boolean.FALSE);
398       return false;
399     }
400 
401     AtomicInteger regionCnt = taskCounterPerRegion.get(loc.getRegionInfo().getRegionName());
402     if (regionCnt != null && regionCnt.get() >= maxConcurrentTasksPerRegion) {
403       // Too many tasks on this region already.
404       regionsIncluded.put(regionId, Boolean.FALSE);
405       return false;
406     }
407 
408     if (serverPrevious == null) {
409       // The region is ok, but we need to decide for this region server.
410       int newServers = 0; // number of servers we're going to contact so far
411       for (Map.Entry<ServerName, Boolean> kv : serversIncluded.entrySet()) {
412         if (kv.getValue()) {
413           newServers++;
414         }
415       }
416 
417       // Do we have too many total tasks already?
418       boolean ok = (newServers + getCurrentTasksCount()) < maxTotalConcurrentTasks;
419 
420       if (ok) {
421         // If the total is fine, is it ok for this individual server?
422         AtomicInteger serverCnt = taskCounterPerServer.get(loc.getServerName());
423         ok = (serverCnt == null || serverCnt.get() < maxConcurrentTasksPerServer);
424       }
425 
426       if (!ok) {
427         regionsIncluded.put(regionId, Boolean.FALSE);
428         serversIncluded.put(loc.getServerName(), Boolean.FALSE);
429         return false;
430       }
431 
432       serversIncluded.put(loc.getServerName(), Boolean.TRUE);
433     } else {
434       assert serverPrevious.equals(Boolean.TRUE);
435     }
436 
437     regionsIncluded.put(regionId, Boolean.TRUE);
438 
439     return true;
440   }
441 
442   /**
443    * Submit immediately the list of rows, whatever the server status. Kept for backward
444    * compatibility: it allows to be used with the batch interface that return an array of objects.
445    *
446    * @param rows the list of rows.
447    */
448   public void submitAll(List<? extends Row> rows) {
449     List<Action<Row>> actions = new ArrayList<Action<Row>>(rows.size());
450 
451     // The position will be used by the processBatch to match the object array returned.
452     int posInList = -1;
453     NonceGenerator ng = this.hConnection.getNonceGenerator();
454     for (Row r : rows) {
455       posInList++;
456       if (r instanceof Put) {
457         Put put = (Put) r;
458         if (put.isEmpty()) {
459           throw new IllegalArgumentException("No columns to insert for #" + (posInList+1)+ " item");
460         }
461       }
462       Action<Row> action = new Action<Row>(r, posInList);
463       setNonce(ng, r, action);
464       actions.add(action);
465     }
466     HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
467     submit(actions, actions, 1, errorsByServer);
468   }
469 
470   private void setNonce(NonceGenerator ng, Row r, Action<Row> action) {
471     if (!(r instanceof Append) && !(r instanceof Increment)) return;
472     action.setNonce(ng.newNonce()); // Action handles NO_NONCE, so it's ok if ng is disabled.
473   }
474 
475 
476   /**
477    * Group a list of actions per region servers, and send them. The created MultiActions are
478    * added to the inProgress list. Does not take into account the region/server load.
479    *
480    * @param initialActions - the full list of the actions in progress
481    * @param currentActions - the list of row to submit
482    * @param numAttempt - the current numAttempt (first attempt is 1)
483    */
484   private void submit(List<Action<Row>> initialActions,
485                       List<Action<Row>> currentActions, int numAttempt,
486                       final HConnectionManager.ServerErrorTracker errorsByServer) {
487 
488     if (numAttempt > 1){
489       retriesCnt.incrementAndGet();
490     }
491 
492     // group per location => regions server
493     final Map<HRegionLocation, MultiAction<Row>> actionsByServer =
494         new HashMap<HRegionLocation, MultiAction<Row>>();
495 
496     NonceGenerator ng = this.hConnection.getNonceGenerator();
497     for (Action<Row> action : currentActions) {
498       HRegionLocation loc = findDestLocation(action.getAction(), action.getOriginalIndex());
499       if (loc != null) {
500         addAction(loc, action, actionsByServer, ng);
501       }
502     }
503 
504     if (!actionsByServer.isEmpty()) {
505       sendMultiAction(initialActions, actionsByServer, numAttempt, errorsByServer);
506     }
507   }
508 
509   /**
510    * Send a multi action structure to the servers, after a delay depending on the attempt
511    * number. Asynchronous.
512    *
513    * @param initialActions  the list of the actions, flat.
514    * @param actionsByServer the actions structured by regions
515    * @param numAttempt      the attempt number.
516    */
517   public void sendMultiAction(final List<Action<Row>> initialActions,
518                               Map<HRegionLocation, MultiAction<Row>> actionsByServer,
519                               final int numAttempt,
520                               final HConnectionManager.ServerErrorTracker errorsByServer) {
521     // Send the queries and add them to the inProgress list
522     // This iteration is by server (the HRegionLocation comparator is by server portion only).
523     for (Map.Entry<HRegionLocation, MultiAction<Row>> e : actionsByServer.entrySet()) {
524       final HRegionLocation loc = e.getKey();
525       final MultiAction<Row> multiAction = e.getValue();
526       incTaskCounters(multiAction.getRegions(), loc.getServerName());
527       Runnable runnable = Trace.wrap("AsyncProcess.sendMultiAction", new Runnable() {
528         @Override
529         public void run() {
530           MultiResponse res;
531           try {
532             MultiServerCallable<Row> callable = createCallable(loc, multiAction);
533             try {
534               res = createCaller(callable).callWithoutRetries(callable);
535             } catch (IOException e) {
536               // The service itself failed . It may be an error coming from the communication
537               //   layer, but, as well, a functional error raised by the server.
538               receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, e,
539                   errorsByServer);
540               return;
541             } catch (Throwable t) {
542               // This should not happen. Let's log & retry anyway.
543               LOG.error("#" + id + ", Caught throwable while calling. This is unexpected." +
544                   " Retrying. Server is " + loc.getServerName() + ", tableName=" + tableName, t);
545               receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, t,
546                   errorsByServer);
547               return;
548             }
549 
550             // Nominal case: we received an answer from the server, and it's not an exception.
551             receiveMultiAction(initialActions, multiAction, loc, res, numAttempt, errorsByServer);
552 
553           } finally {
554             decTaskCounters(multiAction.getRegions(), loc.getServerName());
555           }
556         }
557       });
558 
559       try {
560         this.pool.submit(runnable);
561       } catch (RejectedExecutionException ree) {
562         // This should never happen. But as the pool is provided by the end user, let's secure
563         //  this a little.
564         decTaskCounters(multiAction.getRegions(), loc.getServerName());
565         LOG.warn("#" + id + ", the task was rejected by the pool. This is unexpected." +
566             " Server is " + loc.getServerName(), ree);
567         // We're likely to fail again, but this will increment the attempt counter, so it will
568         //  finish.
569         receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, ree, errorsByServer);
570       }
571     }
572   }
573 
574   /**
575    * Create a callable. Isolated to be easily overridden in the tests.
576    */
577   protected MultiServerCallable<Row> createCallable(final HRegionLocation location,
578       final MultiAction<Row> multi) {
579     return new MultiServerCallable<Row>(hConnection, tableName, location, multi);
580   }
581 
582   /**
583    * For tests.
584    * @param callable: used in tests.
585    * @return Returns a caller.
586    */
587   protected RpcRetryingCaller<MultiResponse> createCaller(MultiServerCallable<Row> callable) {
588     return rpcCallerFactory.<MultiResponse> newCaller();
589   }
590 
591   /**
592    * Check that we can retry acts accordingly: logs, set the error status, call the callbacks.
593    *
594    * @param originalIndex the position in the list sent
595    * @param row           the row
596    * @param canRetry      if false, we won't retry whatever the settings.
597    * @param throwable     the throwable, if any (can be null)
598    * @param location      the location, if any (can be null)
599    * @return true if the action can be retried, false otherwise.
600    */
601   private boolean manageError(int originalIndex, Row row, boolean canRetry,
602                               Throwable throwable, HRegionLocation location) {
603     if (canRetry && throwable != null && throwable instanceof DoNotRetryIOException) {
604       canRetry = false;
605     }
606 
607     byte[] region = null;
608     if (canRetry && callback != null) {
609       region = location == null ? null : location.getRegionInfo().getEncodedNameAsBytes();
610       canRetry = callback.retriableFailure(originalIndex, row, region, throwable);
611     }
612 
613     if (!canRetry) {
614       if (callback != null) {
615         if (region == null && location != null) {
616           region = location.getRegionInfo().getEncodedNameAsBytes();
617         }
618         callback.failure(originalIndex, region, row, throwable);
619       }
620       errors.add(throwable, row, location);
621       this.hasError.set(true);
622     }
623 
624     return canRetry;
625   }
626 
627   /**
628    * Resubmit all the actions from this multiaction after a failure.
629    *
630    * @param initialActions the full initial action list
631    * @param rsActions  the actions still to do from the initial list
632    * @param location   the destination
633    * @param numAttempt the number of attempts so far
634    * @param t the throwable (if any) that caused the resubmit
635    */
636   private void receiveGlobalFailure(List<Action<Row>> initialActions, MultiAction<Row> rsActions,
637                                     HRegionLocation location, int numAttempt, Throwable t,
638                                     HConnectionManager.ServerErrorTracker errorsByServer) {
639     // Do not use the exception for updating cache because it might be coming from
640     // any of the regions in the MultiAction.
641     hConnection.updateCachedLocations(tableName,
642       rsActions.actions.values().iterator().next().get(0).getAction().getRow(), null, location);
643     errorsByServer.reportServerError(location);
644 
645     List<Action<Row>> toReplay = new ArrayList<Action<Row>>(initialActions.size());
646     for (Map.Entry<byte[], List<Action<Row>>> e : rsActions.actions.entrySet()) {
647       for (Action<Row> action : e.getValue()) {
648         if (manageError(action.getOriginalIndex(), action.getAction(), true, t, location)) {
649           toReplay.add(action);
650         }
651       }
652     }
653 
654     logAndResubmit(initialActions, location, toReplay, numAttempt, rsActions.size(),
655         t, errorsByServer);
656   }
657 
658   /**
659    * Log as many info as possible, and, if there is something to replay, submit it again after
660    *  a back off sleep.
661    */
662   private void logAndResubmit(List<Action<Row>> initialActions, HRegionLocation oldLocation,
663                               List<Action<Row>> toReplay, int numAttempt, int failureCount,
664                               Throwable throwable,
665                               HConnectionManager.ServerErrorTracker errorsByServer) {
666     if (toReplay.isEmpty()) {
667       // it's either a success or a last failure
668       if (failureCount != 0) {
669         // We have a failure but nothing to retry. We're done, it's a final failure..
670         LOG.warn(createLog(numAttempt, failureCount, toReplay.size(),
671             oldLocation.getServerName(), throwable, -1, false,
672             errorsByServer.getStartTrackingTime()));
673       } else if (numAttempt > startLogErrorsCnt + 1) {
674         // The operation was successful, but needed several attempts. Let's log this.
675         LOG.info(createLog(numAttempt, failureCount, 0,
676             oldLocation.getServerName(), throwable, -1, false,
677             errorsByServer.getStartTrackingTime()));
678       }
679       return;
680     }
681 
682     // We have something to replay. We're going to sleep a little before.
683 
684     // We have two contradicting needs here:
685     //  1) We want to get the new location after having slept, as it may change.
686     //  2) We want to take into account the location when calculating the sleep time.
687     // It should be possible to have some heuristics to take the right decision. Short term,
688     //  we go for one.
689     long backOffTime = errorsByServer.calculateBackoffTime(oldLocation, pause);
690 
691     if (numAttempt > startLogErrorsCnt) {
692       // We use this value to have some logs when we have multiple failures, but not too many
693       //  logs, as errors are to be expected when a region moves, splits and so on
694       LOG.info(createLog(numAttempt, failureCount, toReplay.size(),
695           oldLocation.getServerName(), throwable, backOffTime, true,
696           errorsByServer.getStartTrackingTime()));
697     }
698 
699     try {
700       Thread.sleep(backOffTime);
701     } catch (InterruptedException e) {
702       LOG.warn("#" + id + ", not sent: " + toReplay.size() + " operations, " + oldLocation, e);
703       Thread.currentThread().interrupt();
704       return;
705     }
706 
707     submit(initialActions, toReplay, numAttempt + 1, errorsByServer);
708   }
709 
710   /**
711    * Called when we receive the result of a server query.
712    *
713    * @param initialActions - the whole action list
714    * @param multiAction    - the multiAction we sent
715    * @param location       - the location. It's used as a server name.
716    * @param responses      - the response, if any
717    * @param numAttempt     - the attempt
718    */
719   private void receiveMultiAction(List<Action<Row>> initialActions, MultiAction<Row> multiAction,
720                                   HRegionLocation location,
721                                   MultiResponse responses, int numAttempt,
722                                   HConnectionManager.ServerErrorTracker errorsByServer) {
723      assert responses != null;
724 
725     // Success or partial success
726     // Analyze detailed results. We can still have individual failures to be redo.
727     // two specific throwables are managed:
728     //  - DoNotRetryIOException: we continue to retry for other actions
729     //  - RegionMovedException: we update the cache with the new region location
730 
731     List<Action<Row>> toReplay = new ArrayList<Action<Row>>();
732     Throwable throwable = null;
733     int failureCount = 0;
734     boolean canRetry = true;
735 
736     for (Map.Entry<byte[], List<Pair<Integer, Object>>> resultsForRS :
737         responses.getResults().entrySet()) {
738 
739       boolean regionFailureRegistered = false;
740       for (Pair<Integer, Object> regionResult : resultsForRS.getValue()) {
741         Object result = regionResult.getSecond();
742 
743         // Failure: retry if it's make sense else update the errors lists
744         if (result == null || result instanceof Throwable) {
745           throwable = (Throwable) result;
746           Action<Row> correspondingAction = initialActions.get(regionResult.getFirst());
747           Row row = correspondingAction.getAction();
748           failureCount++;
749           if (!regionFailureRegistered) { // We're doing this once per location.
750             regionFailureRegistered= true;
751             // The location here is used as a server name.
752             hConnection.updateCachedLocations(this.tableName, row.getRow(), result, location);
753             if (failureCount == 1) {
754               errorsByServer.reportServerError(location);
755               canRetry = errorsByServer.canRetryMore(numAttempt);
756             }
757           }
758 
759           if (manageError(correspondingAction.getOriginalIndex(), row, canRetry,
760               throwable, location)) {
761             toReplay.add(correspondingAction);
762           }
763         } else { // success
764           if (callback != null) {
765             int index = regionResult.getFirst();
766             Action<Row> correspondingAction = initialActions.get(index);
767             Row row = correspondingAction.getAction();
768             //noinspection unchecked
769             this.callback.success(index, resultsForRS.getKey(), row, (CResult) result);
770           }
771         }
772       }
773     }
774 
775     // The failures global to a region. We will use for multiAction we sent previously to find the
776     //   actions to replay.
777 
778     for (Map.Entry<byte[], Throwable> throwableEntry : responses.getExceptions().entrySet()) {
779       throwable = throwableEntry.getValue();
780       byte[] region =throwableEntry.getKey();
781       List<Action<Row>> actions = multiAction.actions.get(region);
782       if (actions == null || actions.isEmpty()) {
783         throw new IllegalStateException("Wrong response for the region: " +
784             HRegionInfo.encodeRegionName(region));
785       }
786 
787       if (failureCount == 0) {
788         errorsByServer.reportServerError(location);
789         canRetry = errorsByServer.canRetryMore(numAttempt);
790       }
791       hConnection.updateCachedLocations(this.tableName, actions.get(0).getAction().getRow(),
792           throwable, location);
793       failureCount += actions.size();
794 
795       for (Action<Row> action : actions) {
796         Row row = action.getAction();
797         if (manageError(action.getOriginalIndex(), row, canRetry, throwable, location)) {
798           toReplay.add(action);
799         }
800       }
801     }
802 
803     logAndResubmit(initialActions, location, toReplay, numAttempt, failureCount,
804         throwable, errorsByServer);
805   }
806 
807   private String createLog(int numAttempt, int failureCount, int replaySize, ServerName sn,
808                            Throwable error, long backOffTime, boolean willRetry, String startTime){
809     StringBuilder sb = new StringBuilder();
810 
811     sb.append("#").append(id).append(", table=").append(tableName).
812         append(", attempt=").append(numAttempt).append("/").append(numTries).append(" ");
813 
814     if (failureCount > 0 || error != null){
815       sb.append("failed ").append(failureCount).append(" ops").append(", last exception: ").
816           append(error == null ? "null" : error);
817     } else {
818       sb.append("SUCCEEDED");
819     }
820 
821     sb.append(" on ").append(sn);
822 
823     sb.append(", tracking started ").append(startTime);
824 
825     if (willRetry) {
826       sb.append(", retrying after ").append(backOffTime).append(" ms").
827           append(", replay ").append(replaySize).append(" ops.");
828     } else if (failureCount > 0) {
829       sb.append(" - FAILED, NOT RETRYING ANYMORE");
830     }
831 
832     return sb.toString();
833   }
834 
835   /**
836    * Waits for another task to finish.
837    * @param currentNumberOfTask - the number of task finished when calling the method.
838    */
839   protected void waitForNextTaskDone(long currentNumberOfTask) throws InterruptedIOException {
840     while (currentNumberOfTask == tasksDone.get()) {
841       try {
842         synchronized (this.tasksDone) {
843           this.tasksDone.wait(100);
844         }
845       } catch (InterruptedException e) {
846         throw new InterruptedIOException("#" + id + ", interrupted." +
847             " currentNumberOfTask=" + currentNumberOfTask +
848             ",  tableName=" + tableName + ", tasksDone=" + tasksDone.get());
849       }
850     }
851   }
852 
853   /**
854    * Wait until the async does not have more than max tasks in progress.
855    */
856   private void waitForMaximumCurrentTasks(int max) throws InterruptedIOException {
857     long lastLog = EnvironmentEdgeManager.currentTimeMillis();
858     long currentTasksDone = this.tasksDone.get();
859 
860     while ((tasksSent.get() - currentTasksDone) > max) {
861       long now = EnvironmentEdgeManager.currentTimeMillis();
862       if (now > lastLog + 10000) {
863         lastLog = now;
864         LOG.info("#" + id + ", waiting for some tasks to finish. Expected max="
865             + max + ", tasksSent=" + tasksSent.get() + ", tasksDone=" + tasksDone.get() +
866             ", currentTasksDone=" + currentTasksDone + ", retries=" + retriesCnt.get() +
867             " hasError=" + hasError() + ", tableName=" + tableName);
868       }
869       waitForNextTaskDone(currentTasksDone);
870       currentTasksDone = this.tasksDone.get();
871     }
872   }
873 
874   private long getCurrentTasksCount(){
875     return  tasksSent.get() - tasksDone.get();
876   }
877 
878   /**
879    * Wait until all tasks are executed, successfully or not.
880    */
881   public void waitUntilDone() throws InterruptedIOException {
882     waitForMaximumCurrentTasks(0);
883   }
884 
885 
886   public boolean hasError() {
887     return hasError.get();
888   }
889 
890   public List<? extends Row> getFailedOperations() {
891     return errors.actions;
892   }
893 
894   /**
895    * Clean the errors stacks. Should be called only when there are no actions in progress.
896    */
897   public void clearErrors() {
898     errors.clear();
899     hasError.set(false);
900   }
901 
902   public RetriesExhaustedWithDetailsException getErrors() {
903     return errors.makeException();
904   }
905 
906   /**
907    * increment the tasks counters for a given set of regions. MT safe.
908    */
909   protected void incTaskCounters(Collection<byte[]> regions, ServerName sn) {
910     tasksSent.incrementAndGet();
911 
912     AtomicInteger serverCnt = taskCounterPerServer.get(sn);
913     if (serverCnt == null) {
914       taskCounterPerServer.putIfAbsent(sn, new AtomicInteger());
915       serverCnt = taskCounterPerServer.get(sn);
916     }
917     serverCnt.incrementAndGet();
918 
919     for (byte[] regBytes : regions) {
920       AtomicInteger regionCnt = taskCounterPerRegion.get(regBytes);
921       if (regionCnt == null) {
922         regionCnt = new AtomicInteger();
923         AtomicInteger oldCnt = taskCounterPerRegion.putIfAbsent(regBytes, regionCnt);
924         if (oldCnt != null) {
925           regionCnt = oldCnt;
926         }
927       }
928       regionCnt.incrementAndGet();
929     }
930   }
931 
932   /**
933    * Decrements the counters for a given region and the region server. MT Safe.
934    */
935   protected void decTaskCounters(Collection<byte[]> regions, ServerName sn) {
936     for (byte[] regBytes : regions) {
937       AtomicInteger regionCnt = taskCounterPerRegion.get(regBytes);
938       regionCnt.decrementAndGet();
939     }
940 
941     taskCounterPerServer.get(sn).decrementAndGet();
942 
943     tasksDone.incrementAndGet();
944     synchronized (tasksDone) {
945       tasksDone.notifyAll();
946     }
947   }
948 
949   /**
950    * Creates the server error tracker to use inside process.
951    * Currently, to preserve the main assumption about current retries, and to work well with
952    * the retry-limit-based calculation, the calculation is local per Process object.
953    * We may benefit from connection-wide tracking of server errors.
954    * @return ServerErrorTracker to use, null if there is no ServerErrorTracker on this connection
955    */
956   protected HConnectionManager.ServerErrorTracker createServerErrorTracker() {
957     return new HConnectionManager.ServerErrorTracker(this.serverTrackerTimeout, this.numTries);
958   }
959 }