1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.Collection;
27 import java.util.HashMap;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.ConcurrentHashMap;
32 import java.util.concurrent.ConcurrentMap;
33 import java.util.concurrent.ConcurrentSkipListMap;
34 import java.util.concurrent.ExecutorService;
35 import java.util.concurrent.RejectedExecutionException;
36 import java.util.concurrent.atomic.AtomicBoolean;
37 import java.util.concurrent.atomic.AtomicInteger;
38 import java.util.concurrent.atomic.AtomicLong;
39
40 import org.apache.commons.logging.Log;
41 import org.apache.commons.logging.LogFactory;
42 import org.apache.hadoop.conf.Configuration;
43 import org.apache.hadoop.hbase.DoNotRetryIOException;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HRegionLocation;
47 import org.apache.hadoop.hbase.ServerName;
48 import org.apache.hadoop.hbase.TableName;
49 import org.apache.hadoop.hbase.ipc.RpcControllerFactory;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
52 import org.apache.hadoop.hbase.util.Pair;
53 import org.cloudera.htrace.Trace;
54
55 import com.google.common.base.Preconditions;
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93 class AsyncProcess<CResult> {
94 private static final Log LOG = LogFactory.getLog(AsyncProcess.class);
95 protected static final AtomicLong COUNTER = new AtomicLong();
96 protected final long id;
97 private final int startLogErrorsCnt;
98 protected final HConnection hConnection;
99 protected final TableName tableName;
100 protected final ExecutorService pool;
101 protected final AsyncProcessCallback<CResult> callback;
102 protected final BatchErrors errors = new BatchErrors();
103 protected final AtomicBoolean hasError = new AtomicBoolean(false);
104 protected final AtomicLong tasksSent = new AtomicLong(0);
105 protected final AtomicLong tasksDone = new AtomicLong(0);
106 protected final AtomicLong retriesCnt = new AtomicLong(0);
107 protected final ConcurrentMap<byte[], AtomicInteger> taskCounterPerRegion =
108 new ConcurrentSkipListMap<byte[], AtomicInteger>(Bytes.BYTES_COMPARATOR);
109 protected final ConcurrentMap<ServerName, AtomicInteger> taskCounterPerServer =
110 new ConcurrentHashMap<ServerName, AtomicInteger>();
111
112
113
114
115 protected final int maxTotalConcurrentTasks;
116
117
118
119
120
121
122
123 protected final int maxConcurrentTasksPerRegion;
124
125
126
127
128 protected final int maxConcurrentTasksPerServer;
129 protected final long pause;
130 protected int numTries;
131 protected int serverTrackerTimeout;
132 protected RpcRetryingCallerFactory rpcCallerFactory;
133 private RpcControllerFactory rpcFactory;
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149 interface AsyncProcessCallback<CResult> {
150
151
152
153
154 void success(int originalIndex, byte[] region, Row row, CResult result);
155
156
157
158
159
160
161
162
163
164 boolean failure(int originalIndex, byte[] region, Row row, Throwable t);
165
166
167
168
169
170
171
172 boolean retriableFailure(int originalIndex, Row row, byte[] region, Throwable exception);
173 }
174
175 private static class BatchErrors {
176 private final List<Throwable> throwables = new ArrayList<Throwable>();
177 private final List<Row> actions = new ArrayList<Row>();
178 private final List<String> addresses = new ArrayList<String>();
179
180 public synchronized void add(Throwable ex, Row row, HRegionLocation location) {
181 if (row == null){
182 throw new IllegalArgumentException("row cannot be null. location=" + location);
183 }
184
185 throwables.add(ex);
186 actions.add(row);
187 addresses.add(location != null ? location.getServerName().toString() : "null location");
188 }
189
190 private synchronized RetriesExhaustedWithDetailsException makeException() {
191 return new RetriesExhaustedWithDetailsException(
192 new ArrayList<Throwable>(throwables),
193 new ArrayList<Row>(actions), new ArrayList<String>(addresses));
194 }
195
196 public synchronized void clear() {
197 throwables.clear();
198 actions.clear();
199 addresses.clear();
200 }
201 }
202
203 public AsyncProcess(HConnection hc, TableName tableName, ExecutorService pool,
204 AsyncProcessCallback<CResult> callback, Configuration conf,
205 RpcRetryingCallerFactory rpcCaller, RpcControllerFactory rpcFactory) {
206 if (hc == null){
207 throw new IllegalArgumentException("HConnection cannot be null.");
208 }
209
210 this.hConnection = hc;
211 this.tableName = tableName;
212 this.pool = pool;
213 this.callback = callback;
214
215 this.id = COUNTER.incrementAndGet();
216
217 this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
218 HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
219 this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
220 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
221
222 this.maxTotalConcurrentTasks = conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
223 HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS);
224 this.maxConcurrentTasksPerServer = conf.getInt(HConstants.HBASE_CLIENT_MAX_PERSERVER_TASKS,
225 HConstants.DEFAULT_HBASE_CLIENT_MAX_PERSERVER_TASKS);
226 this.maxConcurrentTasksPerRegion = conf.getInt(HConstants.HBASE_CLIENT_MAX_PERREGION_TASKS,
227 HConstants.DEFAULT_HBASE_CLIENT_MAX_PERREGION_TASKS);
228
229
230
231
232 this.startLogErrorsCnt = conf.getInt("hbase.client.start.log.errors.counter", 9);
233
234 if (this.maxTotalConcurrentTasks <= 0) {
235 throw new IllegalArgumentException("maxTotalConcurrentTasks=" + maxTotalConcurrentTasks);
236 }
237 if (this.maxConcurrentTasksPerServer <= 0) {
238 throw new IllegalArgumentException("maxConcurrentTasksPerServer=" +
239 maxConcurrentTasksPerServer);
240 }
241 if (this.maxConcurrentTasksPerRegion <= 0) {
242 throw new IllegalArgumentException("maxConcurrentTasksPerRegion=" +
243 maxConcurrentTasksPerRegion);
244 }
245
246
247
248
249
250
251
252
253 this.serverTrackerTimeout = 0;
254 for (int i = 0; i < this.numTries; ++i) {
255 serverTrackerTimeout += ConnectionUtils.getPauseTime(this.pause, i);
256 }
257
258 this.rpcCallerFactory = rpcCaller;
259 Preconditions.checkNotNull(rpcFactory);
260 this.rpcFactory = rpcFactory;
261 }
262
263
264
265
266
267
268
269
270 public void submit(List<? extends Row> rows, boolean atLeastOne) throws InterruptedIOException {
271 if (rows.isEmpty()) {
272 return;
273 }
274
275
276
277 Map<HRegionLocation, MultiAction<Row>> actionsByServer =
278 new HashMap<HRegionLocation, MultiAction<Row>>();
279 List<Action<Row>> retainedActions = new ArrayList<Action<Row>>(rows.size());
280
281 long currentTaskCnt = tasksDone.get();
282 boolean alreadyLooped = false;
283
284 NonceGenerator ng = this.hConnection.getNonceGenerator();
285 do {
286 if (alreadyLooped){
287
288 waitForNextTaskDone(currentTaskCnt);
289 currentTaskCnt = tasksDone.get();
290 } else {
291 alreadyLooped = true;
292 }
293
294
295 waitForMaximumCurrentTasks(maxTotalConcurrentTasks - 1);
296
297
298
299 Map<Long, Boolean> regionIncluded = new HashMap<Long, Boolean>();
300 Map<ServerName, Boolean> serverIncluded = new HashMap<ServerName, Boolean>();
301
302 int posInList = -1;
303 Iterator<? extends Row> it = rows.iterator();
304 while (it.hasNext()) {
305 Row r = it.next();
306 HRegionLocation loc = findDestLocation(r, posInList);
307
308 if (loc == null) {
309 it.remove();
310 } else if (canTakeOperation(loc, regionIncluded, serverIncluded)) {
311 Action<Row> action = new Action<Row>(r, ++posInList);
312 setNonce(ng, r, action);
313 retainedActions.add(action);
314 addAction(loc, action, actionsByServer, ng);
315 it.remove();
316 }
317 }
318 } while (retainedActions.isEmpty() && atLeastOne && !hasError());
319
320 HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
321 sendMultiAction(retainedActions, actionsByServer, 1, errorsByServer);
322 }
323
324
325
326
327
328
329
330
331
332 private void addAction(HRegionLocation loc, Action<Row> action, Map<HRegionLocation,
333 MultiAction<Row>> actionsByServer, NonceGenerator ng) {
334 final byte[] regionName = loc.getRegionInfo().getRegionName();
335 MultiAction<Row> multiAction = actionsByServer.get(loc);
336 if (multiAction == null) {
337 multiAction = new MultiAction<Row>();
338 actionsByServer.put(loc, multiAction);
339 }
340 if (action.hasNonce() && !multiAction.hasNonceGroup()) {
341
342
343 multiAction.setNonceGroup(ng.getNonceGroup());
344 }
345
346 multiAction.add(regionName, action);
347 }
348
349
350
351
352
353
354
355
356 private HRegionLocation findDestLocation(Row row, int posInList) {
357 if (row == null) throw new IllegalArgumentException("#" + id + ", row cannot be null");
358 HRegionLocation loc = null;
359 IOException locationException = null;
360 try {
361 loc = hConnection.locateRegion(this.tableName, row.getRow());
362 if (loc == null) {
363 locationException = new IOException("#" + id + ", no location found, aborting submit for" +
364 " tableName=" + tableName +
365 " rowkey=" + Arrays.toString(row.getRow()));
366 }
367 } catch (IOException e) {
368 locationException = e;
369 }
370 if (locationException != null) {
371
372
373 manageError(posInList, row, false, locationException, null);
374 return null;
375 }
376
377 return loc;
378 }
379
380
381
382
383
384
385
386
387
388 protected boolean canTakeOperation(HRegionLocation loc,
389 Map<Long, Boolean> regionsIncluded,
390 Map<ServerName, Boolean> serversIncluded) {
391 long regionId = loc.getRegionInfo().getRegionId();
392 Boolean regionPrevious = regionsIncluded.get(regionId);
393
394 if (regionPrevious != null) {
395
396 return regionPrevious;
397 }
398
399 Boolean serverPrevious = serversIncluded.get(loc.getServerName());
400 if (Boolean.FALSE.equals(serverPrevious)) {
401
402 regionsIncluded.put(regionId, Boolean.FALSE);
403 return false;
404 }
405
406 AtomicInteger regionCnt = taskCounterPerRegion.get(loc.getRegionInfo().getRegionName());
407 if (regionCnt != null && regionCnt.get() >= maxConcurrentTasksPerRegion) {
408
409 regionsIncluded.put(regionId, Boolean.FALSE);
410 return false;
411 }
412
413 if (serverPrevious == null) {
414
415 int newServers = 0;
416 for (Map.Entry<ServerName, Boolean> kv : serversIncluded.entrySet()) {
417 if (kv.getValue()) {
418 newServers++;
419 }
420 }
421
422
423 boolean ok = (newServers + getCurrentTasksCount()) < maxTotalConcurrentTasks;
424
425 if (ok) {
426
427 AtomicInteger serverCnt = taskCounterPerServer.get(loc.getServerName());
428 ok = (serverCnt == null || serverCnt.get() < maxConcurrentTasksPerServer);
429 }
430
431 if (!ok) {
432 regionsIncluded.put(regionId, Boolean.FALSE);
433 serversIncluded.put(loc.getServerName(), Boolean.FALSE);
434 return false;
435 }
436
437 serversIncluded.put(loc.getServerName(), Boolean.TRUE);
438 } else {
439 assert serverPrevious.equals(Boolean.TRUE);
440 }
441
442 regionsIncluded.put(regionId, Boolean.TRUE);
443
444 return true;
445 }
446
447
448
449
450
451
452
453 public void submitAll(List<? extends Row> rows) {
454 List<Action<Row>> actions = new ArrayList<Action<Row>>(rows.size());
455
456
457 int posInList = -1;
458 NonceGenerator ng = this.hConnection.getNonceGenerator();
459 for (Row r : rows) {
460 posInList++;
461 if (r instanceof Put) {
462 Put put = (Put) r;
463 if (put.isEmpty()) {
464 throw new IllegalArgumentException("No columns to insert for #" + (posInList+1)+ " item");
465 }
466 }
467 Action<Row> action = new Action<Row>(r, posInList);
468 setNonce(ng, r, action);
469 actions.add(action);
470 }
471 HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
472 submit(actions, actions, 1, errorsByServer);
473 }
474
475 private void setNonce(NonceGenerator ng, Row r, Action<Row> action) {
476 if (!(r instanceof Append) && !(r instanceof Increment)) return;
477 action.setNonce(ng.newNonce());
478 }
479
480
481
482
483
484
485
486
487
488
489 private void submit(List<Action<Row>> initialActions,
490 List<Action<Row>> currentActions, int numAttempt,
491 final HConnectionManager.ServerErrorTracker errorsByServer) {
492
493 if (numAttempt > 1){
494 retriesCnt.incrementAndGet();
495 }
496
497
498 final Map<HRegionLocation, MultiAction<Row>> actionsByServer =
499 new HashMap<HRegionLocation, MultiAction<Row>>();
500
501 NonceGenerator ng = this.hConnection.getNonceGenerator();
502 for (Action<Row> action : currentActions) {
503 HRegionLocation loc = findDestLocation(action.getAction(), action.getOriginalIndex());
504 if (loc != null) {
505 addAction(loc, action, actionsByServer, ng);
506 }
507 }
508
509 if (!actionsByServer.isEmpty()) {
510 sendMultiAction(initialActions, actionsByServer, numAttempt, errorsByServer);
511 }
512 }
513
514
515
516
517
518
519
520
521
522 public void sendMultiAction(final List<Action<Row>> initialActions,
523 Map<HRegionLocation, MultiAction<Row>> actionsByServer,
524 final int numAttempt,
525 final HConnectionManager.ServerErrorTracker errorsByServer) {
526
527
528 for (Map.Entry<HRegionLocation, MultiAction<Row>> e : actionsByServer.entrySet()) {
529 final HRegionLocation loc = e.getKey();
530 final MultiAction<Row> multiAction = e.getValue();
531 incTaskCounters(multiAction.getRegions(), loc.getServerName());
532 Runnable runnable = Trace.wrap("AsyncProcess.sendMultiAction", new Runnable() {
533 @Override
534 public void run() {
535 MultiResponse res;
536 try {
537 MultiServerCallable<Row> callable = createCallable(loc, multiAction);
538 try {
539 res = createCaller(callable).callWithoutRetries(callable);
540 } catch (IOException e) {
541
542
543 receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, e,
544 errorsByServer);
545 return;
546 } catch (Throwable t) {
547
548 LOG.error("#" + id + ", Caught throwable while calling. This is unexpected." +
549 " Retrying. Server is " + loc.getServerName() + ", tableName=" + tableName, t);
550 receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, t,
551 errorsByServer);
552 return;
553 }
554
555
556 receiveMultiAction(initialActions, multiAction, loc, res, numAttempt, errorsByServer);
557
558 } finally {
559 decTaskCounters(multiAction.getRegions(), loc.getServerName());
560 }
561 }
562 });
563
564 try {
565 this.pool.submit(runnable);
566 } catch (RejectedExecutionException ree) {
567
568
569 decTaskCounters(multiAction.getRegions(), loc.getServerName());
570 LOG.warn("#" + id + ", the task was rejected by the pool. This is unexpected." +
571 " Server is " + loc.getServerName(), ree);
572
573
574 receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, ree, errorsByServer);
575 }
576 }
577 }
578
579
580
581
582 protected MultiServerCallable<Row> createCallable(final HRegionLocation location,
583 final MultiAction<Row> multi) {
584 return new MultiServerCallable<Row>(hConnection, tableName, location, this.rpcFactory, multi);
585 }
586
587
588
589
590
591
592 protected RpcRetryingCaller<MultiResponse> createCaller(MultiServerCallable<Row> callable) {
593 return rpcCallerFactory.<MultiResponse> newCaller();
594 }
595
596
597
598
599
600
601
602
603
604
605
606 private boolean manageError(int originalIndex, Row row, boolean canRetry,
607 Throwable throwable, HRegionLocation location) {
608 if (canRetry && throwable != null && throwable instanceof DoNotRetryIOException) {
609 canRetry = false;
610 }
611
612 byte[] region = null;
613 if (canRetry && callback != null) {
614 region = location == null ? null : location.getRegionInfo().getEncodedNameAsBytes();
615 canRetry = callback.retriableFailure(originalIndex, row, region, throwable);
616 }
617
618 if (!canRetry) {
619 if (callback != null) {
620 if (region == null && location != null) {
621 region = location.getRegionInfo().getEncodedNameAsBytes();
622 }
623 callback.failure(originalIndex, region, row, throwable);
624 }
625 errors.add(throwable, row, location);
626 this.hasError.set(true);
627 }
628
629 return canRetry;
630 }
631
632
633
634
635
636
637
638
639
640
641 private void receiveGlobalFailure(List<Action<Row>> initialActions, MultiAction<Row> rsActions,
642 HRegionLocation location, int numAttempt, Throwable t,
643 HConnectionManager.ServerErrorTracker errorsByServer) {
644
645
646 hConnection.updateCachedLocations(tableName,
647 rsActions.actions.values().iterator().next().get(0).getAction().getRow(), null, location);
648 errorsByServer.reportServerError(location);
649
650 List<Action<Row>> toReplay = new ArrayList<Action<Row>>(initialActions.size());
651 for (Map.Entry<byte[], List<Action<Row>>> e : rsActions.actions.entrySet()) {
652 for (Action<Row> action : e.getValue()) {
653 if (manageError(action.getOriginalIndex(), action.getAction(), true, t, location)) {
654 toReplay.add(action);
655 }
656 }
657 }
658
659 logAndResubmit(initialActions, location, toReplay, numAttempt, rsActions.size(),
660 t, errorsByServer);
661 }
662
663
664
665
666
667 private void logAndResubmit(List<Action<Row>> initialActions, HRegionLocation oldLocation,
668 List<Action<Row>> toReplay, int numAttempt, int failureCount,
669 Throwable throwable,
670 HConnectionManager.ServerErrorTracker errorsByServer) {
671 if (toReplay.isEmpty()) {
672
673 if (failureCount != 0) {
674
675 LOG.warn(createLog(numAttempt, failureCount, toReplay.size(),
676 oldLocation.getServerName(), throwable, -1, false,
677 errorsByServer.getStartTrackingTime()));
678 } else if (numAttempt > startLogErrorsCnt + 1) {
679
680 LOG.info(createLog(numAttempt, failureCount, 0,
681 oldLocation.getServerName(), throwable, -1, false,
682 errorsByServer.getStartTrackingTime()));
683 }
684 return;
685 }
686
687
688
689
690
691
692
693
694 long backOffTime = errorsByServer.calculateBackoffTime(oldLocation, pause);
695
696 if (numAttempt > startLogErrorsCnt) {
697
698
699 LOG.info(createLog(numAttempt, failureCount, toReplay.size(),
700 oldLocation.getServerName(), throwable, backOffTime, true,
701 errorsByServer.getStartTrackingTime()));
702 }
703
704 try {
705 Thread.sleep(backOffTime);
706 } catch (InterruptedException e) {
707 LOG.warn("#" + id + ", not sent: " + toReplay.size() + " operations, " + oldLocation, e);
708 Thread.currentThread().interrupt();
709 return;
710 }
711
712 submit(initialActions, toReplay, numAttempt + 1, errorsByServer);
713 }
714
715
716
717
718
719
720
721
722
723
724 private void receiveMultiAction(List<Action<Row>> initialActions, MultiAction<Row> multiAction,
725 HRegionLocation location,
726 MultiResponse responses, int numAttempt,
727 HConnectionManager.ServerErrorTracker errorsByServer) {
728 assert responses != null;
729
730
731
732
733
734
735
736 List<Action<Row>> toReplay = new ArrayList<Action<Row>>();
737 Throwable throwable = null;
738 int failureCount = 0;
739 boolean canRetry = true;
740
741 for (Map.Entry<byte[], List<Pair<Integer, Object>>> resultsForRS :
742 responses.getResults().entrySet()) {
743
744 boolean regionFailureRegistered = false;
745 for (Pair<Integer, Object> regionResult : resultsForRS.getValue()) {
746 Object result = regionResult.getSecond();
747
748
749 if (result == null || result instanceof Throwable) {
750 throwable = (Throwable) result;
751 Action<Row> correspondingAction = initialActions.get(regionResult.getFirst());
752 Row row = correspondingAction.getAction();
753 failureCount++;
754 if (!regionFailureRegistered) {
755 regionFailureRegistered= true;
756
757 hConnection.updateCachedLocations(this.tableName, row.getRow(), result, location);
758 if (failureCount == 1) {
759 errorsByServer.reportServerError(location);
760 canRetry = errorsByServer.canRetryMore(numAttempt);
761 }
762 }
763
764 if (manageError(correspondingAction.getOriginalIndex(), row, canRetry,
765 throwable, location)) {
766 toReplay.add(correspondingAction);
767 }
768 } else {
769 if (callback != null) {
770 int index = regionResult.getFirst();
771 Action<Row> correspondingAction = initialActions.get(index);
772 Row row = correspondingAction.getAction();
773
774 this.callback.success(index, resultsForRS.getKey(), row, (CResult) result);
775 }
776 }
777 }
778 }
779
780
781
782
783 for (Map.Entry<byte[], Throwable> throwableEntry : responses.getExceptions().entrySet()) {
784 throwable = throwableEntry.getValue();
785 byte[] region =throwableEntry.getKey();
786 List<Action<Row>> actions = multiAction.actions.get(region);
787 if (actions == null || actions.isEmpty()) {
788 throw new IllegalStateException("Wrong response for the region: " +
789 HRegionInfo.encodeRegionName(region));
790 }
791
792 if (failureCount == 0) {
793 errorsByServer.reportServerError(location);
794 canRetry = errorsByServer.canRetryMore(numAttempt);
795 }
796 hConnection.updateCachedLocations(this.tableName, actions.get(0).getAction().getRow(),
797 throwable, location);
798 failureCount += actions.size();
799
800 for (Action<Row> action : actions) {
801 Row row = action.getAction();
802 if (manageError(action.getOriginalIndex(), row, canRetry, throwable, location)) {
803 toReplay.add(action);
804 }
805 }
806 }
807
808 logAndResubmit(initialActions, location, toReplay, numAttempt, failureCount,
809 throwable, errorsByServer);
810 }
811
812 private String createLog(int numAttempt, int failureCount, int replaySize, ServerName sn,
813 Throwable error, long backOffTime, boolean willRetry, String startTime){
814 StringBuilder sb = new StringBuilder();
815
816 sb.append("#").append(id).append(", table=").append(tableName).
817 append(", attempt=").append(numAttempt).append("/").append(numTries).append(" ");
818
819 if (failureCount > 0 || error != null){
820 sb.append("failed ").append(failureCount).append(" ops").append(", last exception: ").
821 append(error == null ? "null" : error);
822 } else {
823 sb.append("SUCCEEDED");
824 }
825
826 sb.append(" on ").append(sn);
827
828 sb.append(", tracking started ").append(startTime);
829
830 if (willRetry) {
831 sb.append(", retrying after ").append(backOffTime).append(" ms").
832 append(", replay ").append(replaySize).append(" ops.");
833 } else if (failureCount > 0) {
834 sb.append(" - FAILED, NOT RETRYING ANYMORE");
835 }
836
837 return sb.toString();
838 }
839
840
841
842
843
844 protected void waitForNextTaskDone(long currentNumberOfTask) throws InterruptedIOException {
845 while (currentNumberOfTask == tasksDone.get()) {
846 try {
847 synchronized (this.tasksDone) {
848 this.tasksDone.wait(100);
849 }
850 } catch (InterruptedException e) {
851 throw new InterruptedIOException("#" + id + ", interrupted." +
852 " currentNumberOfTask=" + currentNumberOfTask +
853 ", tableName=" + tableName + ", tasksDone=" + tasksDone.get());
854 }
855 }
856 }
857
858
859
860
861 private void waitForMaximumCurrentTasks(int max) throws InterruptedIOException {
862 long lastLog = EnvironmentEdgeManager.currentTimeMillis();
863 long currentTasksDone = this.tasksDone.get();
864
865 while ((tasksSent.get() - currentTasksDone) > max) {
866 long now = EnvironmentEdgeManager.currentTimeMillis();
867 if (now > lastLog + 10000) {
868 lastLog = now;
869 LOG.info("#" + id + ", waiting for some tasks to finish. Expected max="
870 + max + ", tasksSent=" + tasksSent.get() + ", tasksDone=" + tasksDone.get() +
871 ", currentTasksDone=" + currentTasksDone + ", retries=" + retriesCnt.get() +
872 " hasError=" + hasError() + ", tableName=" + tableName);
873 }
874 waitForNextTaskDone(currentTasksDone);
875 currentTasksDone = this.tasksDone.get();
876 }
877 }
878
879 private long getCurrentTasksCount(){
880 return tasksSent.get() - tasksDone.get();
881 }
882
883
884
885
886 public void waitUntilDone() throws InterruptedIOException {
887 waitForMaximumCurrentTasks(0);
888 }
889
890
891 public boolean hasError() {
892 return hasError.get();
893 }
894
895 public List<? extends Row> getFailedOperations() {
896 return errors.actions;
897 }
898
899
900
901
902 public void clearErrors() {
903 errors.clear();
904 hasError.set(false);
905 }
906
907 public RetriesExhaustedWithDetailsException getErrors() {
908 return errors.makeException();
909 }
910
911
912
913
914 protected void incTaskCounters(Collection<byte[]> regions, ServerName sn) {
915 tasksSent.incrementAndGet();
916
917 AtomicInteger serverCnt = taskCounterPerServer.get(sn);
918 if (serverCnt == null) {
919 taskCounterPerServer.putIfAbsent(sn, new AtomicInteger());
920 serverCnt = taskCounterPerServer.get(sn);
921 }
922 serverCnt.incrementAndGet();
923
924 for (byte[] regBytes : regions) {
925 AtomicInteger regionCnt = taskCounterPerRegion.get(regBytes);
926 if (regionCnt == null) {
927 regionCnt = new AtomicInteger();
928 AtomicInteger oldCnt = taskCounterPerRegion.putIfAbsent(regBytes, regionCnt);
929 if (oldCnt != null) {
930 regionCnt = oldCnt;
931 }
932 }
933 regionCnt.incrementAndGet();
934 }
935 }
936
937
938
939
940 protected void decTaskCounters(Collection<byte[]> regions, ServerName sn) {
941 for (byte[] regBytes : regions) {
942 AtomicInteger regionCnt = taskCounterPerRegion.get(regBytes);
943 regionCnt.decrementAndGet();
944 }
945
946 taskCounterPerServer.get(sn).decrementAndGet();
947
948 tasksDone.incrementAndGet();
949 synchronized (tasksDone) {
950 tasksDone.notifyAll();
951 }
952 }
953
954
955
956
957
958
959
960
961 protected HConnectionManager.ServerErrorTracker createServerErrorTracker() {
962 return new HConnectionManager.ServerErrorTracker(this.serverTrackerTimeout, this.numTries);
963 }
964 }