1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import java.io.IOException;
23 import java.io.InterruptedIOException;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.Collection;
27 import java.util.HashMap;
28 import java.util.Iterator;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.concurrent.ConcurrentHashMap;
32 import java.util.concurrent.ConcurrentMap;
33 import java.util.concurrent.ConcurrentSkipListMap;
34 import java.util.concurrent.ExecutorService;
35 import java.util.concurrent.RejectedExecutionException;
36 import java.util.concurrent.atomic.AtomicBoolean;
37 import java.util.concurrent.atomic.AtomicInteger;
38 import java.util.concurrent.atomic.AtomicLong;
39
40 import org.apache.commons.logging.Log;
41 import org.apache.commons.logging.LogFactory;
42 import org.apache.hadoop.conf.Configuration;
43 import org.apache.hadoop.hbase.DoNotRetryIOException;
44 import org.apache.hadoop.hbase.HConstants;
45 import org.apache.hadoop.hbase.HRegionInfo;
46 import org.apache.hadoop.hbase.HRegionLocation;
47 import org.apache.hadoop.hbase.ServerName;
48 import org.apache.hadoop.hbase.TableName;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
51 import org.apache.hadoop.hbase.util.Pair;
52 import org.cloudera.htrace.Trace;
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90 class AsyncProcess<CResult> {
91 private static final Log LOG = LogFactory.getLog(AsyncProcess.class);
92 protected static final AtomicLong COUNTER = new AtomicLong();
93 protected final long id;
94 private final int startLogErrorsCnt;
95 protected final HConnection hConnection;
96 protected final TableName tableName;
97 protected final ExecutorService pool;
98 protected final AsyncProcessCallback<CResult> callback;
99 protected final BatchErrors errors = new BatchErrors();
100 protected final AtomicBoolean hasError = new AtomicBoolean(false);
101 protected final AtomicLong tasksSent = new AtomicLong(0);
102 protected final AtomicLong tasksDone = new AtomicLong(0);
103 protected final AtomicLong retriesCnt = new AtomicLong(0);
104 protected final ConcurrentMap<byte[], AtomicInteger> taskCounterPerRegion =
105 new ConcurrentSkipListMap<byte[], AtomicInteger>(Bytes.BYTES_COMPARATOR);
106 protected final ConcurrentMap<ServerName, AtomicInteger> taskCounterPerServer =
107 new ConcurrentHashMap<ServerName, AtomicInteger>();
108
109
110
111
112 protected final int maxTotalConcurrentTasks;
113
114
115
116
117
118
119
120 protected final int maxConcurrentTasksPerRegion;
121
122
123
124
125 protected final int maxConcurrentTasksPerServer;
126 protected final long pause;
127 protected int numTries;
128 protected int serverTrackerTimeout;
129 protected RpcRetryingCallerFactory rpcCallerFactory;
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145 interface AsyncProcessCallback<CResult> {
146
147
148
149
150 void success(int originalIndex, byte[] region, Row row, CResult result);
151
152
153
154
155
156
157
158
159
160 boolean failure(int originalIndex, byte[] region, Row row, Throwable t);
161
162
163
164
165
166
167
168 boolean retriableFailure(int originalIndex, Row row, byte[] region, Throwable exception);
169 }
170
171 private static class BatchErrors {
172 private final List<Throwable> throwables = new ArrayList<Throwable>();
173 private final List<Row> actions = new ArrayList<Row>();
174 private final List<String> addresses = new ArrayList<String>();
175
176 public synchronized void add(Throwable ex, Row row, HRegionLocation location) {
177 if (row == null){
178 throw new IllegalArgumentException("row cannot be null. location=" + location);
179 }
180
181 throwables.add(ex);
182 actions.add(row);
183 addresses.add(location != null ? location.getServerName().toString() : "null location");
184 }
185
186 private synchronized RetriesExhaustedWithDetailsException makeException() {
187 return new RetriesExhaustedWithDetailsException(
188 new ArrayList<Throwable>(throwables),
189 new ArrayList<Row>(actions), new ArrayList<String>(addresses));
190 }
191
192 public synchronized void clear() {
193 throwables.clear();
194 actions.clear();
195 addresses.clear();
196 }
197 }
198
199 public AsyncProcess(HConnection hc, TableName tableName, ExecutorService pool,
200 AsyncProcessCallback<CResult> callback, Configuration conf,
201 RpcRetryingCallerFactory rpcCaller) {
202 if (hc == null){
203 throw new IllegalArgumentException("HConnection cannot be null.");
204 }
205
206 this.hConnection = hc;
207 this.tableName = tableName;
208 this.pool = pool;
209 this.callback = callback;
210
211 this.id = COUNTER.incrementAndGet();
212
213 this.pause = conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
214 HConstants.DEFAULT_HBASE_CLIENT_PAUSE);
215 this.numTries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
216 HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
217
218 this.maxTotalConcurrentTasks = conf.getInt(HConstants.HBASE_CLIENT_MAX_TOTAL_TASKS,
219 HConstants.DEFAULT_HBASE_CLIENT_MAX_TOTAL_TASKS);
220 this.maxConcurrentTasksPerServer = conf.getInt(HConstants.HBASE_CLIENT_MAX_PERSERVER_TASKS,
221 HConstants.DEFAULT_HBASE_CLIENT_MAX_PERSERVER_TASKS);
222 this.maxConcurrentTasksPerRegion = conf.getInt(HConstants.HBASE_CLIENT_MAX_PERREGION_TASKS,
223 HConstants.DEFAULT_HBASE_CLIENT_MAX_PERREGION_TASKS);
224
225
226
227
228 this.startLogErrorsCnt = conf.getInt("hbase.client.start.log.errors.counter", 9);
229
230 if (this.maxTotalConcurrentTasks <= 0) {
231 throw new IllegalArgumentException("maxTotalConcurrentTasks=" + maxTotalConcurrentTasks);
232 }
233 if (this.maxConcurrentTasksPerServer <= 0) {
234 throw new IllegalArgumentException("maxConcurrentTasksPerServer=" +
235 maxConcurrentTasksPerServer);
236 }
237 if (this.maxConcurrentTasksPerRegion <= 0) {
238 throw new IllegalArgumentException("maxConcurrentTasksPerRegion=" +
239 maxConcurrentTasksPerRegion);
240 }
241
242
243
244
245
246
247
248
249 this.serverTrackerTimeout = 0;
250 for (int i = 0; i < this.numTries; ++i) {
251 serverTrackerTimeout += ConnectionUtils.getPauseTime(this.pause, i);
252 }
253
254
255 this.rpcCallerFactory = rpcCaller;
256 }
257
258
259
260
261
262
263
264
265 public void submit(List<? extends Row> rows, boolean atLeastOne) throws InterruptedIOException {
266 if (rows.isEmpty()) {
267 return;
268 }
269
270
271
272 Map<HRegionLocation, MultiAction<Row>> actionsByServer =
273 new HashMap<HRegionLocation, MultiAction<Row>>();
274 List<Action<Row>> retainedActions = new ArrayList<Action<Row>>(rows.size());
275
276 long currentTaskCnt = tasksDone.get();
277 boolean alreadyLooped = false;
278
279 NonceGenerator ng = this.hConnection.getNonceGenerator();
280 do {
281 if (alreadyLooped){
282
283 waitForNextTaskDone(currentTaskCnt);
284 currentTaskCnt = tasksDone.get();
285 } else {
286 alreadyLooped = true;
287 }
288
289
290 waitForMaximumCurrentTasks(maxTotalConcurrentTasks - 1);
291
292
293
294 Map<Long, Boolean> regionIncluded = new HashMap<Long, Boolean>();
295 Map<ServerName, Boolean> serverIncluded = new HashMap<ServerName, Boolean>();
296
297 int posInList = -1;
298 Iterator<? extends Row> it = rows.iterator();
299 while (it.hasNext()) {
300 Row r = it.next();
301 HRegionLocation loc = findDestLocation(r, posInList);
302
303 if (loc == null) {
304 it.remove();
305 } else if (canTakeOperation(loc, regionIncluded, serverIncluded)) {
306 Action<Row> action = new Action<Row>(r, ++posInList);
307 setNonce(ng, r, action);
308 retainedActions.add(action);
309 addAction(loc, action, actionsByServer, ng);
310 it.remove();
311 }
312 }
313 } while (retainedActions.isEmpty() && atLeastOne && !hasError());
314
315 HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
316 sendMultiAction(retainedActions, actionsByServer, 1, errorsByServer);
317 }
318
319
320
321
322
323
324
325
326
327 private void addAction(HRegionLocation loc, Action<Row> action, Map<HRegionLocation,
328 MultiAction<Row>> actionsByServer, NonceGenerator ng) {
329 final byte[] regionName = loc.getRegionInfo().getRegionName();
330 MultiAction<Row> multiAction = actionsByServer.get(loc);
331 if (multiAction == null) {
332 multiAction = new MultiAction<Row>();
333 actionsByServer.put(loc, multiAction);
334 }
335 if (action.hasNonce() && !multiAction.hasNonceGroup()) {
336
337
338 multiAction.setNonceGroup(ng.getNonceGroup());
339 }
340
341 multiAction.add(regionName, action);
342 }
343
344
345
346
347
348
349
350
351 private HRegionLocation findDestLocation(Row row, int posInList) {
352 if (row == null) throw new IllegalArgumentException("#" + id + ", row cannot be null");
353 HRegionLocation loc = null;
354 IOException locationException = null;
355 try {
356 loc = hConnection.locateRegion(this.tableName, row.getRow());
357 if (loc == null) {
358 locationException = new IOException("#" + id + ", no location found, aborting submit for" +
359 " tableName=" + tableName +
360 " rowkey=" + Arrays.toString(row.getRow()));
361 }
362 } catch (IOException e) {
363 locationException = e;
364 }
365 if (locationException != null) {
366
367
368 manageError(posInList, row, false, locationException, null);
369 return null;
370 }
371
372 return loc;
373 }
374
375
376
377
378
379
380
381
382
383 protected boolean canTakeOperation(HRegionLocation loc,
384 Map<Long, Boolean> regionsIncluded,
385 Map<ServerName, Boolean> serversIncluded) {
386 long regionId = loc.getRegionInfo().getRegionId();
387 Boolean regionPrevious = regionsIncluded.get(regionId);
388
389 if (regionPrevious != null) {
390
391 return regionPrevious;
392 }
393
394 Boolean serverPrevious = serversIncluded.get(loc.getServerName());
395 if (Boolean.FALSE.equals(serverPrevious)) {
396
397 regionsIncluded.put(regionId, Boolean.FALSE);
398 return false;
399 }
400
401 AtomicInteger regionCnt = taskCounterPerRegion.get(loc.getRegionInfo().getRegionName());
402 if (regionCnt != null && regionCnt.get() >= maxConcurrentTasksPerRegion) {
403
404 regionsIncluded.put(regionId, Boolean.FALSE);
405 return false;
406 }
407
408 if (serverPrevious == null) {
409
410 int newServers = 0;
411 for (Map.Entry<ServerName, Boolean> kv : serversIncluded.entrySet()) {
412 if (kv.getValue()) {
413 newServers++;
414 }
415 }
416
417
418 boolean ok = (newServers + getCurrentTasksCount()) < maxTotalConcurrentTasks;
419
420 if (ok) {
421
422 AtomicInteger serverCnt = taskCounterPerServer.get(loc.getServerName());
423 ok = (serverCnt == null || serverCnt.get() < maxConcurrentTasksPerServer);
424 }
425
426 if (!ok) {
427 regionsIncluded.put(regionId, Boolean.FALSE);
428 serversIncluded.put(loc.getServerName(), Boolean.FALSE);
429 return false;
430 }
431
432 serversIncluded.put(loc.getServerName(), Boolean.TRUE);
433 } else {
434 assert serverPrevious.equals(Boolean.TRUE);
435 }
436
437 regionsIncluded.put(regionId, Boolean.TRUE);
438
439 return true;
440 }
441
442
443
444
445
446
447
448 public void submitAll(List<? extends Row> rows) {
449 List<Action<Row>> actions = new ArrayList<Action<Row>>(rows.size());
450
451
452 int posInList = -1;
453 NonceGenerator ng = this.hConnection.getNonceGenerator();
454 for (Row r : rows) {
455 posInList++;
456 if (r instanceof Put) {
457 Put put = (Put) r;
458 if (put.isEmpty()) {
459 throw new IllegalArgumentException("No columns to insert for #" + (posInList+1)+ " item");
460 }
461 }
462 Action<Row> action = new Action<Row>(r, posInList);
463 setNonce(ng, r, action);
464 actions.add(action);
465 }
466 HConnectionManager.ServerErrorTracker errorsByServer = createServerErrorTracker();
467 submit(actions, actions, 1, errorsByServer);
468 }
469
470 private void setNonce(NonceGenerator ng, Row r, Action<Row> action) {
471 if (!(r instanceof Append) && !(r instanceof Increment)) return;
472 action.setNonce(ng.newNonce());
473 }
474
475
476
477
478
479
480
481
482
483
484 private void submit(List<Action<Row>> initialActions,
485 List<Action<Row>> currentActions, int numAttempt,
486 final HConnectionManager.ServerErrorTracker errorsByServer) {
487
488 if (numAttempt > 1){
489 retriesCnt.incrementAndGet();
490 }
491
492
493 final Map<HRegionLocation, MultiAction<Row>> actionsByServer =
494 new HashMap<HRegionLocation, MultiAction<Row>>();
495
496 NonceGenerator ng = this.hConnection.getNonceGenerator();
497 for (Action<Row> action : currentActions) {
498 HRegionLocation loc = findDestLocation(action.getAction(), action.getOriginalIndex());
499 if (loc != null) {
500 addAction(loc, action, actionsByServer, ng);
501 }
502 }
503
504 if (!actionsByServer.isEmpty()) {
505 sendMultiAction(initialActions, actionsByServer, numAttempt, errorsByServer);
506 }
507 }
508
509
510
511
512
513
514
515
516
517 public void sendMultiAction(final List<Action<Row>> initialActions,
518 Map<HRegionLocation, MultiAction<Row>> actionsByServer,
519 final int numAttempt,
520 final HConnectionManager.ServerErrorTracker errorsByServer) {
521
522
523 for (Map.Entry<HRegionLocation, MultiAction<Row>> e : actionsByServer.entrySet()) {
524 final HRegionLocation loc = e.getKey();
525 final MultiAction<Row> multiAction = e.getValue();
526 incTaskCounters(multiAction.getRegions(), loc.getServerName());
527 Runnable runnable = Trace.wrap("AsyncProcess.sendMultiAction", new Runnable() {
528 @Override
529 public void run() {
530 MultiResponse res;
531 try {
532 MultiServerCallable<Row> callable = createCallable(loc, multiAction);
533 try {
534 res = createCaller(callable).callWithoutRetries(callable);
535 } catch (IOException e) {
536
537
538 receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, e,
539 errorsByServer);
540 return;
541 } catch (Throwable t) {
542
543 LOG.error("#" + id + ", Caught throwable while calling. This is unexpected." +
544 " Retrying. Server is " + loc.getServerName() + ", tableName=" + tableName, t);
545 receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, t,
546 errorsByServer);
547 return;
548 }
549
550
551 receiveMultiAction(initialActions, multiAction, loc, res, numAttempt, errorsByServer);
552
553 } finally {
554 decTaskCounters(multiAction.getRegions(), loc.getServerName());
555 }
556 }
557 });
558
559 try {
560 this.pool.submit(runnable);
561 } catch (RejectedExecutionException ree) {
562
563
564 decTaskCounters(multiAction.getRegions(), loc.getServerName());
565 LOG.warn("#" + id + ", the task was rejected by the pool. This is unexpected." +
566 " Server is " + loc.getServerName(), ree);
567
568
569 receiveGlobalFailure(initialActions, multiAction, loc, numAttempt, ree, errorsByServer);
570 }
571 }
572 }
573
574
575
576
577 protected MultiServerCallable<Row> createCallable(final HRegionLocation location,
578 final MultiAction<Row> multi) {
579 return new MultiServerCallable<Row>(hConnection, tableName, location, multi);
580 }
581
582
583
584
585
586
587 protected RpcRetryingCaller<MultiResponse> createCaller(MultiServerCallable<Row> callable) {
588 return rpcCallerFactory.<MultiResponse> newCaller();
589 }
590
591
592
593
594
595
596
597
598
599
600
601 private boolean manageError(int originalIndex, Row row, boolean canRetry,
602 Throwable throwable, HRegionLocation location) {
603 if (canRetry && throwable != null && throwable instanceof DoNotRetryIOException) {
604 canRetry = false;
605 }
606
607 byte[] region = null;
608 if (canRetry && callback != null) {
609 region = location == null ? null : location.getRegionInfo().getEncodedNameAsBytes();
610 canRetry = callback.retriableFailure(originalIndex, row, region, throwable);
611 }
612
613 if (!canRetry) {
614 if (callback != null) {
615 if (region == null && location != null) {
616 region = location.getRegionInfo().getEncodedNameAsBytes();
617 }
618 callback.failure(originalIndex, region, row, throwable);
619 }
620 errors.add(throwable, row, location);
621 this.hasError.set(true);
622 }
623
624 return canRetry;
625 }
626
627
628
629
630
631
632
633
634
635
636 private void receiveGlobalFailure(List<Action<Row>> initialActions, MultiAction<Row> rsActions,
637 HRegionLocation location, int numAttempt, Throwable t,
638 HConnectionManager.ServerErrorTracker errorsByServer) {
639
640
641 hConnection.updateCachedLocations(tableName,
642 rsActions.actions.values().iterator().next().get(0).getAction().getRow(), null, location);
643 errorsByServer.reportServerError(location);
644
645 List<Action<Row>> toReplay = new ArrayList<Action<Row>>(initialActions.size());
646 for (Map.Entry<byte[], List<Action<Row>>> e : rsActions.actions.entrySet()) {
647 for (Action<Row> action : e.getValue()) {
648 if (manageError(action.getOriginalIndex(), action.getAction(), true, t, location)) {
649 toReplay.add(action);
650 }
651 }
652 }
653
654 logAndResubmit(initialActions, location, toReplay, numAttempt, rsActions.size(),
655 t, errorsByServer);
656 }
657
658
659
660
661
662 private void logAndResubmit(List<Action<Row>> initialActions, HRegionLocation oldLocation,
663 List<Action<Row>> toReplay, int numAttempt, int failureCount,
664 Throwable throwable,
665 HConnectionManager.ServerErrorTracker errorsByServer) {
666 if (toReplay.isEmpty()) {
667
668 if (failureCount != 0) {
669
670 LOG.warn(createLog(numAttempt, failureCount, toReplay.size(),
671 oldLocation.getServerName(), throwable, -1, false,
672 errorsByServer.getStartTrackingTime()));
673 } else if (numAttempt > startLogErrorsCnt + 1) {
674
675 LOG.info(createLog(numAttempt, failureCount, 0,
676 oldLocation.getServerName(), throwable, -1, false,
677 errorsByServer.getStartTrackingTime()));
678 }
679 return;
680 }
681
682
683
684
685
686
687
688
689 long backOffTime = errorsByServer.calculateBackoffTime(oldLocation, pause);
690
691 if (numAttempt > startLogErrorsCnt) {
692
693
694 LOG.info(createLog(numAttempt, failureCount, toReplay.size(),
695 oldLocation.getServerName(), throwable, backOffTime, true,
696 errorsByServer.getStartTrackingTime()));
697 }
698
699 try {
700 Thread.sleep(backOffTime);
701 } catch (InterruptedException e) {
702 LOG.warn("#" + id + ", not sent: " + toReplay.size() + " operations, " + oldLocation, e);
703 Thread.currentThread().interrupt();
704 return;
705 }
706
707 submit(initialActions, toReplay, numAttempt + 1, errorsByServer);
708 }
709
710
711
712
713
714
715
716
717
718
719 private void receiveMultiAction(List<Action<Row>> initialActions, MultiAction<Row> multiAction,
720 HRegionLocation location,
721 MultiResponse responses, int numAttempt,
722 HConnectionManager.ServerErrorTracker errorsByServer) {
723 assert responses != null;
724
725
726
727
728
729
730
731 List<Action<Row>> toReplay = new ArrayList<Action<Row>>();
732 Throwable throwable = null;
733 int failureCount = 0;
734 boolean canRetry = true;
735
736 for (Map.Entry<byte[], List<Pair<Integer, Object>>> resultsForRS :
737 responses.getResults().entrySet()) {
738
739 boolean regionFailureRegistered = false;
740 for (Pair<Integer, Object> regionResult : resultsForRS.getValue()) {
741 Object result = regionResult.getSecond();
742
743
744 if (result == null || result instanceof Throwable) {
745 throwable = (Throwable) result;
746 Action<Row> correspondingAction = initialActions.get(regionResult.getFirst());
747 Row row = correspondingAction.getAction();
748 failureCount++;
749 if (!regionFailureRegistered) {
750 regionFailureRegistered= true;
751
752 hConnection.updateCachedLocations(this.tableName, row.getRow(), result, location);
753 if (failureCount == 1) {
754 errorsByServer.reportServerError(location);
755 canRetry = errorsByServer.canRetryMore(numAttempt);
756 }
757 }
758
759 if (manageError(correspondingAction.getOriginalIndex(), row, canRetry,
760 throwable, location)) {
761 toReplay.add(correspondingAction);
762 }
763 } else {
764 if (callback != null) {
765 int index = regionResult.getFirst();
766 Action<Row> correspondingAction = initialActions.get(index);
767 Row row = correspondingAction.getAction();
768
769 this.callback.success(index, resultsForRS.getKey(), row, (CResult) result);
770 }
771 }
772 }
773 }
774
775
776
777
778 for (Map.Entry<byte[], Throwable> throwableEntry : responses.getExceptions().entrySet()) {
779 throwable = throwableEntry.getValue();
780 byte[] region =throwableEntry.getKey();
781 List<Action<Row>> actions = multiAction.actions.get(region);
782 if (actions == null || actions.isEmpty()) {
783 throw new IllegalStateException("Wrong response for the region: " +
784 HRegionInfo.encodeRegionName(region));
785 }
786
787 if (failureCount == 0) {
788 errorsByServer.reportServerError(location);
789 canRetry = errorsByServer.canRetryMore(numAttempt);
790 }
791 hConnection.updateCachedLocations(this.tableName, actions.get(0).getAction().getRow(),
792 throwable, location);
793 failureCount += actions.size();
794
795 for (Action<Row> action : actions) {
796 Row row = action.getAction();
797 if (manageError(action.getOriginalIndex(), row, canRetry, throwable, location)) {
798 toReplay.add(action);
799 }
800 }
801 }
802
803 logAndResubmit(initialActions, location, toReplay, numAttempt, failureCount,
804 throwable, errorsByServer);
805 }
806
807 private String createLog(int numAttempt, int failureCount, int replaySize, ServerName sn,
808 Throwable error, long backOffTime, boolean willRetry, String startTime){
809 StringBuilder sb = new StringBuilder();
810
811 sb.append("#").append(id).append(", table=").append(tableName).
812 append(", attempt=").append(numAttempt).append("/").append(numTries).append(" ");
813
814 if (failureCount > 0 || error != null){
815 sb.append("failed ").append(failureCount).append(" ops").append(", last exception: ").
816 append(error == null ? "null" : error);
817 } else {
818 sb.append("SUCCEEDED");
819 }
820
821 sb.append(" on ").append(sn);
822
823 sb.append(", tracking started ").append(startTime);
824
825 if (willRetry) {
826 sb.append(", retrying after ").append(backOffTime).append(" ms").
827 append(", replay ").append(replaySize).append(" ops.");
828 } else if (failureCount > 0) {
829 sb.append(" - FAILED, NOT RETRYING ANYMORE");
830 }
831
832 return sb.toString();
833 }
834
835
836
837
838
839 protected void waitForNextTaskDone(long currentNumberOfTask) throws InterruptedIOException {
840 while (currentNumberOfTask == tasksDone.get()) {
841 try {
842 synchronized (this.tasksDone) {
843 this.tasksDone.wait(100);
844 }
845 } catch (InterruptedException e) {
846 throw new InterruptedIOException("#" + id + ", interrupted." +
847 " currentNumberOfTask=" + currentNumberOfTask +
848 ", tableName=" + tableName + ", tasksDone=" + tasksDone.get());
849 }
850 }
851 }
852
853
854
855
856 private void waitForMaximumCurrentTasks(int max) throws InterruptedIOException {
857 long lastLog = EnvironmentEdgeManager.currentTimeMillis();
858 long currentTasksDone = this.tasksDone.get();
859
860 while ((tasksSent.get() - currentTasksDone) > max) {
861 long now = EnvironmentEdgeManager.currentTimeMillis();
862 if (now > lastLog + 10000) {
863 lastLog = now;
864 LOG.info("#" + id + ", waiting for some tasks to finish. Expected max="
865 + max + ", tasksSent=" + tasksSent.get() + ", tasksDone=" + tasksDone.get() +
866 ", currentTasksDone=" + currentTasksDone + ", retries=" + retriesCnt.get() +
867 " hasError=" + hasError() + ", tableName=" + tableName);
868 }
869 waitForNextTaskDone(currentTasksDone);
870 currentTasksDone = this.tasksDone.get();
871 }
872 }
873
874 private long getCurrentTasksCount(){
875 return tasksSent.get() - tasksDone.get();
876 }
877
878
879
880
881 public void waitUntilDone() throws InterruptedIOException {
882 waitForMaximumCurrentTasks(0);
883 }
884
885
886 public boolean hasError() {
887 return hasError.get();
888 }
889
890 public List<? extends Row> getFailedOperations() {
891 return errors.actions;
892 }
893
894
895
896
897 public void clearErrors() {
898 errors.clear();
899 hasError.set(false);
900 }
901
902 public RetriesExhaustedWithDetailsException getErrors() {
903 return errors.makeException();
904 }
905
906
907
908
909 protected void incTaskCounters(Collection<byte[]> regions, ServerName sn) {
910 tasksSent.incrementAndGet();
911
912 AtomicInteger serverCnt = taskCounterPerServer.get(sn);
913 if (serverCnt == null) {
914 taskCounterPerServer.putIfAbsent(sn, new AtomicInteger());
915 serverCnt = taskCounterPerServer.get(sn);
916 }
917 serverCnt.incrementAndGet();
918
919 for (byte[] regBytes : regions) {
920 AtomicInteger regionCnt = taskCounterPerRegion.get(regBytes);
921 if (regionCnt == null) {
922 regionCnt = new AtomicInteger();
923 AtomicInteger oldCnt = taskCounterPerRegion.putIfAbsent(regBytes, regionCnt);
924 if (oldCnt != null) {
925 regionCnt = oldCnt;
926 }
927 }
928 regionCnt.incrementAndGet();
929 }
930 }
931
932
933
934
935 protected void decTaskCounters(Collection<byte[]> regions, ServerName sn) {
936 for (byte[] regBytes : regions) {
937 AtomicInteger regionCnt = taskCounterPerRegion.get(regBytes);
938 regionCnt.decrementAndGet();
939 }
940
941 taskCounterPerServer.get(sn).decrementAndGet();
942
943 tasksDone.incrementAndGet();
944 synchronized (tasksDone) {
945 tasksDone.notifyAll();
946 }
947 }
948
949
950
951
952
953
954
955
956 protected HConnectionManager.ServerErrorTracker createServerErrorTracker() {
957 return new HConnectionManager.ServerErrorTracker(this.serverTrackerTimeout, this.numTries);
958 }
959 }