001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 package org.apache.camel.processor; 018 019 import java.util.Collection; 020 import java.util.Iterator; 021 import java.util.LinkedList; 022 import java.util.Queue; 023 import java.util.concurrent.TimeUnit; 024 import java.util.concurrent.locks.Condition; 025 import java.util.concurrent.locks.Lock; 026 import java.util.concurrent.locks.ReentrantLock; 027 028 import org.apache.camel.Exchange; 029 import org.apache.camel.Processor; 030 import org.apache.camel.impl.LoggingExceptionHandler; 031 import org.apache.camel.impl.ServiceSupport; 032 import org.apache.camel.spi.ExceptionHandler; 033 import org.apache.camel.util.ServiceHelper; 034 035 /** 036 * A base class for any kind of {@link Processor} which implements some kind of batch processing. 037 * 038 * @version $Revision: 765825 $ 039 */ 040 public class BatchProcessor extends ServiceSupport implements Processor { 041 042 public static final long DEFAULT_BATCH_TIMEOUT = 1000L; 043 public static final int DEFAULT_BATCH_SIZE = 100; 044 045 private long batchTimeout = DEFAULT_BATCH_TIMEOUT; 046 private int batchSize = DEFAULT_BATCH_SIZE; 047 private int outBatchSize; 048 049 private Processor processor; 050 private Collection<Exchange> collection; 051 private ExceptionHandler exceptionHandler; 052 053 private BatchSender sender; 054 055 public BatchProcessor(Processor processor, Collection<Exchange> collection) { 056 this.processor = processor; 057 this.collection = collection; 058 this.sender = new BatchSender(); 059 } 060 061 @Override 062 public String toString() { 063 return "BatchProcessor[to: " + processor + "]"; 064 } 065 066 // Properties 067 // ------------------------------------------------------------------------- 068 public ExceptionHandler getExceptionHandler() { 069 if (exceptionHandler == null) { 070 exceptionHandler = new LoggingExceptionHandler(getClass()); 071 } 072 return exceptionHandler; 073 } 074 075 public void setExceptionHandler(ExceptionHandler exceptionHandler) { 076 this.exceptionHandler = exceptionHandler; 077 } 078 079 public int getBatchSize() { 080 return batchSize; 081 } 082 083 /** 084 * Sets the <b>in</b> batch size. This is the number of incoming exchanges that this batch processor will 085 * process before its completed. The default value is {@link #DEFAULT_BATCH_SIZE}. 086 * 087 * @param batchSize the size 088 */ 089 public void setBatchSize(int batchSize) { 090 this.batchSize = batchSize; 091 } 092 093 public int getOutBatchSize() { 094 return outBatchSize; 095 } 096 097 /** 098 * Sets the <b>out</b> batch size. If the batch processor holds more exchanges than this out size then the 099 * completion is triggered. Can for instance be used to ensure that this batch is completed when a certain 100 * number of exchanges has been collected. By default this feature is <b>not</b> enabled. 101 * 102 * @param outBatchSize the size 103 */ 104 public void setOutBatchSize(int outBatchSize) { 105 this.outBatchSize = outBatchSize; 106 } 107 108 public long getBatchTimeout() { 109 return batchTimeout; 110 } 111 112 public void setBatchTimeout(long batchTimeout) { 113 this.batchTimeout = batchTimeout; 114 } 115 116 public Processor getProcessor() { 117 return processor; 118 } 119 120 /** 121 * A strategy method to decide if the "in" batch is completed. That is, whether the resulting exchanges in 122 * the in queue should be drained to the "out" collection. 123 */ 124 private boolean isInBatchCompleted(int num) { 125 return num >= batchSize; 126 } 127 128 /** 129 * A strategy method to decide if the "out" batch is completed. That is, whether the resulting exchange in 130 * the out collection should be sent. 131 */ 132 private boolean isOutBatchCompleted() { 133 if (outBatchSize == 0) { 134 // out batch is disabled, so go ahead and send. 135 return true; 136 } 137 return collection.size() > 0 && collection.size() >= outBatchSize; 138 } 139 140 /** 141 * Strategy Method to process an exchange in the batch. This method allows derived classes to perform 142 * custom processing before or after an individual exchange is processed 143 */ 144 protected void processExchange(Exchange exchange) throws Exception { 145 processor.process(exchange); 146 } 147 148 protected void doStart() throws Exception { 149 ServiceHelper.startServices(processor); 150 sender.start(); 151 } 152 153 protected void doStop() throws Exception { 154 sender.cancel(); 155 ServiceHelper.stopServices(processor); 156 collection.clear(); 157 } 158 159 /** 160 * Enqueues an exchange for later batch processing. 161 */ 162 public void process(Exchange exchange) throws Exception { 163 sender.enqueueExchange(exchange); 164 } 165 166 /** 167 * Sender thread for queued-up exchanges. 168 */ 169 private class BatchSender extends Thread { 170 171 private Queue<Exchange> queue; 172 private Lock queueLock = new ReentrantLock(); 173 private boolean exchangeEnqueued; 174 private Condition exchangeEnqueuedCondition = queueLock.newCondition(); 175 176 public BatchSender() { 177 super("Batch Sender"); 178 this.queue = new LinkedList<Exchange>(); 179 } 180 181 @Override 182 public void run() { 183 // Wait until one of either: 184 // * an exchange being queued; 185 // * the batch timeout expiring; or 186 // * the thread being cancelled. 187 // 188 // If an exchange is queued then we need to determine whether the 189 // batch is complete. If it is complete then we send out the batched 190 // exchanges. Otherwise we move back into our wait state. 191 // 192 // If the batch times out then we send out the batched exchanges 193 // collected so far. 194 // 195 // If we receive an interrupt then all blocking operations are 196 // interrupted and our thread terminates. 197 // 198 // The goal of the following algorithm in terms of synchronisation 199 // is to provide fine grained locking i.e. retaining the lock only 200 // when required. Special consideration is given to releasing the 201 // lock when calling an overloaded method i.e. sendExchanges. 202 // Unlocking is important as the process of sending out the exchanges 203 // would otherwise block new exchanges from being queued. 204 205 queueLock.lock(); 206 try { 207 do { 208 try { 209 if (!exchangeEnqueued) { 210 exchangeEnqueuedCondition.await(batchTimeout, TimeUnit.MILLISECONDS); 211 } 212 213 if (!exchangeEnqueued) { 214 drainQueueTo(collection, batchSize); 215 } else { 216 exchangeEnqueued = false; 217 while (isInBatchCompleted(queue.size())) { 218 drainQueueTo(collection, batchSize); 219 } 220 221 if (!isOutBatchCompleted()) { 222 continue; 223 } 224 } 225 226 queueLock.unlock(); 227 try { 228 try { 229 sendExchanges(); 230 } catch (Exception e) { 231 getExceptionHandler().handleException(e); 232 } 233 } finally { 234 queueLock.lock(); 235 } 236 237 } catch (InterruptedException e) { 238 break; 239 } 240 241 } while (true); 242 243 } finally { 244 queueLock.unlock(); 245 } 246 } 247 248 /** 249 * This method should be called with queueLock held 250 */ 251 private void drainQueueTo(Collection<Exchange> collection, int batchSize) { 252 for (int i = 0; i < batchSize; ++i) { 253 Exchange e = queue.poll(); 254 if (e != null) { 255 collection.add(e); 256 } else { 257 break; 258 } 259 } 260 } 261 262 public void cancel() { 263 interrupt(); 264 } 265 266 public void enqueueExchange(Exchange exchange) { 267 queueLock.lock(); 268 try { 269 queue.add(exchange); 270 exchangeEnqueued = true; 271 exchangeEnqueuedCondition.signal(); 272 } finally { 273 queueLock.unlock(); 274 } 275 } 276 277 private void sendExchanges() throws Exception { 278 Iterator<Exchange> iter = collection.iterator(); 279 while (iter.hasNext()) { 280 Exchange exchange = iter.next(); 281 iter.remove(); 282 processExchange(exchange); 283 } 284 } 285 } 286 287 }