001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one or more
003     * contributor license agreements.  See the NOTICE file distributed with
004     * this work for additional information regarding copyright ownership.
005     * The ASF licenses this file to You under the Apache License, Version 2.0
006     * (the "License"); you may not use this file except in compliance with
007     * the License.  You may obtain a copy of the License at
008     *
009     *      http://www.apache.org/licenses/LICENSE-2.0
010     *
011     * Unless required by applicable law or agreed to in writing, software
012     * distributed under the License is distributed on an "AS IS" BASIS,
013     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014     * See the License for the specific language governing permissions and
015     * limitations under the License.
016     */
017    package org.apache.camel.processor;
018    
019    import java.util.Collection;
020    import java.util.Iterator;
021    import java.util.LinkedList;
022    import java.util.Queue;
023    import java.util.concurrent.TimeUnit;
024    import java.util.concurrent.locks.Condition;
025    import java.util.concurrent.locks.Lock;
026    import java.util.concurrent.locks.ReentrantLock;
027    
028    import org.apache.camel.Exchange;
029    import org.apache.camel.Processor;
030    import org.apache.camel.impl.LoggingExceptionHandler;
031    import org.apache.camel.impl.ServiceSupport;
032    import org.apache.camel.spi.ExceptionHandler;
033    import org.apache.camel.util.ServiceHelper;
034    
035    /**
036     * A base class for any kind of {@link Processor} which implements some kind of batch processing.
037     * 
038     * @version $Revision: 765825 $
039     */
040    public class BatchProcessor extends ServiceSupport implements Processor {
041    
042        public static final long DEFAULT_BATCH_TIMEOUT = 1000L;
043        public static final int DEFAULT_BATCH_SIZE = 100;
044    
045        private long batchTimeout = DEFAULT_BATCH_TIMEOUT;
046        private int batchSize = DEFAULT_BATCH_SIZE;
047        private int outBatchSize;
048    
049        private Processor processor;
050        private Collection<Exchange> collection;
051        private ExceptionHandler exceptionHandler;
052    
053        private BatchSender sender;
054        
055        public BatchProcessor(Processor processor, Collection<Exchange> collection) {
056            this.processor = processor;
057            this.collection = collection;
058            this.sender = new BatchSender();
059        }
060    
061        @Override
062        public String toString() {
063            return "BatchProcessor[to: " + processor + "]";
064        }
065    
066        // Properties
067        // -------------------------------------------------------------------------
068        public ExceptionHandler getExceptionHandler() {
069            if (exceptionHandler == null) {
070                exceptionHandler = new LoggingExceptionHandler(getClass());
071            }
072            return exceptionHandler;
073        }
074    
075        public void setExceptionHandler(ExceptionHandler exceptionHandler) {
076            this.exceptionHandler = exceptionHandler;
077        }
078    
079        public int getBatchSize() {
080            return batchSize;
081        }
082    
083        /**
084         * Sets the <b>in</b> batch size. This is the number of incoming exchanges that this batch processor will
085         * process before its completed. The default value is {@link #DEFAULT_BATCH_SIZE}.
086         * 
087         * @param batchSize the size
088         */
089        public void setBatchSize(int batchSize) {
090            this.batchSize = batchSize;
091        }
092    
093        public int getOutBatchSize() {
094            return outBatchSize;
095        }
096    
097        /**
098         * Sets the <b>out</b> batch size. If the batch processor holds more exchanges than this out size then the
099         * completion is triggered. Can for instance be used to ensure that this batch is completed when a certain
100         * number of exchanges has been collected. By default this feature is <b>not</b> enabled.
101         * 
102         * @param outBatchSize the size
103         */
104        public void setOutBatchSize(int outBatchSize) {
105            this.outBatchSize = outBatchSize;
106        }
107    
108        public long getBatchTimeout() {
109            return batchTimeout;
110        }
111    
112        public void setBatchTimeout(long batchTimeout) {
113            this.batchTimeout = batchTimeout;
114        }
115    
116        public Processor getProcessor() {
117            return processor;
118        }
119    
120        /**
121         * A strategy method to decide if the "in" batch is completed. That is, whether the resulting exchanges in
122         * the in queue should be drained to the "out" collection.
123         */
124        private boolean isInBatchCompleted(int num) {
125            return num >= batchSize;
126        }
127    
128        /**
129         * A strategy method to decide if the "out" batch is completed. That is, whether the resulting exchange in
130         * the out collection should be sent.
131         */
132        private boolean isOutBatchCompleted() {
133            if (outBatchSize == 0) {
134                // out batch is disabled, so go ahead and send.
135                return true;
136            }
137            return collection.size() > 0 && collection.size() >= outBatchSize;
138        }
139    
140        /**
141         * Strategy Method to process an exchange in the batch. This method allows derived classes to perform
142         * custom processing before or after an individual exchange is processed
143         */
144        protected void processExchange(Exchange exchange) throws Exception {
145            processor.process(exchange);
146        }
147    
148        protected void doStart() throws Exception {
149            ServiceHelper.startServices(processor);
150            sender.start();
151        }
152    
153        protected void doStop() throws Exception {
154            sender.cancel();
155            ServiceHelper.stopServices(processor);
156            collection.clear();
157        }
158    
159        /**
160         * Enqueues an exchange for later batch processing.
161         */
162        public void process(Exchange exchange) throws Exception {
163            sender.enqueueExchange(exchange);
164        }
165    
166        /**
167         * Sender thread for queued-up exchanges.
168         */
169        private class BatchSender extends Thread {
170    
171            private Queue<Exchange> queue;
172            private Lock queueLock = new ReentrantLock();
173            private boolean exchangeEnqueued;
174            private Condition exchangeEnqueuedCondition = queueLock.newCondition();
175    
176            public BatchSender() {
177                super("Batch Sender");
178                this.queue = new LinkedList<Exchange>();
179            }
180    
181            @Override
182            public void run() {
183                // Wait until one of either:
184                // * an exchange being queued;
185                // * the batch timeout expiring; or
186                // * the thread being cancelled.
187                //
188                // If an exchange is queued then we need to determine whether the
189                // batch is complete. If it is complete then we send out the batched
190                // exchanges. Otherwise we move back into our wait state.
191                //
192                // If the batch times out then we send out the batched exchanges
193                // collected so far.
194                //
195                // If we receive an interrupt then all blocking operations are
196                // interrupted and our thread terminates.
197                //
198                // The goal of the following algorithm in terms of synchronisation
199                // is to provide fine grained locking i.e. retaining the lock only
200                // when required. Special consideration is given to releasing the
201                // lock when calling an overloaded method i.e. sendExchanges. 
202                // Unlocking is important as the process of sending out the exchanges
203                // would otherwise block new exchanges from being queued.
204    
205                queueLock.lock();
206                try {
207                    do {
208                        try {
209                            if (!exchangeEnqueued) {
210                                exchangeEnqueuedCondition.await(batchTimeout, TimeUnit.MILLISECONDS);
211                            }
212    
213                            if (!exchangeEnqueued) {
214                                drainQueueTo(collection, batchSize);
215                            } else {             
216                                exchangeEnqueued = false;
217                                while (isInBatchCompleted(queue.size())) {   
218                                    drainQueueTo(collection, batchSize);
219                                }
220                                
221                                if (!isOutBatchCompleted()) {
222                                    continue;
223                                }
224                            }
225    
226                            queueLock.unlock();
227                            try {
228                                try {
229                                    sendExchanges();
230                                } catch (Exception e) {
231                                    getExceptionHandler().handleException(e);
232                                }
233                            } finally {
234                                queueLock.lock();
235                            }
236    
237                        } catch (InterruptedException e) {
238                            break;
239                        }
240    
241                    } while (true);
242    
243                } finally {
244                    queueLock.unlock();
245                }
246            }
247    
248            /**
249             * This method should be called with queueLock held
250             */
251            private void drainQueueTo(Collection<Exchange> collection, int batchSize) {
252                for (int i = 0; i < batchSize; ++i) {
253                    Exchange e = queue.poll();
254                    if (e != null) {
255                        collection.add(e);
256                    } else {
257                        break;
258                    }
259                }
260            }
261    
262            public void cancel() {
263                interrupt();
264            }
265    
266            public void enqueueExchange(Exchange exchange) {
267                queueLock.lock();
268                try {
269                    queue.add(exchange);
270                    exchangeEnqueued = true;
271                    exchangeEnqueuedCondition.signal();
272                } finally {
273                    queueLock.unlock();
274                }
275            }
276    
277            private void sendExchanges() throws Exception {
278                Iterator<Exchange> iter = collection.iterator();
279                while (iter.hasNext()) {
280                    Exchange exchange = iter.next();
281                    iter.remove();
282                    processExchange(exchange);
283                }
284            }
285        }
286    
287    }