001    /**
002     * Licensed to the Apache Software Foundation (ASF) under one or more
003     * contributor license agreements.  See the NOTICE file distributed with
004     * this work for additional information regarding copyright ownership.
005     * The ASF licenses this file to You under the Apache License, Version 2.0
006     * (the "License"); you may not use this file except in compliance with
007     * the License.  You may obtain a copy of the License at
008     *
009     *      http://www.apache.org/licenses/LICENSE-2.0
010     *
011     * Unless required by applicable law or agreed to in writing, software
012     * distributed under the License is distributed on an "AS IS" BASIS,
013     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014     * See the License for the specific language governing permissions and
015     * limitations under the License.
016     */
017    package org.apache.camel.component.file;
018    
019    import java.io.File;
020    import java.util.concurrent.ConcurrentHashMap;
021    
022    import org.apache.camel.AsyncCallback;
023    import org.apache.camel.Processor;
024    import org.apache.camel.impl.ScheduledPollConsumer;
025    import org.apache.commons.logging.Log;
026    import org.apache.commons.logging.LogFactory;
027    
028    /**
029     * @version $Revision: 640438 $
030     */
031    public class FileConsumer extends ScheduledPollConsumer<FileExchange> {
032        private static final transient Log LOG = LogFactory.getLog(FileConsumer.class);
033        ConcurrentHashMap<File, File> filesBeingProcessed = new ConcurrentHashMap<File, File>();
034        boolean generateEmptyExchangeWhenIdle;
035        private final FileEndpoint endpoint;
036        private boolean recursive = true;
037        private String regexPattern = "";
038        private long lastPollTime;
039    
040        private int unchangedDelay;
041        private boolean unchangedSize;
042        private ConcurrentHashMap<File, Long> fileSizes = new ConcurrentHashMap<File, Long>();
043    
044        public FileConsumer(final FileEndpoint endpoint, Processor processor) {
045            super(endpoint, processor);
046            this.endpoint = endpoint;
047        }
048    
049        protected synchronized void poll() throws Exception {
050            int rc = pollFileOrDirectory(endpoint.getFile(), isRecursive());
051            if (rc == 0 && generateEmptyExchangeWhenIdle) {
052                final FileExchange exchange = endpoint.createExchange((File)null);
053                getAsyncProcessor().process(exchange, new AsyncCallback() {
054                    public void done(boolean sync) {
055                    }
056                });
057            }
058            lastPollTime = System.currentTimeMillis();
059        }
060    
061        /**
062         * @param fileOrDirectory
063         * @param processDir
064         * @return the number of files processed or being processed async.
065         */
066        protected int pollFileOrDirectory(File fileOrDirectory, boolean processDir) {
067            if (!fileOrDirectory.isDirectory()) {
068                return pollFile(fileOrDirectory); // process the file
069            } else if (processDir) {
070                int rc = 0;
071                if (isValidFile(fileOrDirectory)) {
072                    LOG.debug("Polling directory " + fileOrDirectory);
073                    File[] files = fileOrDirectory.listFiles();
074                    for (int i = 0; i < files.length; i++) {
075                        rc += pollFileOrDirectory(files[i], isRecursive()); // self-recursion
076                    }
077                }
078                return rc;
079            } else {
080                LOG.debug("Skipping directory " + fileOrDirectory);
081                return 0;
082            }
083        }
084    
085        /**
086         * @param file
087         * @return the number of files processed or being processed async.
088         */
089        protected int pollFile(final File file) {
090    
091            if (!file.exists()) {
092                return 0;
093            }
094            if (!isValidFile(file)) {
095                return 0;
096            }
097            // we only care about file modified times if we are not deleting/moving
098            // files
099            if (endpoint.isNoop()) {
100                long fileModified = file.lastModified();
101                if (fileModified <= lastPollTime) {
102                    if (LOG.isDebugEnabled()) {
103                        LOG.debug("Ignoring file: " + file + " as modified time: " + fileModified
104                                  + " less than last poll time: " + lastPollTime);
105                    }
106                    return 0;
107                }
108            } else {
109                if (filesBeingProcessed.contains(file)) {
110                    return 1;
111                }
112                filesBeingProcessed.put(file, file);
113            }
114    
115            final FileProcessStrategy processStrategy = endpoint.getFileStrategy();
116            final FileExchange exchange = endpoint.createExchange(file);
117    
118            endpoint.configureMessage(file, exchange.getIn());
119            try {
120                if (LOG.isDebugEnabled()) {
121                    LOG.debug("About to process file:  " + file + " using exchange: " + exchange);
122                }
123                if (processStrategy.begin(endpoint, exchange, file)) {
124    
125                    // Use the async processor interface so that processing of
126                    // the
127                    // exchange can happen asynchronously
128                    getAsyncProcessor().process(exchange, new AsyncCallback() {
129                        public void done(boolean sync) {
130                            if (exchange.getException() == null) {
131                                try {
132                                    processStrategy.commit(endpoint, (FileExchange)exchange, file);
133                                } catch (Exception e) {
134                                    handleException(e);
135                                }
136                            } else {
137                                handleException(exchange.getException());
138                            }
139                            filesBeingProcessed.remove(file);
140                        }
141                    });
142    
143                } else {
144                    if (LOG.isDebugEnabled()) {
145                        LOG.debug(endpoint + " cannot process file: " + file);
146                    }
147                }
148            } catch (Throwable e) {
149                handleException(e);
150            }
151            return 1;
152        }
153    
154        protected boolean isValidFile(File file) {
155            boolean result = false;
156            if (file != null && file.exists()) {
157                // TODO: maybe use a configurable strategy instead of the
158                // hardcoded one based on last file change
159                if (isMatched(file) && isUnchanged(file)) {
160                    result = true;
161                }
162            }
163            return result;
164        }
165    
166    
167        protected boolean isUnchanged(File file) {
168            if (file == null) {
169                // Sanity check
170                return false;
171            } else if (file.isDirectory()) {
172                // Allow recursive polling to descend into this directory
173                return true;
174            } else {
175                boolean lastModifiedCheck = true;
176                long modifiedDuration = 0;
177                if (getUnchangedDelay() > 0) {
178                    modifiedDuration = System.currentTimeMillis() - file.lastModified();
179                    lastModifiedCheck = modifiedDuration >= getUnchangedDelay();
180                }
181    
182                boolean sizeCheck = true;
183                long sizeDifference = 0;
184                if (isUnchangedSize()) {
185                    long prevFileSize = (fileSizes.get(file) == null) ? 0 : fileSizes.get(file).longValue();
186                    sizeDifference = file.length() - prevFileSize;
187                    sizeCheck = 0 == sizeDifference;
188                }
189    
190                boolean answer = lastModifiedCheck && sizeCheck;
191    
192                if (LOG.isDebugEnabled()) {
193                    LOG.debug("file:" + file + " isUnchanged:" + answer + " " + "sizeCheck:" + sizeCheck + "("
194                              + sizeDifference + ") " + "lastModifiedCheck:" + lastModifiedCheck + "("
195                              + modifiedDuration + ")");
196                }
197    
198                if (isUnchangedSize()) {
199                    if (answer) {
200                        fileSizes.remove(file);
201                    } else {
202                        fileSizes.put(file, file.length());
203                    }
204                }
205    
206                return answer;
207            }
208        }
209    
210        protected boolean isMatched(File file) {
211            String name = file.getName();
212            if (regexPattern != null && regexPattern.length() > 0) {
213                if (!name.matches(getRegexPattern())) {
214                    return false;
215                }
216            }
217            String[] prefixes = endpoint.getExcludedNamePrefixes();
218            if (prefixes != null) {
219                for (String prefix : prefixes) {
220                    if (name.startsWith(prefix)) {
221                        return false;
222                    }
223                }
224            }
225            String[] postfixes = endpoint.getExcludedNamePostfixes();
226            if (postfixes != null) {
227                for (String postfix : postfixes) {
228                    if (name.endsWith(postfix)) {
229                        return false;
230                    }
231                }
232            }
233            return true;
234        }
235    
236        /**
237         * @return the recursive
238         */
239        public boolean isRecursive() {
240            return this.recursive;
241        }
242    
243        /**
244         * @param recursive the recursive to set
245         */
246        public void setRecursive(boolean recursive) {
247            this.recursive = recursive;
248        }
249    
250        /**
251         * @return the regexPattern
252         */
253        public String getRegexPattern() {
254            return this.regexPattern;
255        }
256    
257        /**
258         * @param regexPattern the regexPattern to set
259         */
260        public void setRegexPattern(String regexPattern) {
261            this.regexPattern = regexPattern;
262        }
263    
264        public boolean isGenerateEmptyExchangeWhenIdle() {
265            return generateEmptyExchangeWhenIdle;
266        }
267    
268        public void setGenerateEmptyExchangeWhenIdle(boolean generateEmptyExchangeWhenIdle) {
269            this.generateEmptyExchangeWhenIdle = generateEmptyExchangeWhenIdle;
270        }
271    
272        public int getUnchangedDelay() {
273            return unchangedDelay;
274        }
275    
276        public void setUnchangedDelay(int unchangedDelay) {
277            this.unchangedDelay = unchangedDelay;
278        }
279    
280        public boolean isUnchangedSize() {
281            return unchangedSize;
282        }
283    
284        public void setUnchangedSize(boolean unchangedSize) {
285            this.unchangedSize = unchangedSize;
286        }
287    
288    }