001 /** 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017 package org.apache.camel.component.file; 018 019 import java.io.File; 020 import java.util.concurrent.ConcurrentHashMap; 021 022 import org.apache.camel.AsyncCallback; 023 import org.apache.camel.Processor; 024 import org.apache.camel.impl.ScheduledPollConsumer; 025 import org.apache.commons.logging.Log; 026 import org.apache.commons.logging.LogFactory; 027 028 /** 029 * @version $Revision: 640438 $ 030 */ 031 public class FileConsumer extends ScheduledPollConsumer<FileExchange> { 032 private static final transient Log LOG = LogFactory.getLog(FileConsumer.class); 033 ConcurrentHashMap<File, File> filesBeingProcessed = new ConcurrentHashMap<File, File>(); 034 boolean generateEmptyExchangeWhenIdle; 035 private final FileEndpoint endpoint; 036 private boolean recursive = true; 037 private String regexPattern = ""; 038 private long lastPollTime; 039 040 private int unchangedDelay; 041 private boolean unchangedSize; 042 private ConcurrentHashMap<File, Long> fileSizes = new ConcurrentHashMap<File, Long>(); 043 044 public FileConsumer(final FileEndpoint endpoint, Processor processor) { 045 super(endpoint, processor); 046 this.endpoint = endpoint; 047 } 048 049 protected synchronized void poll() throws Exception { 050 int rc = pollFileOrDirectory(endpoint.getFile(), isRecursive()); 051 if (rc == 0 && generateEmptyExchangeWhenIdle) { 052 final FileExchange exchange = endpoint.createExchange((File)null); 053 getAsyncProcessor().process(exchange, new AsyncCallback() { 054 public void done(boolean sync) { 055 } 056 }); 057 } 058 lastPollTime = System.currentTimeMillis(); 059 } 060 061 /** 062 * @param fileOrDirectory 063 * @param processDir 064 * @return the number of files processed or being processed async. 065 */ 066 protected int pollFileOrDirectory(File fileOrDirectory, boolean processDir) { 067 if (!fileOrDirectory.isDirectory()) { 068 return pollFile(fileOrDirectory); // process the file 069 } else if (processDir) { 070 int rc = 0; 071 if (isValidFile(fileOrDirectory)) { 072 LOG.debug("Polling directory " + fileOrDirectory); 073 File[] files = fileOrDirectory.listFiles(); 074 for (int i = 0; i < files.length; i++) { 075 rc += pollFileOrDirectory(files[i], isRecursive()); // self-recursion 076 } 077 } 078 return rc; 079 } else { 080 LOG.debug("Skipping directory " + fileOrDirectory); 081 return 0; 082 } 083 } 084 085 /** 086 * @param file 087 * @return the number of files processed or being processed async. 088 */ 089 protected int pollFile(final File file) { 090 091 if (!file.exists()) { 092 return 0; 093 } 094 if (!isValidFile(file)) { 095 return 0; 096 } 097 // we only care about file modified times if we are not deleting/moving 098 // files 099 if (endpoint.isNoop()) { 100 long fileModified = file.lastModified(); 101 if (fileModified <= lastPollTime) { 102 if (LOG.isDebugEnabled()) { 103 LOG.debug("Ignoring file: " + file + " as modified time: " + fileModified 104 + " less than last poll time: " + lastPollTime); 105 } 106 return 0; 107 } 108 } else { 109 if (filesBeingProcessed.contains(file)) { 110 return 1; 111 } 112 filesBeingProcessed.put(file, file); 113 } 114 115 final FileProcessStrategy processStrategy = endpoint.getFileStrategy(); 116 final FileExchange exchange = endpoint.createExchange(file); 117 118 endpoint.configureMessage(file, exchange.getIn()); 119 try { 120 if (LOG.isDebugEnabled()) { 121 LOG.debug("About to process file: " + file + " using exchange: " + exchange); 122 } 123 if (processStrategy.begin(endpoint, exchange, file)) { 124 125 // Use the async processor interface so that processing of 126 // the 127 // exchange can happen asynchronously 128 getAsyncProcessor().process(exchange, new AsyncCallback() { 129 public void done(boolean sync) { 130 if (exchange.getException() == null) { 131 try { 132 processStrategy.commit(endpoint, (FileExchange)exchange, file); 133 } catch (Exception e) { 134 handleException(e); 135 } 136 } else { 137 handleException(exchange.getException()); 138 } 139 filesBeingProcessed.remove(file); 140 } 141 }); 142 143 } else { 144 if (LOG.isDebugEnabled()) { 145 LOG.debug(endpoint + " cannot process file: " + file); 146 } 147 } 148 } catch (Throwable e) { 149 handleException(e); 150 } 151 return 1; 152 } 153 154 protected boolean isValidFile(File file) { 155 boolean result = false; 156 if (file != null && file.exists()) { 157 // TODO: maybe use a configurable strategy instead of the 158 // hardcoded one based on last file change 159 if (isMatched(file) && isUnchanged(file)) { 160 result = true; 161 } 162 } 163 return result; 164 } 165 166 167 protected boolean isUnchanged(File file) { 168 if (file == null) { 169 // Sanity check 170 return false; 171 } else if (file.isDirectory()) { 172 // Allow recursive polling to descend into this directory 173 return true; 174 } else { 175 boolean lastModifiedCheck = true; 176 long modifiedDuration = 0; 177 if (getUnchangedDelay() > 0) { 178 modifiedDuration = System.currentTimeMillis() - file.lastModified(); 179 lastModifiedCheck = modifiedDuration >= getUnchangedDelay(); 180 } 181 182 boolean sizeCheck = true; 183 long sizeDifference = 0; 184 if (isUnchangedSize()) { 185 long prevFileSize = (fileSizes.get(file) == null) ? 0 : fileSizes.get(file).longValue(); 186 sizeDifference = file.length() - prevFileSize; 187 sizeCheck = 0 == sizeDifference; 188 } 189 190 boolean answer = lastModifiedCheck && sizeCheck; 191 192 if (LOG.isDebugEnabled()) { 193 LOG.debug("file:" + file + " isUnchanged:" + answer + " " + "sizeCheck:" + sizeCheck + "(" 194 + sizeDifference + ") " + "lastModifiedCheck:" + lastModifiedCheck + "(" 195 + modifiedDuration + ")"); 196 } 197 198 if (isUnchangedSize()) { 199 if (answer) { 200 fileSizes.remove(file); 201 } else { 202 fileSizes.put(file, file.length()); 203 } 204 } 205 206 return answer; 207 } 208 } 209 210 protected boolean isMatched(File file) { 211 String name = file.getName(); 212 if (regexPattern != null && regexPattern.length() > 0) { 213 if (!name.matches(getRegexPattern())) { 214 return false; 215 } 216 } 217 String[] prefixes = endpoint.getExcludedNamePrefixes(); 218 if (prefixes != null) { 219 for (String prefix : prefixes) { 220 if (name.startsWith(prefix)) { 221 return false; 222 } 223 } 224 } 225 String[] postfixes = endpoint.getExcludedNamePostfixes(); 226 if (postfixes != null) { 227 for (String postfix : postfixes) { 228 if (name.endsWith(postfix)) { 229 return false; 230 } 231 } 232 } 233 return true; 234 } 235 236 /** 237 * @return the recursive 238 */ 239 public boolean isRecursive() { 240 return this.recursive; 241 } 242 243 /** 244 * @param recursive the recursive to set 245 */ 246 public void setRecursive(boolean recursive) { 247 this.recursive = recursive; 248 } 249 250 /** 251 * @return the regexPattern 252 */ 253 public String getRegexPattern() { 254 return this.regexPattern; 255 } 256 257 /** 258 * @param regexPattern the regexPattern to set 259 */ 260 public void setRegexPattern(String regexPattern) { 261 this.regexPattern = regexPattern; 262 } 263 264 public boolean isGenerateEmptyExchangeWhenIdle() { 265 return generateEmptyExchangeWhenIdle; 266 } 267 268 public void setGenerateEmptyExchangeWhenIdle(boolean generateEmptyExchangeWhenIdle) { 269 this.generateEmptyExchangeWhenIdle = generateEmptyExchangeWhenIdle; 270 } 271 272 public int getUnchangedDelay() { 273 return unchangedDelay; 274 } 275 276 public void setUnchangedDelay(int unchangedDelay) { 277 this.unchangedDelay = unchangedDelay; 278 } 279 280 public boolean isUnchangedSize() { 281 return unchangedSize; 282 } 283 284 public void setUnchangedSize(boolean unchangedSize) { 285 this.unchangedSize = unchangedSize; 286 } 287 288 }