001    /****************************************************************
002     * Licensed to the Apache Software Foundation (ASF) under one   *
003     * or more contributor license agreements.  See the NOTICE file *
004     * distributed with this work for additional information        *
005     * regarding copyright ownership.  The ASF licenses this file   *
006     * to you under the Apache License, Version 2.0 (the            *
007     * "License"); you may not use this file except in compliance   *
008     * with the License.  You may obtain a copy of the License at   *
009     *                                                              *
010     *   http://www.apache.org/licenses/LICENSE-2.0                 *
011     *                                                              *
012     * Unless required by applicable law or agreed to in writing,   *
013     * software distributed under the License is distributed on an  *
014     * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
015     * KIND, either express or implied.  See the License for the    *
016     * specific language governing permissions and limitations      *
017     * under the License.                                           *
018     ****************************************************************/
019    
020    
021    package org.apache.james.jspf.core;
022    
023    /**
024     * This Class is used to convert all macros which can used in SPF-Records to the
025     * right values!
026     * 
027     */
028    
029    import org.apache.james.jspf.core.exceptions.NeutralException;
030    import org.apache.james.jspf.core.exceptions.NoneException;
031    import org.apache.james.jspf.core.exceptions.PermErrorException;
032    import org.apache.james.jspf.core.exceptions.TempErrorException;
033    import org.apache.james.jspf.core.exceptions.TimeoutException;
034    
035    import java.io.UnsupportedEncodingException;
036    import java.net.URLEncoder;
037    import java.util.ArrayList;
038    import java.util.Iterator;
039    import java.util.List;
040    import java.util.regex.Matcher;
041    import java.util.regex.Pattern;
042    
043    public class MacroExpand {
044    
045        private Pattern domainSpecPattern;
046    
047        private Pattern macroStringPattern;
048    
049        private Pattern macroLettersPattern;
050    
051        private Pattern macroLettersExpPattern;
052    
053        private Pattern cellPattern;
054    
055        private Logger log;
056    
057        private DNSService dnsProbe;
058    
059        public static final boolean EXPLANATION = true;
060        
061        public static final boolean DOMAIN = false;
062        
063        public static class RequireClientDomainException extends Exception {
064    
065            private static final long serialVersionUID = 3834282981657676530L;
066            
067        }
068    
069        /**
070         * Construct MacroExpand
071         * 
072         * @param logger the logget to use
073         * @param dnsProbe the dns service to use
074         */
075        public MacroExpand(Logger logger, DNSService dnsProbe) {
076            // This matches 2 groups
077            domainSpecPattern = Pattern.compile(SPFTermsRegexps.DOMAIN_SPEC_REGEX_R);
078            // The real pattern replacer
079            macroStringPattern = Pattern.compile(SPFTermsRegexps.MACRO_STRING_REGEX_TOKEN);
080            // The macro letters pattern
081            macroLettersExpPattern = Pattern.compile(SPFTermsRegexps.MACRO_LETTER_PATTERN_EXP);
082            macroLettersPattern = Pattern.compile(SPFTermsRegexps.MACRO_LETTER_PATTERN);
083            log = logger;
084            this.dnsProbe = dnsProbe;
085        }
086        
087    
088        private static final class AResponseListener implements
089                SPFCheckerDNSResponseListener {
090            
091            /**
092             * @see org.apache.james.jspf.core.SPFCheckerDNSResponseListener#onDNSResponse(org.apache.james.jspf.core.DNSResponse, org.apache.james.jspf.core.SPFSession)
093             */
094            public DNSLookupContinuation onDNSResponse(DNSResponse response, SPFSession session)
095                    throws PermErrorException, NoneException, TempErrorException,
096                    NeutralException {
097                // just return the default "unknown" if we cannot find anything
098                // later
099                session.setClientDomain("unknown");
100                try {
101                    List<String> records = response.getResponse();
102                    if (records != null && records.size() > 0) {
103                        Iterator<String> i = records.iterator();
104                        while (i.hasNext()) {
105                            String next = i.next();
106                            if (IPAddr.getAddress(session.getIpAddress())
107                                    .toString().equals(
108                                            IPAddr.getAddress(next).toString())) {
109                                session
110                                        .setClientDomain((String) session
111                                                .getAttribute(ATTRIBUTE_MACRO_EXPAND_CHECKED_RECORD));
112                                break;
113                            }
114                        }
115                    }
116                } catch (TimeoutException e) {
117                    // just return the default "unknown".
118                } catch (PermErrorException e) {
119                    // just return the default "unknown".
120                }
121                return null;
122            }
123        }
124    
125        private static final class PTRResponseListener implements
126                SPFCheckerDNSResponseListener {
127    
128            /**
129             * @see org.apache.james.jspf.core.SPFCheckerDNSResponseListener#onDNSResponse(org.apache.james.jspf.core.DNSResponse, org.apache.james.jspf.core.SPFSession)
130             */
131            public DNSLookupContinuation onDNSResponse(DNSResponse response, SPFSession session)
132                    throws PermErrorException, NoneException, TempErrorException,
133                    NeutralException {
134    
135                try {
136                    boolean ip6 = IPAddr.isIPV6(session.getIpAddress());
137                    List<String> records = response.getResponse();
138    
139                    if (records != null && records.size() > 0) {
140                        String record = records.get(0);
141                        session.setAttribute(ATTRIBUTE_MACRO_EXPAND_CHECKED_RECORD,
142                                record);
143    
144                        return new DNSLookupContinuation(new DNSRequest(record,
145                                ip6 ? DNSRequest.AAAA : DNSRequest.A), 
146                                new AResponseListener());
147    
148                    }
149                } catch (TimeoutException e) {
150                    // just return the default "unknown".
151                    session.setClientDomain("unknown");
152                }
153                return null;
154    
155            }
156        }
157    
158        private static final String ATTRIBUTE_MACRO_EXPAND_CHECKED_RECORD = "MacroExpand.checkedRecord";
159    
160        public DNSLookupContinuation checkExpand(String input, SPFSession session, boolean isExplanation) throws PermErrorException, NoneException {
161            if (input != null) {
162                String host = this.expand(input, session, isExplanation);
163                if (host == null) {
164    
165                    return new DNSLookupContinuation(new DNSRequest(IPAddr
166                            .getAddress(session.getIpAddress()).getReverseIP(),
167                            DNSRequest.PTR), new PTRResponseListener());
168                }
169            }
170            return null;
171        }
172        
173        public String expand(String input, MacroData macroData, boolean isExplanation) throws PermErrorException {
174            try {
175                if (isExplanation) {
176                    return expandExplanation(input, macroData);
177                } else {
178                    return expandDomain(input, macroData);
179                }
180            } catch (RequireClientDomainException e) {
181                return null;
182            }
183        }
184    
185        /**
186         * This method expand the given a explanation
187         * 
188         * @param input
189         *            The explanation which should be expanded
190         * @return expanded The expanded explanation
191         * @throws PermErrorException
192         *             Get thrown if invalid macros are used
193         * @throws RequireClientDomain 
194         */
195        private String expandExplanation(String input, MacroData macroData) throws PermErrorException, RequireClientDomainException {
196    
197            log.debug("Start do expand explanation: " + input);
198    
199            String[] parts = input.split(" ");
200            StringBuffer res = new StringBuffer();
201            for (int i = 0; i < parts.length; i++) {
202                if (i > 0) res.append(" ");
203                res.append(expandMacroString(parts[i], macroData, true));
204            }
205            log.debug("Done expand explanation: " + res);
206            
207            return res.toString();
208        }
209    
210        /**
211         * This method expand the given domain. So all known macros get replaced
212         * 
213         * @param input
214         *            The domain which should be expand
215         * @return expanded The domain with replaced macros
216         * @throws PermErrorException
217         *             This get thrown if invalid macros are used
218         * @throws RequireClientDomain 
219         */
220        private String expandDomain(String input, MacroData macroData) throws PermErrorException, RequireClientDomainException {
221    
222            log.debug("Start expand domain: " + input);
223    
224            Matcher inputMatcher = domainSpecPattern.matcher(input);
225            if (!inputMatcher.matches() || inputMatcher.groupCount() != 2) {
226                throw new PermErrorException("Invalid DomainSpec: "+input);
227            }
228    
229            StringBuffer res = new StringBuffer();
230            if (inputMatcher.group(1) != null && inputMatcher.group(1).length() > 0) {
231                res.append(expandMacroString(inputMatcher.group(1), macroData, false));
232            }
233            if (inputMatcher.group(2) != null && inputMatcher.group(2).length() > 0) {
234                if (inputMatcher.group(2).startsWith(".")) {
235                    res.append(inputMatcher.group(2));
236                } else {
237                    res.append(expandMacroString(inputMatcher.group(2), macroData, false));
238                }
239            }
240            
241            String domainName = expandMacroString(input, macroData, false);
242            // reduce to less than 255 characters, deleting subdomains from left
243            int split = 0;
244            while (domainName.length() > 255 && split > -1) {
245                split = domainName.indexOf(".");
246                domainName = domainName.substring(split + 1);
247            }
248    
249            log.debug("Domain expanded: " + domainName);
250            
251            return domainName;
252        }
253    
254        /**
255         * Expand the given String
256         * 
257         * @param input
258         *            The inputString which should get expanded
259         * @return expanded The expanded given String
260         * @throws PermErrorException
261         *             This get thrown if invalid macros are used
262         * @throws RequireClientDomain 
263         */
264        private String expandMacroString(String input, MacroData macroData, boolean isExplanation) throws PermErrorException, RequireClientDomainException {
265    
266            StringBuffer decodedValue = new StringBuffer();
267            Matcher inputMatcher = macroStringPattern.matcher(input);
268            String macroCell;
269            int pos = 0;
270    
271            while (inputMatcher.find()) {
272                String match2 = inputMatcher.group();
273                if (pos != inputMatcher.start()) {
274                    throw new PermErrorException("Middle part does not match: "+input.substring(0,pos)+">>"+input.substring(pos, inputMatcher.start())+"<<"+input.substring(inputMatcher.start())+" ["+input+"]");
275                }
276                if (match2.length() > 0) {
277                    if (match2.startsWith("%{")) {
278                        macroCell = input.substring(inputMatcher.start() + 2, inputMatcher
279                                .end() - 1);
280                        inputMatcher
281                                .appendReplacement(decodedValue, escapeForMatcher(replaceCell(macroCell, macroData, isExplanation)));
282                    } else if (match2.length() == 2 && match2.startsWith("%")) {
283                        // handle the % escaping
284                        /*
285                         * From RFC4408:
286                         * 
287                         * A literal "%" is expressed by "%%".
288                         *   "%_" expands to a single " " space.
289                         *   "%-" expands to a URL-encoded space, viz., "%20".
290                         */
291                        if ("%_".equals(match2)) {
292                            inputMatcher.appendReplacement(decodedValue, " ");
293                        } else if ("%-".equals(match2)) {
294                            inputMatcher.appendReplacement(decodedValue, "%20");
295                        } else {
296                            inputMatcher.appendReplacement(decodedValue, escapeForMatcher(match2.substring(1)));
297                        }
298                    }
299                }
300                
301                pos = inputMatcher.end();
302            }
303            
304            if (input.length() != pos) {
305                throw new PermErrorException("End part does not match: "+input.substring(pos));
306            }
307            
308            inputMatcher.appendTail(decodedValue);
309    
310            return decodedValue.toString();
311        }
312    
313        /**
314         * Replace the macros in given String
315         * 
316         * @param replaceValue
317         *            The String in which known macros should get replaced
318         * @return returnData The String with replaced macros
319         * @throws PermErrorException
320         *             Get thrown if an error in processing happen
321         * @throws RequireClientDomain 
322         */
323        private String replaceCell(String replaceValue, MacroData macroData, boolean isExplanation) throws PermErrorException, RequireClientDomainException {
324    
325            String variable = "";
326            String domainNumber = "";
327            boolean isReversed = false;
328            String delimeters = ".";
329    
330            
331            // Get only command character so that 'r' command and 'r' modifier don't
332            // clash
333            String commandCharacter = replaceValue.substring(0, 1);
334            Matcher cellMatcher;
335            // Find command
336            if (isExplanation) {
337                cellMatcher = macroLettersExpPattern.matcher(commandCharacter);
338            } else {
339                cellMatcher = macroLettersPattern.matcher(commandCharacter);
340            }
341            if (cellMatcher.find()) {
342                if (cellMatcher.group().toUpperCase().equals(cellMatcher.group())) {
343                    variable = encodeURL(matchMacro(cellMatcher.group(), macroData));
344                } else {
345                    variable = matchMacro(cellMatcher.group(), macroData);
346                }
347                // Remove Macro code so that r macro code does not clash with r the
348                // reverse modifier
349                replaceValue = replaceValue.substring(1);
350            } else {
351                throw new PermErrorException("MacroLetter not found: "+replaceValue);
352            }
353    
354            // Find number of domains to use
355            cellPattern = Pattern.compile("\\d+");
356            cellMatcher = cellPattern.matcher(replaceValue);
357            while (cellMatcher.find()) {
358                domainNumber = cellMatcher.group();
359                if (Integer.parseInt(domainNumber) == 0) {
360                    throw new PermErrorException(
361                            "Digit transformer must be non-zero");
362                }
363            }
364            // find if reversed
365            cellPattern = Pattern.compile("r");
366            cellMatcher = cellPattern.matcher(replaceValue);
367            while (cellMatcher.find()) {
368                isReversed = true;
369            }
370    
371            // find delimeters
372            cellPattern = Pattern.compile("[\\.\\-\\+\\,\\/\\_\\=]+");
373            cellMatcher = cellPattern.matcher(replaceValue);
374            while (cellMatcher.find()) {
375                delimeters = cellMatcher.group();
376            }
377    
378            // Reverse domains as necessary
379            ArrayList<String> data = split(variable, delimeters);
380            if (isReversed) {
381                data = reverse(data);
382            }
383    
384            // Truncate domain name to number of sub sections
385            String returnData;
386            if (!domainNumber.equals("")) {
387                returnData = subset(data, Integer.parseInt(domainNumber));
388            } else {
389                returnData = subset(data);
390            }
391    
392            return returnData;
393    
394        }
395    
396        /**
397         * Get the value for the given macro like descripted in the RFC
398         * 
399         * @param macro
400         *            The macro we want to get the value for
401         * @return rValue The value for the given macro
402         * @throws PermErrorException
403         *             Get thrown if the given variable is an unknown macro
404         * @throws RequireClientDomain requireClientDomain if the client domain is needed
405         *             and not yet resolved.
406         */
407        private String matchMacro(String macro, MacroData macroData) throws PermErrorException, RequireClientDomainException {
408    
409            String rValue = null;
410    
411            String variable = macro.toLowerCase();
412            if (variable.equalsIgnoreCase("i")) {
413                rValue = macroData.getMacroIpAddress();
414            } else if (variable.equalsIgnoreCase("s")) {
415                rValue = macroData.getMailFrom();
416            } else if (variable.equalsIgnoreCase("h")) {
417                rValue = macroData.getHostName();
418            } else if (variable.equalsIgnoreCase("l")) {
419                rValue = macroData.getCurrentSenderPart();
420            } else if (variable.equalsIgnoreCase("d")) {
421                rValue = macroData.getCurrentDomain();
422            } else if (variable.equalsIgnoreCase("v")) {
423                rValue = macroData.getInAddress();
424            } else if (variable.equalsIgnoreCase("t")) {
425                rValue = Long.toString(macroData.getTimeStamp());
426            } else if (variable.equalsIgnoreCase("c")) {
427                rValue = macroData.getReadableIP();
428            } else if (variable.equalsIgnoreCase("p")) {
429                rValue = macroData.getClientDomain();
430                if (rValue == null) {
431                    throw new RequireClientDomainException();
432                }
433            } else if (variable.equalsIgnoreCase("o")) {
434                rValue = macroData.getSenderDomain();
435            } else if (variable.equalsIgnoreCase("r")) {
436                rValue = macroData.getReceivingDomain();
437                if (rValue == null) {
438                    rValue = "unknown";
439                    List<String> dNames = dnsProbe.getLocalDomainNames();
440    
441                    for (int i = 0; i < dNames.size(); i++) {
442                        // check if the domainname is a FQDN
443                        if (SPF1Utils.checkFQDN(dNames.get(i).toString())) {
444                            rValue = dNames.get(i).toString();
445                            if (macroData instanceof SPFSession) {
446                                ((SPFSession) macroData).setReceivingDomain(rValue);
447                            }
448                            break;
449                        }
450                    }
451                }
452            }
453    
454            if (rValue == null) {
455                throw new PermErrorException("Unknown command : " + variable);
456    
457            } else {
458                log.debug("Used macro: " + macro + " replaced with: " + rValue);
459    
460                return rValue;
461            }
462        }
463    
464        /**
465         * Create an ArrayList by the given String. The String get splitted by given
466         * delimeters and one entry in the Array will be made for each splited
467         * String
468         * 
469         * @param data
470         *            The String we want to put in the Array
471         * @param delimeters
472         *            The delimeter we want to use to split the String
473         * @return ArrayList which contains the String parts
474         */
475        private ArrayList<String> split(String data, String delimeters) {
476    
477            String currentChar;
478            StringBuffer element = new StringBuffer();
479            ArrayList<String> splitParts = new ArrayList<String>();
480    
481            for (int i = 0; i < data.length(); i++) {
482                currentChar = data.substring(i, i + 1);
483                if (delimeters.indexOf(currentChar) > -1) {
484                    splitParts.add(element.toString());
485                    element.setLength(0);
486                } else {
487                    element.append(currentChar);
488                }
489            }
490            splitParts.add(element.toString());
491            return splitParts;
492        }
493    
494        /**
495         * Reverse an ArrayList
496         * 
497         * @param data
498         *            The ArrayList we want to get reversed
499         * @return reversed The reversed given ArrayList
500         */
501        private ArrayList<String> reverse(ArrayList<String> data) {
502    
503            ArrayList<String> reversed = new ArrayList<String>();
504            for (int i = 0; i < data.size(); i++) {
505                reversed.add(0, data.get(i));
506            }
507            return reversed;
508        }
509    
510        /**
511         * @see #subset(ArrayList, int)
512         */
513        private String subset(ArrayList<String> data) {
514            return subset(data, data.size());
515        }
516    
517        /**
518         * Convert a ArrayList to a String which holds the entries seperated by dots
519         * 
520         * @param data The ArrayList which should be converted
521         * @param length The ArrayLength
522         * @return A String which holds all entries seperated by dots
523         */
524        private String subset(ArrayList<String> data, int length) {
525    
526            StringBuffer buildString = new StringBuffer();
527            if (data.size() < length) {
528                length = data.size();
529            }
530            int start = data.size() - length;
531            for (int i = start; i < data.size(); i++) {
532                if (buildString.length() > 0) {
533                    buildString.append(".");
534                }
535                buildString.append(data.get(i));
536            }
537            return buildString.toString();
538    
539        }
540    
541        /**
542         * Encode the given URL to UTF-8
543         * 
544         * @param data
545         *            url to encode
546         * @return encoded URL
547         */
548        private String encodeURL(String data) {
549    
550            try {
551                // TODO URLEncoder method is not RFC2396 compatible, known
552                // difference
553                // is Space character gets converted to "+" rather than "%20"
554                // Is there anything else which is not correct with URLEncoder?
555                // Couldn't find a RFC2396 encoder
556                data = URLEncoder.encode(data, "UTF-8");
557            } catch (UnsupportedEncodingException e) {
558                // This shouldn't happen ignore it!
559            }
560    
561            // workaround for the above descripted problem
562            return data.replaceAll("\\+", "%20");
563    
564        }
565        
566        /**
567         * Because Dollar signs may be treated as references to captured subsequences in method Matcher.appendReplacement
568         * its necessary to escape Dollar signs because its allowed in the local-part of an emailaddress.
569         * 
570         * See JSPF-71 for the bugreport
571         * 
572         * @param raw
573         * @return escaped string
574         */
575        private String escapeForMatcher(String raw) {
576            StringBuffer sb = new StringBuffer();
577    
578            for (int i = 0; i < raw.length(); i++) {
579                char c = raw.charAt(i);
580                if (c == '$' || c == '\\') {
581                    sb.append('\\');
582                }
583                sb.append(c);
584            }
585            return sb.toString();
586        }
587    
588    }