001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.io.input;
018
019import java.io.BufferedInputStream;
020import java.io.BufferedReader;
021import java.io.File;
022import java.io.IOException;
023import java.io.InputStream;
024import java.io.InputStreamReader;
025import java.io.Reader;
026import java.io.StringReader;
027import java.net.HttpURLConnection;
028import java.net.URL;
029import java.net.URLConnection;
030import java.nio.charset.Charset;
031import java.nio.charset.StandardCharsets;
032import java.nio.file.Files;
033import java.nio.file.Path;
034import java.text.MessageFormat;
035import java.util.Locale;
036import java.util.Objects;
037import java.util.regex.Matcher;
038import java.util.regex.Pattern;
039
040import org.apache.commons.io.ByteOrderMark;
041import org.apache.commons.io.Charsets;
042import org.apache.commons.io.IOUtils;
043import org.apache.commons.io.build.AbstractStreamBuilder;
044import org.apache.commons.io.function.IOConsumer;
045import org.apache.commons.io.output.XmlStreamWriter;
046
047/**
048 * Character stream that handles all the necessary Voodoo to figure out the charset encoding of the XML document within the stream.
049 * <p>
050 * IMPORTANT: This class is not related in any way to the org.xml.sax.XMLReader. This one IS a character stream.
051 * </p>
052 * <p>
053 * All this has to be done without consuming characters from the stream, if not the XML parser will not recognized the document as a valid XML. This is not 100%
054 * true, but it's close enough (UTF-8 BOM is not handled by all parsers right now, XmlStreamReader handles it and things work in all parsers).
055 * </p>
056 * <p>
057 * The XmlStreamReader class handles the charset encoding of XML documents in Files, raw streams and HTTP streams by offering a wide set of constructors.
058 * </p>
059 * <p>
060 * By default the charset encoding detection is lenient, the constructor with the lenient flag can be used for a script (following HTTP MIME and XML
061 * specifications). All this is nicely explained by Mark Pilgrim in his blog, <a href="http://diveintomark.org/archives/2004/02/13/xml-media-types"> Determining
062 * the character encoding of a feed</a>.
063 * </p>
064 * <p>
065 * To build an instance, see {@link Builder}.
066 * </p>
067 * <p>
068 * Originally developed for <a href="http://rome.dev.java.net">ROME</a> under Apache License 2.0.
069 * </p>
070 *
071 * @see org.apache.commons.io.output.XmlStreamWriter
072 * @since 2.0
073 */
074public class XmlStreamReader extends Reader {
075
076    /**
077     * Builds a new {@link XmlStreamWriter} instance.
078     *
079     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
080     * <p>
081     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
082     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
083     * </p>
084     * <p>
085     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
086     * </p>
087     * <p>
088     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
089     * </p>
090     * <p>
091     * Else if the XML prolog had a charset encoding that encoding is used.
092     * </p>
093     * <p>
094     * Else if the content type had a charset encoding that encoding is used.
095     * </p>
096     * <p>
097     * Else 'UTF-8' is used.
098     * </p>
099     * <p>
100     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
101     * </p>
102     * <p>
103     * For example:
104     * </p>
105     *
106     * <pre>{@code
107     * XmlStreamReader r = XmlStreamReader.builder()
108     *   .setPath(path)
109     *   .setCharset(StandardCharsets.UTF_8)
110     *   .get();}
111     * </pre>
112     *
113     * @since 2.12.0
114     */
115    public static class Builder extends AbstractStreamBuilder<XmlStreamReader, Builder> {
116
117        private boolean nullCharset = true;
118        private boolean lenient = true;
119        private String httpContentType;
120
121        /**
122         * Constructs a new instance.
123         * <p>
124         * This builder use the aspect InputStream, OpenOption[], httpContentType, lenient, and defaultEncoding.
125         * </p>
126         * <p>
127         * You must provide an origin that can be converted to an InputStream by this builder, otherwise, this call will throw an
128         * {@link UnsupportedOperationException}.
129         * </p>
130         *
131         * @return a new instance.
132         * @throws UnsupportedOperationException if the origin cannot provide an InputStream.
133         * @throws IOException                   thrown if there is a problem reading the stream.
134         * @throws XmlStreamReaderException      thrown if the charset encoding could not be determined according to the specification.
135         * @see #getInputStream()
136         */
137        @SuppressWarnings("resource")
138        @Override
139        public XmlStreamReader get() throws IOException {
140            final String defaultEncoding = nullCharset ? null : getCharset().name();
141            // @formatter:off
142            return httpContentType == null
143                    ? new XmlStreamReader(getInputStream(), lenient, defaultEncoding)
144                    : new XmlStreamReader(getInputStream(), httpContentType, lenient, defaultEncoding);
145            // @formatter:on
146        }
147
148        @Override
149        public Builder setCharset(final Charset charset) {
150            nullCharset = charset == null;
151            return super.setCharset(charset);
152        }
153
154        @Override
155        public Builder setCharset(final String charset) {
156            nullCharset = charset == null;
157            return super.setCharset(Charsets.toCharset(charset, getCharsetDefault()));
158        }
159
160        /**
161         * Sets the HTTP content type.
162         *
163         * @param httpContentType the HTTP content type.
164         * @return this.
165         */
166        public Builder setHttpContentType(final String httpContentType) {
167            this.httpContentType = httpContentType;
168            return this;
169        }
170
171        /**
172         * Sets the lenient toggle.
173         *
174         * @param lenient the lenient toggle.
175         * @return this.
176         */
177        public Builder setLenient(final boolean lenient) {
178            this.lenient = lenient;
179            return this;
180        }
181
182    }
183
184    private static final String UTF_8 = StandardCharsets.UTF_8.name();
185
186    private static final String US_ASCII = StandardCharsets.US_ASCII.name();
187
188    private static final String UTF_16BE = StandardCharsets.UTF_16BE.name();
189
190    private static final String UTF_16LE = StandardCharsets.UTF_16LE.name();
191
192    private static final String UTF_32BE = "UTF-32BE";
193
194    private static final String UTF_32LE = "UTF-32LE";
195
196    private static final String UTF_16 = StandardCharsets.UTF_16.name();
197
198    private static final String UTF_32 = "UTF-32";
199
200    private static final String EBCDIC = "CP1047";
201
202    private static final ByteOrderMark[] BOMS = { ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_32BE,
203            ByteOrderMark.UTF_32LE };
204
205    /** UTF_16LE and UTF_32LE have the same two starting BOM bytes. */
206    private static final ByteOrderMark[] XML_GUESS_BYTES = { new ByteOrderMark(UTF_8, 0x3C, 0x3F, 0x78, 0x6D),
207            new ByteOrderMark(UTF_16BE, 0x00, 0x3C, 0x00, 0x3F), new ByteOrderMark(UTF_16LE, 0x3C, 0x00, 0x3F, 0x00),
208            new ByteOrderMark(UTF_32BE, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D),
209            new ByteOrderMark(UTF_32LE, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00),
210            new ByteOrderMark(EBCDIC, 0x4C, 0x6F, 0xA7, 0x94) };
211
212    private static final Pattern CHARSET_PATTERN = Pattern.compile("charset=[\"']?([.[^; \"']]*)[\"']?");
213
214    /**
215     * Pattern capturing the encoding of the "xml" processing instruction.
216     * <p>
217     * See also the <a href="https://www.w3.org/TR/2008/REC-xml-20081126/#NT-EncName">XML specification</a>.
218     * </p>
219     */
220    public static final Pattern ENCODING_PATTERN = Pattern.compile(
221    // @formatter:off
222            "^<\\?xml\\s+"
223            + "version\\s*=\\s*(?:(?:\"1\\.[0-9]+\")|(?:'1.[0-9]+'))\\s+"
224            + "encoding\\s*=\\s*"
225            + "((?:\"[A-Za-z0-9][A-Za-z0-9._+:-]*\")"  // double-quoted
226            +  "|(?:'[A-Za-z0-9][A-Za-z0-9._+:-]*'))", // single-quoted
227            Pattern.MULTILINE);
228    // N.B. the documented pattern is
229    // EncName   ::=   [A-Za-z] ([A-Za-z0-9._] | '-')*
230    // However this does not match all the aliases that are supported by Java.
231    // e.g.  '437', 'ISO_8859-1:1987' and 'ebcdic-de-273+euro'
232    // @formatter:on
233
234    private static final String RAW_EX_1 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] encoding mismatch";
235
236    private static final String RAW_EX_2 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] unknown BOM";
237
238    private static final String HTTP_EX_1 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], BOM must be null";
239
240    private static final String HTTP_EX_2 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], encoding mismatch";
241
242    private static final String HTTP_EX_3 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], Illegal MIME";
243
244    /**
245     * Constructs a new {@link Builder}.
246     *
247     * @return a new {@link Builder}.
248     * @since 2.12.0
249     */
250    public static Builder builder() {
251        return new Builder();
252    }
253
254    /**
255     * Gets the charset parameter value, {@code null} if not present, {@code null} if httpContentType is {@code null}.
256     *
257     * @param httpContentType the HTTP content type
258     * @return The content type encoding (upcased)
259     */
260    static String getContentTypeEncoding(final String httpContentType) {
261        String encoding = null;
262        if (httpContentType != null) {
263            final int i = httpContentType.indexOf(";");
264            if (i > -1) {
265                final String postMime = httpContentType.substring(i + 1);
266                final Matcher m = CHARSET_PATTERN.matcher(postMime);
267                encoding = m.find() ? m.group(1) : null;
268                encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null;
269            }
270        }
271        return encoding;
272    }
273
274    /**
275     * Gets the MIME type or {@code null} if httpContentType is {@code null}.
276     *
277     * @param httpContentType the HTTP content type
278     * @return The mime content type
279     */
280    static String getContentTypeMime(final String httpContentType) {
281        String mime = null;
282        if (httpContentType != null) {
283            final int i = httpContentType.indexOf(";");
284            if (i >= 0) {
285                mime = httpContentType.substring(0, i);
286            } else {
287                mime = httpContentType;
288            }
289            mime = mime.trim();
290        }
291        return mime;
292    }
293
294    /**
295     * Gets the encoding declared in the <?xml encoding=...?>, {@code null} if none.
296     *
297     * @param inputStream InputStream to create the reader from.
298     * @param guessedEnc  guessed encoding
299     * @return the encoding declared in the <?xml encoding=...?>
300     * @throws IOException thrown if there is a problem reading the stream.
301     */
302    private static String getXmlProlog(final InputStream inputStream, final String guessedEnc) throws IOException {
303        String encoding = null;
304        if (guessedEnc != null) {
305            final byte[] bytes = IOUtils.byteArray();
306            inputStream.mark(IOUtils.DEFAULT_BUFFER_SIZE);
307            int offset = 0;
308            int max = IOUtils.DEFAULT_BUFFER_SIZE;
309            int c = inputStream.read(bytes, offset, max);
310            int firstGT = -1;
311            String xmlProlog = ""; // avoid possible NPE warning (cannot happen; this just silences the warning)
312            while (c != -1 && firstGT == -1 && offset < IOUtils.DEFAULT_BUFFER_SIZE) {
313                offset += c;
314                max -= c;
315                c = inputStream.read(bytes, offset, max);
316                xmlProlog = new String(bytes, 0, offset, guessedEnc);
317                firstGT = xmlProlog.indexOf('>');
318            }
319            if (firstGT == -1) {
320                if (c == -1) {
321                    throw new IOException("Unexpected end of XML stream");
322                }
323                throw new IOException("XML prolog or ROOT element not found on first " + offset + " bytes");
324            }
325            final int bytesRead = offset;
326            if (bytesRead > 0) {
327                inputStream.reset();
328                final BufferedReader bReader = new BufferedReader(new StringReader(xmlProlog.substring(0, firstGT + 1)));
329                final StringBuilder prolog = new StringBuilder();
330                IOConsumer.forEach(bReader.lines(), prolog::append);
331                final Matcher m = ENCODING_PATTERN.matcher(prolog);
332                if (m.find()) {
333                    encoding = m.group(1).toUpperCase(Locale.ROOT);
334                    encoding = encoding.substring(1, encoding.length() - 1);
335                }
336            }
337        }
338        return encoding;
339    }
340
341    /**
342     * Tests if the MIME type belongs to the APPLICATION XML family.
343     *
344     * @param mime The mime type
345     * @return true if the mime type belongs to the APPLICATION XML family, otherwise false
346     */
347    static boolean isAppXml(final String mime) {
348        return mime != null && (mime.equals("application/xml") || mime.equals("application/xml-dtd") || mime.equals("application/xml-external-parsed-entity")
349                || mime.startsWith("application/") && mime.endsWith("+xml"));
350    }
351
352    /**
353     * Tests if the MIME type belongs to the TEXT XML family.
354     *
355     * @param mime The mime type
356     * @return true if the mime type belongs to the TEXT XML family, otherwise false
357     */
358    static boolean isTextXml(final String mime) {
359        return mime != null && (mime.equals("text/xml") || mime.equals("text/xml-external-parsed-entity") || mime.startsWith("text/") && mime.endsWith("+xml"));
360    }
361
362    private final Reader reader;
363
364    private final String encoding;
365
366    private final String defaultEncoding;
367
368    /**
369     * Constructs a Reader for a File.
370     * <p>
371     * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8.
372     * </p>
373     * <p>
374     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
375     * </p>
376     *
377     * @param file File to create a Reader from.
378     * @throws NullPointerException if the input is {@code null}.
379     * @throws IOException thrown if there is a problem reading the file.
380     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
381     */
382    @Deprecated
383    public XmlStreamReader(final File file) throws IOException {
384        this(Objects.requireNonNull(file, "file").toPath());
385    }
386
387    /**
388     * Constructs a Reader for a raw InputStream.
389     * <p>
390     * It follows the same logic used for files.
391     * </p>
392     * <p>
393     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
394     * </p>
395     *
396     * @param inputStream InputStream to create a Reader from.
397     * @throws NullPointerException if the input stream is {@code null}.
398     * @throws IOException thrown if there is a problem reading the stream.
399     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
400     */
401    @Deprecated
402    public XmlStreamReader(final InputStream inputStream) throws IOException {
403        this(inputStream, true);
404    }
405
406    /**
407     * Constructs a Reader for a raw InputStream.
408     * <p>
409     * It follows the same logic used for files.
410     * </p>
411     * <p>
412     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
413     * </p>
414     * <p>
415     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
416     * </p>
417     * <p>
418     * Else if the XML prolog had a charset encoding that encoding is used.
419     * </p>
420     * <p>
421     * Else if the content type had a charset encoding that encoding is used.
422     * </p>
423     * <p>
424     * Else 'UTF-8' is used.
425     * </p>
426     * <p>
427     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
428     * </p>
429     *
430     * @param inputStream InputStream to create a Reader from.
431     * @param lenient     indicates if the charset encoding detection should be relaxed.
432     * @throws NullPointerException if the input stream is {@code null}.
433     * @throws IOException              thrown if there is a problem reading the stream.
434     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
435     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
436     */
437    @Deprecated
438    public XmlStreamReader(final InputStream inputStream, final boolean lenient) throws IOException {
439        this(inputStream, lenient, null);
440    }
441
442    /**
443     * Constructs a Reader for a raw InputStream.
444     * <p>
445     * It follows the same logic used for files.
446     * </p>
447     * <p>
448     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
449     * </p>
450     * <p>
451     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
452     * </p>
453     * <p>
454     * Else if the XML prolog had a charset encoding that encoding is used.
455     * </p>
456     * <p>
457     * Else if the content type had a charset encoding that encoding is used.
458     * </p>
459     * <p>
460     * Else 'UTF-8' is used.
461     * </p>
462     * <p>
463     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
464     * </p>
465     *
466     * @param inputStream     InputStream to create a Reader from.
467     * @param lenient         indicates if the charset encoding detection should be relaxed.
468     * @param defaultEncoding The default encoding
469     * @throws NullPointerException if the input stream is {@code null}.
470     * @throws IOException              thrown if there is a problem reading the stream.
471     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
472     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
473     */
474    @Deprecated
475    @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance.
476    public XmlStreamReader(final InputStream inputStream, final boolean lenient, final String defaultEncoding) throws IOException {
477        Objects.requireNonNull(inputStream, "inputStream");
478        this.defaultEncoding = defaultEncoding;
479        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE), false, BOMS);
480        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
481        this.encoding = doRawStream(bom, pis, lenient);
482        this.reader = new InputStreamReader(pis, encoding);
483    }
484
485    /**
486     * Constructs a Reader using an InputStream and the associated content-type header.
487     * <p>
488     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
489     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
490     * </p>
491     * <p>
492     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
493     * </p>
494     *
495     * @param inputStream     InputStream to create the reader from.
496     * @param httpContentType content-type header to use for the resolution of the charset encoding.
497     * @throws NullPointerException if the input stream is {@code null}.
498     * @throws IOException thrown if there is a problem reading the file.
499     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
500     */
501    @Deprecated
502    public XmlStreamReader(final InputStream inputStream, final String httpContentType) throws IOException {
503        this(inputStream, httpContentType, true);
504    }
505
506    /**
507     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
508     * <p>
509     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
510     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
511     * </p>
512     * <p>
513     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
514     * </p>
515     * <p>
516     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
517     * </p>
518     * <p>
519     * Else if the XML prolog had a charset encoding that encoding is used.
520     * </p>
521     * <p>
522     * Else if the content type had a charset encoding that encoding is used.
523     * </p>
524     * <p>
525     * Else 'UTF-8' is used.
526     * </p>
527     * <p>
528     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
529     * </p>
530     *
531     * @param inputStream     InputStream to create the reader from.
532     * @param httpContentType content-type header to use for the resolution of the charset encoding.
533     * @param lenient         indicates if the charset encoding detection should be relaxed.
534     * @throws NullPointerException if the input stream is {@code null}.
535     * @throws IOException              thrown if there is a problem reading the file.
536     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
537     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
538     */
539    @Deprecated
540    public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient) throws IOException {
541        this(inputStream, httpContentType, lenient, null);
542    }
543
544    /**
545     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
546     * <p>
547     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
548     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
549     * </p>
550     * <p>
551     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
552     * </p>
553     * <p>
554     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
555     * </p>
556     * <p>
557     * Else if the XML prolog had a charset encoding that encoding is used.
558     * </p>
559     * <p>
560     * Else if the content type had a charset encoding that encoding is used.
561     * </p>
562     * <p>
563     * Else 'UTF-8' is used.
564     * </p>
565     * <p>
566     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
567     * </p>
568     *
569     * @param inputStream     InputStream to create the reader from.
570     * @param httpContentType content-type header to use for the resolution of the charset encoding.
571     * @param lenient         indicates if the charset encoding detection should be relaxed.
572     * @param defaultEncoding The default encoding
573     * @throws NullPointerException if the input stream is {@code null}.
574     * @throws IOException              thrown if there is a problem reading the file.
575     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
576     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
577     */
578    @Deprecated
579    @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance.
580    public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient, final String defaultEncoding)
581            throws IOException {
582        Objects.requireNonNull(inputStream, "inputStream");
583        this.defaultEncoding = defaultEncoding;
584        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE), false, BOMS);
585        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
586        this.encoding = processHttpStream(bom, pis, httpContentType, lenient);
587        this.reader = new InputStreamReader(pis, encoding);
588    }
589
590    /**
591     * Constructs a Reader for a File.
592     * <p>
593     * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8.
594     * </p>
595     * <p>
596     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
597     * </p>
598     *
599     * @param file File to create a Reader from.
600     * @throws NullPointerException if the input is {@code null}.
601     * @throws IOException thrown if there is a problem reading the file.
602     * @since 2.11.0
603     * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()}
604     */
605    @Deprecated
606    @SuppressWarnings("resource") // InputStream is managed through another reader in this instance.
607    public XmlStreamReader(final Path file) throws IOException {
608        this(Files.newInputStream(Objects.requireNonNull(file, "file")));
609    }
610
611    /**
612     * Constructs a Reader using the InputStream of a URL.
613     * <p>
614     * If the URL is not of type HTTP and there is not 'content-type' header in the fetched data it uses the same logic used for Files.
615     * </p>
616     * <p>
617     * If the URL is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with content-type.
618     * </p>
619     * <p>
620     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
621     * </p>
622     *
623     * @param url URL to create a Reader from.
624     * @throws NullPointerException if the input is {@code null}.
625     * @throws IOException thrown if there is a problem reading the stream of the URL.
626     */
627    public XmlStreamReader(final URL url) throws IOException {
628        this(Objects.requireNonNull(url, "url").openConnection(), null);
629    }
630
631    /**
632     * Constructs a Reader using the InputStream of a URLConnection.
633     * <p>
634     * If the URLConnection is not of type HttpURLConnection and there is not 'content-type' header in the fetched data it uses the same logic used for files.
635     * </p>
636     * <p>
637     * If the URLConnection is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with
638     * content-type.
639     * </p>
640     * <p>
641     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
642     * </p>
643     *
644     * @param urlConnection   URLConnection to create a Reader from.
645     * @param defaultEncoding The default encoding
646     * @throws NullPointerException if the input is {@code null}.
647     * @throws IOException thrown if there is a problem reading the stream of the URLConnection.
648     */
649    public XmlStreamReader(final URLConnection urlConnection, final String defaultEncoding) throws IOException {
650        Objects.requireNonNull(urlConnection, "urlConnection");
651        this.defaultEncoding = defaultEncoding;
652        final boolean lenient = true;
653        final String contentType = urlConnection.getContentType();
654        final InputStream inputStream = urlConnection.getInputStream();
655        @SuppressWarnings("resource") // managed by the InputStreamReader tracked by this instance
656        // @formatter:off
657        final BOMInputStream bomInput = BOMInputStream.builder()
658            .setInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE))
659            .setInclude(false)
660            .setByteOrderMarks(BOMS)
661            .get();
662        @SuppressWarnings("resource")
663        final BOMInputStream piInput = BOMInputStream.builder()
664            .setInputStream(new BufferedInputStream(bomInput, IOUtils.DEFAULT_BUFFER_SIZE))
665            .setInclude(true)
666            .setByteOrderMarks(XML_GUESS_BYTES)
667            .get();
668        // @formatter:on
669        if (urlConnection instanceof HttpURLConnection || contentType != null) {
670            this.encoding = processHttpStream(bomInput, piInput, contentType, lenient);
671        } else {
672            this.encoding = doRawStream(bomInput, piInput, lenient);
673        }
674        this.reader = new InputStreamReader(piInput, encoding);
675    }
676
677    /**
678     * Calculates the HTTP encoding.
679     *
680     * @param httpContentType The HTTP content type
681     * @param bomEnc          BOM encoding
682     * @param xmlGuessEnc     XML Guess encoding
683     * @param xmlEnc          XML encoding
684     * @param lenient         indicates if the charset encoding detection should be relaxed.
685     * @return the HTTP encoding
686     * @throws IOException thrown if there is a problem reading the stream.
687     */
688    String calculateHttpEncoding(final String httpContentType, final String bomEnc, final String xmlGuessEnc, final String xmlEnc, final boolean lenient)
689            throws IOException {
690
691        // Lenient and has XML encoding
692        if (lenient && xmlEnc != null) {
693            return xmlEnc;
694        }
695
696        // Determine mime/encoding content types from HTTP Content Type
697        final String cTMime = getContentTypeMime(httpContentType);
698        final String cTEnc = getContentTypeEncoding(httpContentType);
699        final boolean appXml = isAppXml(cTMime);
700        final boolean textXml = isTextXml(cTMime);
701
702        // Mime type NOT "application/xml" or "text/xml"
703        if (!appXml && !textXml) {
704            final String msg = MessageFormat.format(HTTP_EX_3, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
705            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
706        }
707
708        // No content type encoding
709        if (cTEnc == null) {
710            if (appXml) {
711                return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
712            }
713            return defaultEncoding == null ? US_ASCII : defaultEncoding;
714        }
715
716        // UTF-16BE or UTF-16LE content type encoding
717        if (cTEnc.equals(UTF_16BE) || cTEnc.equals(UTF_16LE)) {
718            if (bomEnc != null) {
719                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
720                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
721            }
722            return cTEnc;
723        }
724
725        // UTF-16 content type encoding
726        if (cTEnc.equals(UTF_16)) {
727            if (bomEnc != null && bomEnc.startsWith(UTF_16)) {
728                return bomEnc;
729            }
730            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
731            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
732        }
733
734        // UTF-32BE or UTF-132E content type encoding
735        if (cTEnc.equals(UTF_32BE) || cTEnc.equals(UTF_32LE)) {
736            if (bomEnc != null) {
737                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
738                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
739            }
740            return cTEnc;
741        }
742
743        // UTF-32 content type encoding
744        if (cTEnc.equals(UTF_32)) {
745            if (bomEnc != null && bomEnc.startsWith(UTF_32)) {
746                return bomEnc;
747            }
748            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
749            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
750        }
751
752        return cTEnc;
753    }
754
755    /**
756     * Calculate the raw encoding.
757     *
758     * @param bomEnc      BOM encoding
759     * @param xmlGuessEnc XML Guess encoding
760     * @param xmlEnc      XML encoding
761     * @return the raw encoding
762     * @throws IOException thrown if there is a problem reading the stream.
763     */
764    String calculateRawEncoding(final String bomEnc, final String xmlGuessEnc, final String xmlEnc) throws IOException {
765
766        // BOM is Null
767        if (bomEnc == null) {
768            if (xmlGuessEnc == null || xmlEnc == null) {
769                return defaultEncoding == null ? UTF_8 : defaultEncoding;
770            }
771            if (xmlEnc.equals(UTF_16) && (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc.equals(UTF_16LE))) {
772                return xmlGuessEnc;
773            }
774            return xmlEnc;
775        }
776
777        // BOM is UTF-8
778        if (bomEnc.equals(UTF_8)) {
779            if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) {
780                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
781                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
782            }
783            if (xmlEnc != null && !xmlEnc.equals(UTF_8)) {
784                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
785                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
786            }
787            return bomEnc;
788        }
789
790        // BOM is UTF-16BE or UTF-16LE
791        if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) {
792            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
793                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
794                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
795            }
796            if (xmlEnc != null && !xmlEnc.equals(UTF_16) && !xmlEnc.equals(bomEnc)) {
797                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
798                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
799            }
800            return bomEnc;
801        }
802
803        // BOM is UTF-32BE or UTF-32LE
804        if (bomEnc.equals(UTF_32BE) || bomEnc.equals(UTF_32LE)) {
805            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
806                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
807                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
808            }
809            if (xmlEnc != null && !xmlEnc.equals(UTF_32) && !xmlEnc.equals(bomEnc)) {
810                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
811                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
812            }
813            return bomEnc;
814        }
815
816        // BOM is something else
817        final String msg = MessageFormat.format(RAW_EX_2, bomEnc, xmlGuessEnc, xmlEnc);
818        throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
819    }
820
821    /**
822     * Closes the XmlStreamReader stream.
823     *
824     * @throws IOException thrown if there was a problem closing the stream.
825     */
826    @Override
827    public void close() throws IOException {
828        reader.close();
829    }
830
831    /**
832     * Does lenient detection.
833     *
834     * @param httpContentType content-type header to use for the resolution of the charset encoding.
835     * @param ex              The thrown exception
836     * @return the encoding
837     * @throws IOException thrown if there is a problem reading the stream.
838     */
839    private String doLenientDetection(String httpContentType, XmlStreamReaderException ex) throws IOException {
840        if (httpContentType != null && httpContentType.startsWith("text/html")) {
841            httpContentType = httpContentType.substring("text/html".length());
842            httpContentType = "text/xml" + httpContentType;
843            try {
844                return calculateHttpEncoding(httpContentType, ex.getBomEncoding(), ex.getXmlGuessEncoding(), ex.getXmlEncoding(), true);
845            } catch (final XmlStreamReaderException ex2) {
846                ex = ex2;
847            }
848        }
849        String encoding = ex.getXmlEncoding();
850        if (encoding == null) {
851            encoding = ex.getContentTypeEncoding();
852        }
853        if (encoding == null) {
854            encoding = defaultEncoding == null ? UTF_8 : defaultEncoding;
855        }
856        return encoding;
857    }
858
859    /**
860     * Process the raw stream.
861     *
862     * @param bom     BOMInputStream to detect byte order marks
863     * @param pis     BOMInputStream to guess XML encoding
864     * @param lenient indicates if the charset encoding detection should be relaxed.
865     * @return the encoding to be used
866     * @throws IOException thrown if there is a problem reading the stream.
867     */
868    private String doRawStream(final BOMInputStream bom, final BOMInputStream pis, final boolean lenient) throws IOException {
869        final String bomEnc = bom.getBOMCharsetName();
870        final String xmlGuessEnc = pis.getBOMCharsetName();
871        final String xmlEnc = getXmlProlog(pis, xmlGuessEnc);
872        try {
873            return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
874        } catch (final XmlStreamReaderException ex) {
875            if (lenient) {
876                return doLenientDetection(null, ex);
877            }
878            throw ex;
879        }
880    }
881
882    /**
883     * Gets the default encoding to use if none is set in HTTP content-type, XML prolog and the rules based on content-type are not adequate.
884     * <p>
885     * If it is {@code null} the content-type based rules are used.
886     * </p>
887     *
888     * @return the default encoding to use.
889     */
890    public String getDefaultEncoding() {
891        return defaultEncoding;
892    }
893
894    /**
895     * Gets the charset encoding of the XmlStreamReader.
896     *
897     * @return charset encoding.
898     */
899    public String getEncoding() {
900        return encoding;
901    }
902
903    /**
904     * Processes an HTTP stream.
905     *
906     * @param bomInput        BOMInputStream to detect byte order marks
907     * @param piInput         BOMInputStream to guess XML encoding
908     * @param httpContentType The HTTP content type
909     * @param lenient         indicates if the charset encoding detection should be relaxed.
910     * @return the encoding to be used
911     * @throws IOException thrown if there is a problem reading the stream.
912     */
913    private String processHttpStream(final BOMInputStream bomInput, final BOMInputStream piInput, final String httpContentType, final boolean lenient)
914            throws IOException {
915        final String bomEnc = bomInput.getBOMCharsetName();
916        final String xmlGuessEnc = piInput.getBOMCharsetName();
917        final String xmlEnc = getXmlProlog(piInput, xmlGuessEnc);
918        try {
919            return calculateHttpEncoding(httpContentType, bomEnc, xmlGuessEnc, xmlEnc, lenient);
920        } catch (final XmlStreamReaderException ex) {
921            if (lenient) {
922                return doLenientDetection(httpContentType, ex);
923            }
924            throw ex;
925        }
926    }
927
928    /**
929     * Reads the underlying reader's {@code read(char[], int, int)} method.
930     *
931     * @param buf    the buffer to read the characters into
932     * @param offset The start offset
933     * @param len    The number of bytes to read
934     * @return the number of characters read or -1 if the end of stream
935     * @throws IOException if an I/O error occurs.
936     */
937    @Override
938    public int read(final char[] buf, final int offset, final int len) throws IOException {
939        return reader.read(buf, offset, len);
940    }
941
942}