001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.commons.io.input;
018
019import java.io.BufferedInputStream;
020import java.io.BufferedReader;
021import java.io.File;
022import java.io.IOException;
023import java.io.InputStream;
024import java.io.InputStreamReader;
025import java.io.Reader;
026import java.io.StringReader;
027import java.net.HttpURLConnection;
028import java.net.URL;
029import java.net.URLConnection;
030import java.nio.charset.Charset;
031import java.nio.charset.StandardCharsets;
032import java.nio.file.Files;
033import java.nio.file.Path;
034import java.text.MessageFormat;
035import java.util.Locale;
036import java.util.Objects;
037import java.util.regex.Matcher;
038import java.util.regex.Pattern;
039
040import org.apache.commons.io.ByteOrderMark;
041import org.apache.commons.io.Charsets;
042import org.apache.commons.io.IOUtils;
043import org.apache.commons.io.build.AbstractStreamBuilder;
044import org.apache.commons.io.function.IOConsumer;
045import org.apache.commons.io.output.XmlStreamWriter;
046
047/**
048 * Character stream that handles all the necessary Voodoo to figure out the charset encoding of the XML document within the stream.
049 * <p>
050 * IMPORTANT: This class is not related in any way to the org.xml.sax.XMLReader. This one IS a character stream.
051 * </p>
052 * <p>
053 * All this has to be done without consuming characters from the stream, if not the XML parser will not recognized the document as a valid XML. This is not 100%
054 * true, but it's close enough (UTF-8 BOM is not handled by all parsers right now, XmlStreamReader handles it and things work in all parsers).
055 * </p>
056 * <p>
057 * The XmlStreamReader class handles the charset encoding of XML documents in Files, raw streams and HTTP streams by offering a wide set of constructors.
058 * </p>
059 * <p>
060 * By default the charset encoding detection is lenient, the constructor with the lenient flag can be used for a script (following HTTP MIME and XML
061 * specifications). All this is nicely explained by Mark Pilgrim in his blog, <a href="http://diveintomark.org/archives/2004/02/13/xml-media-types"> Determining
062 * the character encoding of a feed</a>.
063 * </p>
064 * <p>
065 * Originally developed for <a href="http://rome.dev.java.net">ROME</a> under Apache License 2.0.
066 * </p>
067 *
068 * @see org.apache.commons.io.output.XmlStreamWriter
069 * @since 2.0
070 */
071public class XmlStreamReader extends Reader {
072
073    /**
074     * Builds a new {@link XmlStreamWriter} instance.
075     *
076     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
077     * <p>
078     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
079     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
080     * </p>
081     * <p>
082     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
083     * </p>
084     * <p>
085     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
086     * </p>
087     * <p>
088     * Else if the XML prolog had a charset encoding that encoding is used.
089     * </p>
090     * <p>
091     * Else if the content type had a charset encoding that encoding is used.
092     * </p>
093     * <p>
094     * Else 'UTF-8' is used.
095     * </p>
096     * <p>
097     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
098     * </p>
099     * <p>
100     * For example:
101     * </p>
102     *
103     * <pre>{@code
104     * XmlStreamReader r = XmlStreamReader.builder()
105     *   .setPath(path)
106     *   .setCharset(StandardCharsets.UTF_8)
107     *   .get()}
108     * </pre>
109     * <p>
110     *
111     * @since 2.12.0
112     */
113    public static class Builder extends AbstractStreamBuilder<XmlStreamReader, Builder> {
114
115        private boolean nullCharset = true;
116        private boolean lenient = true;
117        private String httpContentType;
118
119        /**
120         * Constructs a new instance.
121         *
122         * @throws UnsupportedOperationException if the origin cannot be converted to an InputStream.
123         */
124        @SuppressWarnings("resource")
125        @Override
126        public XmlStreamReader get() throws IOException {
127            final String defaultEncoding = nullCharset ? null : getCharset().name();
128            // @formatter:off
129            return httpContentType == null
130                    ? new XmlStreamReader(getOrigin().getInputStream(), lenient, defaultEncoding)
131                    : new XmlStreamReader(getOrigin().getInputStream(), httpContentType, lenient, defaultEncoding);
132            // @formatter:on
133        }
134
135        @Override
136        public Builder setCharset(final Charset charset) {
137            nullCharset = charset == null;
138            return super.setCharset(charset);
139        }
140
141        @Override
142        public Builder setCharset(final String charset) {
143            nullCharset = charset == null;
144            return super.setCharset(Charsets.toCharset(charset, getCharsetDefault()));
145        }
146
147        public Builder setHttpContentType(final String httpContentType) {
148            this.httpContentType = httpContentType;
149            return this;
150        }
151
152        public Builder setLenient(final boolean lenient) {
153            this.lenient = lenient;
154            return this;
155        }
156
157    }
158
159    private static final String UTF_8 = StandardCharsets.UTF_8.name();
160
161    private static final String US_ASCII = StandardCharsets.US_ASCII.name();
162
163    private static final String UTF_16BE = StandardCharsets.UTF_16BE.name();
164
165    private static final String UTF_16LE = StandardCharsets.UTF_16LE.name();
166
167    private static final String UTF_32BE = "UTF-32BE";
168
169    private static final String UTF_32LE = "UTF-32LE";
170
171    private static final String UTF_16 = StandardCharsets.UTF_16.name();
172
173    private static final String UTF_32 = "UTF-32";
174
175    private static final String EBCDIC = "CP1047";
176
177    private static final ByteOrderMark[] BOMS = { ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_32BE,
178            ByteOrderMark.UTF_32LE };
179
180    /** UTF_16LE and UTF_32LE have the same two starting BOM bytes. */
181    private static final ByteOrderMark[] XML_GUESS_BYTES = { new ByteOrderMark(UTF_8, 0x3C, 0x3F, 0x78, 0x6D),
182            new ByteOrderMark(UTF_16BE, 0x00, 0x3C, 0x00, 0x3F), new ByteOrderMark(UTF_16LE, 0x3C, 0x00, 0x3F, 0x00),
183            new ByteOrderMark(UTF_32BE, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D),
184            new ByteOrderMark(UTF_32LE, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00),
185            new ByteOrderMark(EBCDIC, 0x4C, 0x6F, 0xA7, 0x94) };
186
187    private static final Pattern CHARSET_PATTERN = Pattern.compile("charset=[\"']?([.[^; \"']]*)[\"']?");
188
189    /**
190     * Pattern capturing the encoding of the "xml" processing instruction.
191     */
192    public static final Pattern ENCODING_PATTERN = Pattern.compile("<\\?xml.*encoding[\\s]*=[\\s]*((?:\".[^\"]*\")|(?:'.[^']*'))", Pattern.MULTILINE);
193
194    private static final String RAW_EX_1 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] encoding mismatch";
195
196    private static final String RAW_EX_2 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] unknown BOM";
197
198    private static final String HTTP_EX_1 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], BOM must be NULL";
199
200    private static final String HTTP_EX_2 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], encoding mismatch";
201
202    private static final String HTTP_EX_3 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], Illegal MIME";
203
204    /**
205     * Constructs a new {@link Builder}.
206     *
207     * @return a new {@link Builder}.
208     * @since 2.12.0
209     */
210    public static Builder builder() {
211        return new Builder();
212    }
213
214    /**
215     * Gets the charset parameter value, NULL if not present, NULL if httpContentType is NULL.
216     *
217     * @param httpContentType the HTTP content type
218     * @return The content type encoding (upcased)
219     */
220    static String getContentTypeEncoding(final String httpContentType) {
221        String encoding = null;
222        if (httpContentType != null) {
223            final int i = httpContentType.indexOf(";");
224            if (i > -1) {
225                final String postMime = httpContentType.substring(i + 1);
226                final Matcher m = CHARSET_PATTERN.matcher(postMime);
227                encoding = m.find() ? m.group(1) : null;
228                encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null;
229            }
230        }
231        return encoding;
232    }
233
234    /**
235     * Gets the MIME type or NULL if httpContentType is NULL.
236     *
237     * @param httpContentType the HTTP content type
238     * @return The mime content type
239     */
240    static String getContentTypeMime(final String httpContentType) {
241        String mime = null;
242        if (httpContentType != null) {
243            final int i = httpContentType.indexOf(";");
244            if (i >= 0) {
245                mime = httpContentType.substring(0, i);
246            } else {
247                mime = httpContentType;
248            }
249            mime = mime.trim();
250        }
251        return mime;
252    }
253
254    /**
255     * Gets the encoding declared in the <?xml encoding=...?>, NULL if none.
256     *
257     * @param inputStream InputStream to create the reader from.
258     * @param guessedEnc  guessed encoding
259     * @return the encoding declared in the <?xml encoding=...?>
260     * @throws IOException thrown if there is a problem reading the stream.
261     */
262    private static String getXmlProlog(final InputStream inputStream, final String guessedEnc) throws IOException {
263        String encoding = null;
264        if (guessedEnc != null) {
265            final byte[] bytes = IOUtils.byteArray();
266            inputStream.mark(IOUtils.DEFAULT_BUFFER_SIZE);
267            int offset = 0;
268            int max = IOUtils.DEFAULT_BUFFER_SIZE;
269            int c = inputStream.read(bytes, offset, max);
270            int firstGT = -1;
271            String xmlProlog = ""; // avoid possible NPE warning (cannot happen; this just silences the warning)
272            while (c != -1 && firstGT == -1 && offset < IOUtils.DEFAULT_BUFFER_SIZE) {
273                offset += c;
274                max -= c;
275                c = inputStream.read(bytes, offset, max);
276                xmlProlog = new String(bytes, 0, offset, guessedEnc);
277                firstGT = xmlProlog.indexOf('>');
278            }
279            if (firstGT == -1) {
280                if (c == -1) {
281                    throw new IOException("Unexpected end of XML stream");
282                }
283                throw new IOException("XML prolog or ROOT element not found on first " + offset + " bytes");
284            }
285            final int bytesRead = offset;
286            if (bytesRead > 0) {
287                inputStream.reset();
288                final BufferedReader bReader = new BufferedReader(new StringReader(xmlProlog.substring(0, firstGT + 1)));
289                final StringBuilder prolog = new StringBuilder();
290                IOConsumer.forEach(bReader.lines(), prolog::append);
291                final Matcher m = ENCODING_PATTERN.matcher(prolog);
292                if (m.find()) {
293                    encoding = m.group(1).toUpperCase(Locale.ROOT);
294                    encoding = encoding.substring(1, encoding.length() - 1);
295                }
296            }
297        }
298        return encoding;
299    }
300
301    /**
302     * Tests if the MIME type belongs to the APPLICATION XML family.
303     *
304     * @param mime The mime type
305     * @return true if the mime type belongs to the APPLICATION XML family, otherwise false
306     */
307    static boolean isAppXml(final String mime) {
308        return mime != null && (mime.equals("application/xml") || mime.equals("application/xml-dtd") || mime.equals("application/xml-external-parsed-entity")
309                || mime.startsWith("application/") && mime.endsWith("+xml"));
310    }
311
312    /**
313     * Tests if the MIME type belongs to the TEXT XML family.
314     *
315     * @param mime The mime type
316     * @return true if the mime type belongs to the TEXT XML family, otherwise false
317     */
318    static boolean isTextXml(final String mime) {
319        return mime != null && (mime.equals("text/xml") || mime.equals("text/xml-external-parsed-entity") || mime.startsWith("text/") && mime.endsWith("+xml"));
320    }
321
322    private final Reader reader;
323
324    private final String encoding;
325
326    private final String defaultEncoding;
327
328    /**
329     * Constructs a Reader for a File.
330     * <p>
331     * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8.
332     * </p>
333     * <p>
334     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
335     * </p>
336     *
337     * @param file File to create a Reader from.
338     * @throws NullPointerException if the input is {@code null}.
339     * @throws IOException thrown if there is a problem reading the file.
340     * @deprecated Use {@link #builder()}
341     */
342    @Deprecated
343    public XmlStreamReader(final File file) throws IOException {
344        this(Objects.requireNonNull(file, "file").toPath());
345    }
346
347    /**
348     * Constructs a Reader for a raw InputStream.
349     * <p>
350     * It follows the same logic used for files.
351     * </p>
352     * <p>
353     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
354     * </p>
355     *
356     * @param inputStream InputStream to create a Reader from.
357     * @throws NullPointerException if the input stream is {@code null}.
358     * @throws IOException thrown if there is a problem reading the stream.
359     * @deprecated Use {@link #builder()}
360     */
361    @Deprecated
362    public XmlStreamReader(final InputStream inputStream) throws IOException {
363        this(inputStream, true);
364    }
365
366    /**
367     * Constructs a Reader for a raw InputStream.
368     * <p>
369     * It follows the same logic used for files.
370     * </p>
371     * <p>
372     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
373     * </p>
374     * <p>
375     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
376     * </p>
377     * <p>
378     * Else if the XML prolog had a charset encoding that encoding is used.
379     * </p>
380     * <p>
381     * Else if the content type had a charset encoding that encoding is used.
382     * </p>
383     * <p>
384     * Else 'UTF-8' is used.
385     * </p>
386     * <p>
387     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
388     * </p>
389     *
390     * @param inputStream InputStream to create a Reader from.
391     * @param lenient     indicates if the charset encoding detection should be relaxed.
392     * @throws NullPointerException if the input stream is {@code null}.
393     * @throws IOException              thrown if there is a problem reading the stream.
394     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specs.
395     * @deprecated Use {@link #builder()}
396     */
397    @Deprecated
398    public XmlStreamReader(final InputStream inputStream, final boolean lenient) throws IOException {
399        this(inputStream, lenient, null);
400    }
401
402    /**
403     * Constructs a Reader for a raw InputStream.
404     * <p>
405     * It follows the same logic used for files.
406     * </p>
407     * <p>
408     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
409     * </p>
410     * <p>
411     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
412     * </p>
413     * <p>
414     * Else if the XML prolog had a charset encoding that encoding is used.
415     * </p>
416     * <p>
417     * Else if the content type had a charset encoding that encoding is used.
418     * </p>
419     * <p>
420     * Else 'UTF-8' is used.
421     * </p>
422     * <p>
423     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
424     * </p>
425     *
426     * @param inputStream     InputStream to create a Reader from.
427     * @param lenient         indicates if the charset encoding detection should be relaxed.
428     * @param defaultEncoding The default encoding
429     * @throws NullPointerException if the input stream is {@code null}.
430     * @throws IOException              thrown if there is a problem reading the stream.
431     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specs.
432     * @deprecated Use {@link #builder()}
433     */
434    @Deprecated
435    @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance.
436    public XmlStreamReader(final InputStream inputStream, final boolean lenient, final String defaultEncoding) throws IOException {
437        Objects.requireNonNull(inputStream, "inputStream");
438        this.defaultEncoding = defaultEncoding;
439        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE), false, BOMS);
440        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
441        this.encoding = doRawStream(bom, pis, lenient);
442        this.reader = new InputStreamReader(pis, encoding);
443    }
444
445    /**
446     * Constructs a Reader using an InputStream and the associated content-type header.
447     * <p>
448     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
449     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
450     * </p>
451     * <p>
452     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
453     * </p>
454     *
455     * @param inputStream     InputStream to create the reader from.
456     * @param httpContentType content-type header to use for the resolution of the charset encoding.
457     * @throws NullPointerException if the input stream is {@code null}.
458     * @throws IOException thrown if there is a problem reading the file.
459     * @deprecated Use {@link #builder()}
460     */
461    @Deprecated
462    public XmlStreamReader(final InputStream inputStream, final String httpContentType) throws IOException {
463        this(inputStream, httpContentType, true);
464    }
465
466    /**
467     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
468     * <p>
469     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
470     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
471     * </p>
472     * <p>
473     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
474     * </p>
475     * <p>
476     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
477     * </p>
478     * <p>
479     * Else if the XML prolog had a charset encoding that encoding is used.
480     * </p>
481     * <p>
482     * Else if the content type had a charset encoding that encoding is used.
483     * </p>
484     * <p>
485     * Else 'UTF-8' is used.
486     * </p>
487     * <p>
488     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
489     * </p>
490     *
491     * @param inputStream     InputStream to create the reader from.
492     * @param httpContentType content-type header to use for the resolution of the charset encoding.
493     * @param lenient         indicates if the charset encoding detection should be relaxed.
494     * @throws NullPointerException if the input stream is {@code null}.
495     * @throws IOException              thrown if there is a problem reading the file.
496     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
497     * @deprecated Use {@link #builder()}
498     */
499    @Deprecated
500    public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient) throws IOException {
501        this(inputStream, httpContentType, lenient, null);
502    }
503
504    /**
505     * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection.
506     * <p>
507     * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog
508     * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type.
509     * </p>
510     * <p>
511     * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following:
512     * </p>
513     * <p>
514     * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again.
515     * </p>
516     * <p>
517     * Else if the XML prolog had a charset encoding that encoding is used.
518     * </p>
519     * <p>
520     * Else if the content type had a charset encoding that encoding is used.
521     * </p>
522     * <p>
523     * Else 'UTF-8' is used.
524     * </p>
525     * <p>
526     * If lenient detection is indicated an XmlStreamReaderException is never thrown.
527     * </p>
528     *
529     * @param inputStream     InputStream to create the reader from.
530     * @param httpContentType content-type header to use for the resolution of the charset encoding.
531     * @param lenient         indicates if the charset encoding detection should be relaxed.
532     * @param defaultEncoding The default encoding
533     * @throws NullPointerException if the input stream is {@code null}.
534     * @throws IOException              thrown if there is a problem reading the file.
535     * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification.
536     * @deprecated Use {@link #builder()}
537     */
538    @Deprecated
539    @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance.
540    public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient, final String defaultEncoding)
541            throws IOException {
542        Objects.requireNonNull(inputStream, "inputStream");
543        this.defaultEncoding = defaultEncoding;
544        final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE), false, BOMS);
545        final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES);
546        this.encoding = processHttpStream(bom, pis, httpContentType, lenient);
547        this.reader = new InputStreamReader(pis, encoding);
548    }
549
550    /**
551     * Constructs a Reader for a File.
552     * <p>
553     * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8.
554     * </p>
555     * <p>
556     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
557     * </p>
558     *
559     * @param file File to create a Reader from.
560     * @throws NullPointerException if the input is {@code null}.
561     * @throws IOException thrown if there is a problem reading the file.
562     * @since 2.11.0
563     * @deprecated Use {@link #builder()}
564     */
565    @Deprecated
566    @SuppressWarnings("resource") // InputStream is managed through another reader in this instance.
567    public XmlStreamReader(final Path file) throws IOException {
568        this(Files.newInputStream(Objects.requireNonNull(file, "file")));
569    }
570
571    /**
572     * Constructs a Reader using the InputStream of a URL.
573     * <p>
574     * If the URL is not of type HTTP and there is not 'content-type' header in the fetched data it uses the same logic used for Files.
575     * </p>
576     * <p>
577     * If the URL is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with content-type.
578     * </p>
579     * <p>
580     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
581     * </p>
582     *
583     * @param url URL to create a Reader from.
584     * @throws NullPointerException if the input is {@code null}.
585     * @throws IOException thrown if there is a problem reading the stream of the URL.
586     */
587    public XmlStreamReader(final URL url) throws IOException {
588        this(Objects.requireNonNull(url, "url").openConnection(), null);
589    }
590
591    /**
592     * Constructs a Reader using the InputStream of a URLConnection.
593     * <p>
594     * If the URLConnection is not of type HttpURLConnection and there is not 'content-type' header in the fetched data it uses the same logic used for files.
595     * </p>
596     * <p>
597     * If the URLConnection is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with
598     * content-type.
599     * </p>
600     * <p>
601     * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details.
602     * </p>
603     *
604     * @param urlConnection   URLConnection to create a Reader from.
605     * @param defaultEncoding The default encoding
606     * @throws NullPointerException if the input is {@code null}.
607     * @throws IOException thrown if there is a problem reading the stream of the URLConnection.
608     */
609    public XmlStreamReader(final URLConnection urlConnection, final String defaultEncoding) throws IOException {
610        Objects.requireNonNull(urlConnection, "urlConnection");
611        this.defaultEncoding = defaultEncoding;
612        final boolean lenient = true;
613        final String contentType = urlConnection.getContentType();
614        final InputStream inputStream = urlConnection.getInputStream();
615        @SuppressWarnings("resource") // managed by the InputStreamReader tracked by this instance
616        // @formatter:off
617        final BOMInputStream bomInput = BOMInputStream.builder()
618            .setInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE))
619            .setInclude(false)
620            .setByteOrderMarks(BOMS)
621            .get();
622        @SuppressWarnings("resource")
623        final BOMInputStream piInput = BOMInputStream.builder()
624            .setInputStream(new BufferedInputStream(bomInput, IOUtils.DEFAULT_BUFFER_SIZE))
625            .setInclude(true)
626            .setByteOrderMarks(XML_GUESS_BYTES)
627            .get();
628        // @formatter:on
629        if (urlConnection instanceof HttpURLConnection || contentType != null) {
630            this.encoding = processHttpStream(bomInput, piInput, contentType, lenient);
631        } else {
632            this.encoding = doRawStream(bomInput, piInput, lenient);
633        }
634        this.reader = new InputStreamReader(piInput, encoding);
635    }
636
637    /**
638     * Calculates the HTTP encoding.
639     *
640     * @param httpContentType The HTTP content type
641     * @param bomEnc          BOM encoding
642     * @param xmlGuessEnc     XML Guess encoding
643     * @param xmlEnc          XML encoding
644     * @param lenient         indicates if the charset encoding detection should be relaxed.
645     * @return the HTTP encoding
646     * @throws IOException thrown if there is a problem reading the stream.
647     */
648    String calculateHttpEncoding(final String httpContentType, final String bomEnc, final String xmlGuessEnc, final String xmlEnc, final boolean lenient)
649            throws IOException {
650
651        // Lenient and has XML encoding
652        if (lenient && xmlEnc != null) {
653            return xmlEnc;
654        }
655
656        // Determine mime/encoding content types from HTTP Content Type
657        final String cTMime = getContentTypeMime(httpContentType);
658        final String cTEnc = getContentTypeEncoding(httpContentType);
659        final boolean appXml = isAppXml(cTMime);
660        final boolean textXml = isTextXml(cTMime);
661
662        // Mime type NOT "application/xml" or "text/xml"
663        if (!appXml && !textXml) {
664            final String msg = MessageFormat.format(HTTP_EX_3, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
665            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
666        }
667
668        // No content type encoding
669        if (cTEnc == null) {
670            if (appXml) {
671                return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
672            }
673            return defaultEncoding == null ? US_ASCII : defaultEncoding;
674        }
675
676        // UTF-16BE or UTF-16LE content type encoding
677        if (cTEnc.equals(UTF_16BE) || cTEnc.equals(UTF_16LE)) {
678            if (bomEnc != null) {
679                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
680                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
681            }
682            return cTEnc;
683        }
684
685        // UTF-16 content type encoding
686        if (cTEnc.equals(UTF_16)) {
687            if (bomEnc != null && bomEnc.startsWith(UTF_16)) {
688                return bomEnc;
689            }
690            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
691            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
692        }
693
694        // UTF-32BE or UTF-132E content type encoding
695        if (cTEnc.equals(UTF_32BE) || cTEnc.equals(UTF_32LE)) {
696            if (bomEnc != null) {
697                final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
698                throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
699            }
700            return cTEnc;
701        }
702
703        // UTF-32 content type encoding
704        if (cTEnc.equals(UTF_32)) {
705            if (bomEnc != null && bomEnc.startsWith(UTF_32)) {
706                return bomEnc;
707            }
708            final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
709            throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc);
710        }
711
712        return cTEnc;
713    }
714
715    /**
716     * Calculate the raw encoding.
717     *
718     * @param bomEnc      BOM encoding
719     * @param xmlGuessEnc XML Guess encoding
720     * @param xmlEnc      XML encoding
721     * @return the raw encoding
722     * @throws IOException thrown if there is a problem reading the stream.
723     */
724    String calculateRawEncoding(final String bomEnc, final String xmlGuessEnc, final String xmlEnc) throws IOException {
725
726        // BOM is Null
727        if (bomEnc == null) {
728            if (xmlGuessEnc == null || xmlEnc == null) {
729                return defaultEncoding == null ? UTF_8 : defaultEncoding;
730            }
731            if (xmlEnc.equals(UTF_16) && (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc.equals(UTF_16LE))) {
732                return xmlGuessEnc;
733            }
734            return xmlEnc;
735        }
736
737        // BOM is UTF-8
738        if (bomEnc.equals(UTF_8)) {
739            if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) {
740                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
741                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
742            }
743            if (xmlEnc != null && !xmlEnc.equals(UTF_8)) {
744                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
745                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
746            }
747            return bomEnc;
748        }
749
750        // BOM is UTF-16BE or UTF-16LE
751        if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) {
752            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
753                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
754                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
755            }
756            if (xmlEnc != null && !xmlEnc.equals(UTF_16) && !xmlEnc.equals(bomEnc)) {
757                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
758                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
759            }
760            return bomEnc;
761        }
762
763        // BOM is UTF-32BE or UTF-32LE
764        if (bomEnc.equals(UTF_32BE) || bomEnc.equals(UTF_32LE)) {
765            if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) {
766                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
767                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
768            }
769            if (xmlEnc != null && !xmlEnc.equals(UTF_32) && !xmlEnc.equals(bomEnc)) {
770                final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc);
771                throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
772            }
773            return bomEnc;
774        }
775
776        // BOM is something else
777        final String msg = MessageFormat.format(RAW_EX_2, bomEnc, xmlGuessEnc, xmlEnc);
778        throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc);
779    }
780
781    /**
782     * Closes the XmlStreamReader stream.
783     *
784     * @throws IOException thrown if there was a problem closing the stream.
785     */
786    @Override
787    public void close() throws IOException {
788        reader.close();
789    }
790
791    /**
792     * Does lenient detection.
793     *
794     * @param httpContentType content-type header to use for the resolution of the charset encoding.
795     * @param ex              The thrown exception
796     * @return the encoding
797     * @throws IOException thrown if there is a problem reading the stream.
798     */
799    private String doLenientDetection(String httpContentType, XmlStreamReaderException ex) throws IOException {
800        if (httpContentType != null && httpContentType.startsWith("text/html")) {
801            httpContentType = httpContentType.substring("text/html".length());
802            httpContentType = "text/xml" + httpContentType;
803            try {
804                return calculateHttpEncoding(httpContentType, ex.getBomEncoding(), ex.getXmlGuessEncoding(), ex.getXmlEncoding(), true);
805            } catch (final XmlStreamReaderException ex2) {
806                ex = ex2;
807            }
808        }
809        String encoding = ex.getXmlEncoding();
810        if (encoding == null) {
811            encoding = ex.getContentTypeEncoding();
812        }
813        if (encoding == null) {
814            encoding = defaultEncoding == null ? UTF_8 : defaultEncoding;
815        }
816        return encoding;
817    }
818
819    /**
820     * Process the raw stream.
821     *
822     * @param bom     BOMInputStream to detect byte order marks
823     * @param pis     BOMInputStream to guess XML encoding
824     * @param lenient indicates if the charset encoding detection should be relaxed.
825     * @return the encoding to be used
826     * @throws IOException thrown if there is a problem reading the stream.
827     */
828    private String doRawStream(final BOMInputStream bom, final BOMInputStream pis, final boolean lenient) throws IOException {
829        final String bomEnc = bom.getBOMCharsetName();
830        final String xmlGuessEnc = pis.getBOMCharsetName();
831        final String xmlEnc = getXmlProlog(pis, xmlGuessEnc);
832        try {
833            return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc);
834        } catch (final XmlStreamReaderException ex) {
835            if (lenient) {
836                return doLenientDetection(null, ex);
837            }
838            throw ex;
839        }
840    }
841
842    /**
843     * Gets the default encoding to use if none is set in HTTP content-type, XML prolog and the rules based on content-type are not adequate.
844     * <p>
845     * If it is NULL the content-type based rules are used.
846     * </p>
847     *
848     * @return the default encoding to use.
849     */
850    public String getDefaultEncoding() {
851        return defaultEncoding;
852    }
853
854    /**
855     * Gets the charset encoding of the XmlStreamReader.
856     *
857     * @return charset encoding.
858     */
859    public String getEncoding() {
860        return encoding;
861    }
862
863    /**
864     * Processes an HTTP stream.
865     *
866     * @param bomInput        BOMInputStream to detect byte order marks
867     * @param piInput         BOMInputStream to guess XML encoding
868     * @param httpContentType The HTTP content type
869     * @param lenient         indicates if the charset encoding detection should be relaxed.
870     * @return the encoding to be used
871     * @throws IOException thrown if there is a problem reading the stream.
872     */
873    private String processHttpStream(final BOMInputStream bomInput, final BOMInputStream piInput, final String httpContentType, final boolean lenient)
874            throws IOException {
875        final String bomEnc = bomInput.getBOMCharsetName();
876        final String xmlGuessEnc = piInput.getBOMCharsetName();
877        final String xmlEnc = getXmlProlog(piInput, xmlGuessEnc);
878        try {
879            return calculateHttpEncoding(httpContentType, bomEnc, xmlGuessEnc, xmlEnc, lenient);
880        } catch (final XmlStreamReaderException ex) {
881            if (lenient) {
882                return doLenientDetection(httpContentType, ex);
883            }
884            throw ex;
885        }
886    }
887
888    /**
889     * Reads the underlying reader's {@code read(char[], int, int)} method.
890     *
891     * @param buf    the buffer to read the characters into
892     * @param offset The start offset
893     * @param len    The number of bytes to read
894     * @return the number of characters read or -1 if the end of stream
895     * @throws IOException if an I/O error occurs.
896     */
897    @Override
898    public int read(final char[] buf, final int offset, final int len) throws IOException {
899        return reader.read(buf, offset, len);
900    }
901
902}