001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.commons.io.input; 018 019import java.io.BufferedInputStream; 020import java.io.BufferedReader; 021import java.io.File; 022import java.io.IOException; 023import java.io.InputStream; 024import java.io.InputStreamReader; 025import java.io.Reader; 026import java.io.StringReader; 027import java.net.HttpURLConnection; 028import java.net.URL; 029import java.net.URLConnection; 030import java.nio.charset.Charset; 031import java.nio.charset.StandardCharsets; 032import java.nio.file.Files; 033import java.nio.file.Path; 034import java.text.MessageFormat; 035import java.util.Locale; 036import java.util.Objects; 037import java.util.regex.Matcher; 038import java.util.regex.Pattern; 039 040import org.apache.commons.io.ByteOrderMark; 041import org.apache.commons.io.Charsets; 042import org.apache.commons.io.IOUtils; 043import org.apache.commons.io.build.AbstractStreamBuilder; 044import org.apache.commons.io.function.IOConsumer; 045import org.apache.commons.io.output.XmlStreamWriter; 046 047/** 048 * Character stream that handles all the necessary Voodoo to figure out the charset encoding of the XML document within the stream. 049 * <p> 050 * IMPORTANT: This class is not related in any way to the org.xml.sax.XMLReader. This one IS a character stream. 051 * </p> 052 * <p> 053 * All this has to be done without consuming characters from the stream, if not the XML parser will not recognized the document as a valid XML. This is not 100% 054 * true, but it's close enough (UTF-8 BOM is not handled by all parsers right now, XmlStreamReader handles it and things work in all parsers). 055 * </p> 056 * <p> 057 * The XmlStreamReader class handles the charset encoding of XML documents in Files, raw streams and HTTP streams by offering a wide set of constructors. 058 * </p> 059 * <p> 060 * By default the charset encoding detection is lenient, the constructor with the lenient flag can be used for a script (following HTTP MIME and XML 061 * specifications). All this is nicely explained by Mark Pilgrim in his blog, <a href="http://diveintomark.org/archives/2004/02/13/xml-media-types"> Determining 062 * the character encoding of a feed</a>. 063 * </p> 064 * <p> 065 * To build an instance, see {@link Builder}. 066 * </p> 067 * <p> 068 * Originally developed for <a href="http://rome.dev.java.net">ROME</a> under Apache License 2.0. 069 * </p> 070 * 071 * @see org.apache.commons.io.output.XmlStreamWriter 072 * @since 2.0 073 */ 074public class XmlStreamReader extends Reader { 075 076 /** 077 * Builds a new {@link XmlStreamWriter} instance. 078 * 079 * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection. 080 * <p> 081 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 082 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 083 * </p> 084 * <p> 085 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 086 * </p> 087 * <p> 088 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 089 * </p> 090 * <p> 091 * Else if the XML prolog had a charset encoding that encoding is used. 092 * </p> 093 * <p> 094 * Else if the content type had a charset encoding that encoding is used. 095 * </p> 096 * <p> 097 * Else 'UTF-8' is used. 098 * </p> 099 * <p> 100 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 101 * </p> 102 * <p> 103 * For example: 104 * </p> 105 * 106 * <pre>{@code 107 * XmlStreamReader r = XmlStreamReader.builder() 108 * .setPath(path) 109 * .setCharset(StandardCharsets.UTF_8) 110 * .get();} 111 * </pre> 112 * 113 * @since 2.12.0 114 */ 115 public static class Builder extends AbstractStreamBuilder<XmlStreamReader, Builder> { 116 117 private boolean nullCharset = true; 118 private boolean lenient = true; 119 private String httpContentType; 120 121 /** 122 * Constructs a new instance. 123 * <p> 124 * This builder use the aspect InputStream, OpenOption[], httpContentType, lenient, and defaultEncoding. 125 * </p> 126 * <p> 127 * You must provide an origin that can be converted to an InputStream by this builder, otherwise, this call will throw an 128 * {@link UnsupportedOperationException}. 129 * </p> 130 * 131 * @return a new instance. 132 * @throws UnsupportedOperationException if the origin cannot provide an InputStream. 133 * @throws IOException thrown if there is a problem reading the stream. 134 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 135 * @see #getInputStream() 136 */ 137 @SuppressWarnings("resource") 138 @Override 139 public XmlStreamReader get() throws IOException { 140 final String defaultEncoding = nullCharset ? null : getCharset().name(); 141 // @formatter:off 142 return httpContentType == null 143 ? new XmlStreamReader(getInputStream(), lenient, defaultEncoding) 144 : new XmlStreamReader(getInputStream(), httpContentType, lenient, defaultEncoding); 145 // @formatter:on 146 } 147 148 @Override 149 public Builder setCharset(final Charset charset) { 150 nullCharset = charset == null; 151 return super.setCharset(charset); 152 } 153 154 @Override 155 public Builder setCharset(final String charset) { 156 nullCharset = charset == null; 157 return super.setCharset(Charsets.toCharset(charset, getCharsetDefault())); 158 } 159 160 /** 161 * Sets the HTTP content type. 162 * 163 * @param httpContentType the HTTP content type. 164 * @return this. 165 */ 166 public Builder setHttpContentType(final String httpContentType) { 167 this.httpContentType = httpContentType; 168 return this; 169 } 170 171 /** 172 * Sets the lenient toggle. 173 * 174 * @param lenient the lenient toggle. 175 * @return this. 176 */ 177 public Builder setLenient(final boolean lenient) { 178 this.lenient = lenient; 179 return this; 180 } 181 182 } 183 184 private static final String UTF_8 = StandardCharsets.UTF_8.name(); 185 186 private static final String US_ASCII = StandardCharsets.US_ASCII.name(); 187 188 private static final String UTF_16BE = StandardCharsets.UTF_16BE.name(); 189 190 private static final String UTF_16LE = StandardCharsets.UTF_16LE.name(); 191 192 private static final String UTF_32BE = "UTF-32BE"; 193 194 private static final String UTF_32LE = "UTF-32LE"; 195 196 private static final String UTF_16 = StandardCharsets.UTF_16.name(); 197 198 private static final String UTF_32 = "UTF-32"; 199 200 private static final String EBCDIC = "CP1047"; 201 202 private static final ByteOrderMark[] BOMS = { ByteOrderMark.UTF_8, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_32BE, 203 ByteOrderMark.UTF_32LE }; 204 205 /** UTF_16LE and UTF_32LE have the same two starting BOM bytes. */ 206 private static final ByteOrderMark[] XML_GUESS_BYTES = { new ByteOrderMark(UTF_8, 0x3C, 0x3F, 0x78, 0x6D), 207 new ByteOrderMark(UTF_16BE, 0x00, 0x3C, 0x00, 0x3F), new ByteOrderMark(UTF_16LE, 0x3C, 0x00, 0x3F, 0x00), 208 new ByteOrderMark(UTF_32BE, 0x00, 0x00, 0x00, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D), 209 new ByteOrderMark(UTF_32LE, 0x3C, 0x00, 0x00, 0x00, 0x3F, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x6D, 0x00, 0x00, 0x00), 210 new ByteOrderMark(EBCDIC, 0x4C, 0x6F, 0xA7, 0x94) }; 211 212 private static final Pattern CHARSET_PATTERN = Pattern.compile("charset=[\"']?([.[^; \"']]*)[\"']?"); 213 214 /** 215 * Pattern capturing the encoding of the "xml" processing instruction. 216 * <p> 217 * See also the <a href="https://www.w3.org/TR/2008/REC-xml-20081126/#NT-EncName">XML specification</a>. 218 * </p> 219 */ 220 public static final Pattern ENCODING_PATTERN = Pattern.compile( 221 // @formatter:off 222 "^<\\?xml\\s+" 223 + "version\\s*=\\s*(?:(?:\"1\\.[0-9]+\")|(?:'1.[0-9]+'))\\s+" 224 + "encoding\\s*=\\s*" 225 + "((?:\"[A-Za-z0-9][A-Za-z0-9._+:-]*\")" // double-quoted 226 + "|(?:'[A-Za-z0-9][A-Za-z0-9._+:-]*'))", // single-quoted 227 Pattern.MULTILINE); 228 // N.B. the documented pattern is 229 // EncName ::= [A-Za-z] ([A-Za-z0-9._] | '-')* 230 // However this does not match all the aliases that are supported by Java. 231 // e.g. '437', 'ISO_8859-1:1987' and 'ebcdic-de-273+euro' 232 // @formatter:on 233 234 private static final String RAW_EX_1 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] encoding mismatch"; 235 236 private static final String RAW_EX_2 = "Illegal encoding, BOM [{0}] XML guess [{1}] XML prolog [{2}] unknown BOM"; 237 238 private static final String HTTP_EX_1 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], BOM must be null"; 239 240 private static final String HTTP_EX_2 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], encoding mismatch"; 241 242 private static final String HTTP_EX_3 = "Illegal encoding, CT-MIME [{0}] CT-Enc [{1}] BOM [{2}] XML guess [{3}] XML prolog [{4}], Illegal MIME"; 243 244 /** 245 * Constructs a new {@link Builder}. 246 * 247 * @return a new {@link Builder}. 248 * @since 2.12.0 249 */ 250 public static Builder builder() { 251 return new Builder(); 252 } 253 254 /** 255 * Gets the charset parameter value, {@code null} if not present, {@code null} if httpContentType is {@code null}. 256 * 257 * @param httpContentType the HTTP content type 258 * @return The content type encoding (upcased) 259 */ 260 static String getContentTypeEncoding(final String httpContentType) { 261 String encoding = null; 262 if (httpContentType != null) { 263 final int i = httpContentType.indexOf(";"); 264 if (i > -1) { 265 final String postMime = httpContentType.substring(i + 1); 266 final Matcher m = CHARSET_PATTERN.matcher(postMime); 267 encoding = m.find() ? m.group(1) : null; 268 encoding = encoding != null ? encoding.toUpperCase(Locale.ROOT) : null; 269 } 270 } 271 return encoding; 272 } 273 274 /** 275 * Gets the MIME type or {@code null} if httpContentType is {@code null}. 276 * 277 * @param httpContentType the HTTP content type 278 * @return The mime content type 279 */ 280 static String getContentTypeMime(final String httpContentType) { 281 String mime = null; 282 if (httpContentType != null) { 283 final int i = httpContentType.indexOf(";"); 284 if (i >= 0) { 285 mime = httpContentType.substring(0, i); 286 } else { 287 mime = httpContentType; 288 } 289 mime = mime.trim(); 290 } 291 return mime; 292 } 293 294 /** 295 * Gets the encoding declared in the <?xml encoding=...?>, {@code null} if none. 296 * 297 * @param inputStream InputStream to create the reader from. 298 * @param guessedEnc guessed encoding 299 * @return the encoding declared in the <?xml encoding=...?> 300 * @throws IOException thrown if there is a problem reading the stream. 301 */ 302 private static String getXmlProlog(final InputStream inputStream, final String guessedEnc) throws IOException { 303 String encoding = null; 304 if (guessedEnc != null) { 305 final byte[] bytes = IOUtils.byteArray(); 306 inputStream.mark(IOUtils.DEFAULT_BUFFER_SIZE); 307 int offset = 0; 308 int max = IOUtils.DEFAULT_BUFFER_SIZE; 309 int c = inputStream.read(bytes, offset, max); 310 int firstGT = -1; 311 String xmlProlog = ""; // avoid possible NPE warning (cannot happen; this just silences the warning) 312 while (c != -1 && firstGT == -1 && offset < IOUtils.DEFAULT_BUFFER_SIZE) { 313 offset += c; 314 max -= c; 315 c = inputStream.read(bytes, offset, max); 316 xmlProlog = new String(bytes, 0, offset, guessedEnc); 317 firstGT = xmlProlog.indexOf('>'); 318 } 319 if (firstGT == -1) { 320 if (c == -1) { 321 throw new IOException("Unexpected end of XML stream"); 322 } 323 throw new IOException("XML prolog or ROOT element not found on first " + offset + " bytes"); 324 } 325 final int bytesRead = offset; 326 if (bytesRead > 0) { 327 inputStream.reset(); 328 final BufferedReader bReader = new BufferedReader(new StringReader(xmlProlog.substring(0, firstGT + 1))); 329 final StringBuilder prolog = new StringBuilder(); 330 IOConsumer.forEach(bReader.lines(), prolog::append); 331 final Matcher m = ENCODING_PATTERN.matcher(prolog); 332 if (m.find()) { 333 encoding = m.group(1).toUpperCase(Locale.ROOT); 334 encoding = encoding.substring(1, encoding.length() - 1); 335 } 336 } 337 } 338 return encoding; 339 } 340 341 /** 342 * Tests if the MIME type belongs to the APPLICATION XML family. 343 * 344 * @param mime The mime type 345 * @return true if the mime type belongs to the APPLICATION XML family, otherwise false 346 */ 347 static boolean isAppXml(final String mime) { 348 return mime != null && (mime.equals("application/xml") || mime.equals("application/xml-dtd") || mime.equals("application/xml-external-parsed-entity") 349 || mime.startsWith("application/") && mime.endsWith("+xml")); 350 } 351 352 /** 353 * Tests if the MIME type belongs to the TEXT XML family. 354 * 355 * @param mime The mime type 356 * @return true if the mime type belongs to the TEXT XML family, otherwise false 357 */ 358 static boolean isTextXml(final String mime) { 359 return mime != null && (mime.equals("text/xml") || mime.equals("text/xml-external-parsed-entity") || mime.startsWith("text/") && mime.endsWith("+xml")); 360 } 361 362 private final Reader reader; 363 364 private final String encoding; 365 366 private final String defaultEncoding; 367 368 /** 369 * Constructs a Reader for a File. 370 * <p> 371 * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8. 372 * </p> 373 * <p> 374 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 375 * </p> 376 * 377 * @param file File to create a Reader from. 378 * @throws NullPointerException if the input is {@code null}. 379 * @throws IOException thrown if there is a problem reading the file. 380 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 381 */ 382 @Deprecated 383 public XmlStreamReader(final File file) throws IOException { 384 this(Objects.requireNonNull(file, "file").toPath()); 385 } 386 387 /** 388 * Constructs a Reader for a raw InputStream. 389 * <p> 390 * It follows the same logic used for files. 391 * </p> 392 * <p> 393 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 394 * </p> 395 * 396 * @param inputStream InputStream to create a Reader from. 397 * @throws NullPointerException if the input stream is {@code null}. 398 * @throws IOException thrown if there is a problem reading the stream. 399 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 400 */ 401 @Deprecated 402 public XmlStreamReader(final InputStream inputStream) throws IOException { 403 this(inputStream, true); 404 } 405 406 /** 407 * Constructs a Reader for a raw InputStream. 408 * <p> 409 * It follows the same logic used for files. 410 * </p> 411 * <p> 412 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 413 * </p> 414 * <p> 415 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 416 * </p> 417 * <p> 418 * Else if the XML prolog had a charset encoding that encoding is used. 419 * </p> 420 * <p> 421 * Else if the content type had a charset encoding that encoding is used. 422 * </p> 423 * <p> 424 * Else 'UTF-8' is used. 425 * </p> 426 * <p> 427 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 428 * </p> 429 * 430 * @param inputStream InputStream to create a Reader from. 431 * @param lenient indicates if the charset encoding detection should be relaxed. 432 * @throws NullPointerException if the input stream is {@code null}. 433 * @throws IOException thrown if there is a problem reading the stream. 434 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 435 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 436 */ 437 @Deprecated 438 public XmlStreamReader(final InputStream inputStream, final boolean lenient) throws IOException { 439 this(inputStream, lenient, null); 440 } 441 442 /** 443 * Constructs a Reader for a raw InputStream. 444 * <p> 445 * It follows the same logic used for files. 446 * </p> 447 * <p> 448 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 449 * </p> 450 * <p> 451 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 452 * </p> 453 * <p> 454 * Else if the XML prolog had a charset encoding that encoding is used. 455 * </p> 456 * <p> 457 * Else if the content type had a charset encoding that encoding is used. 458 * </p> 459 * <p> 460 * Else 'UTF-8' is used. 461 * </p> 462 * <p> 463 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 464 * </p> 465 * 466 * @param inputStream InputStream to create a Reader from. 467 * @param lenient indicates if the charset encoding detection should be relaxed. 468 * @param defaultEncoding The default encoding 469 * @throws NullPointerException if the input stream is {@code null}. 470 * @throws IOException thrown if there is a problem reading the stream. 471 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 472 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 473 */ 474 @Deprecated 475 @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance. 476 public XmlStreamReader(final InputStream inputStream, final boolean lenient, final String defaultEncoding) throws IOException { 477 Objects.requireNonNull(inputStream, "inputStream"); 478 this.defaultEncoding = defaultEncoding; 479 final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE), false, BOMS); 480 final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES); 481 this.encoding = doRawStream(bom, pis, lenient); 482 this.reader = new InputStreamReader(pis, encoding); 483 } 484 485 /** 486 * Constructs a Reader using an InputStream and the associated content-type header. 487 * <p> 488 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 489 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 490 * </p> 491 * <p> 492 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 493 * </p> 494 * 495 * @param inputStream InputStream to create the reader from. 496 * @param httpContentType content-type header to use for the resolution of the charset encoding. 497 * @throws NullPointerException if the input stream is {@code null}. 498 * @throws IOException thrown if there is a problem reading the file. 499 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 500 */ 501 @Deprecated 502 public XmlStreamReader(final InputStream inputStream, final String httpContentType) throws IOException { 503 this(inputStream, httpContentType, true); 504 } 505 506 /** 507 * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection. 508 * <p> 509 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 510 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 511 * </p> 512 * <p> 513 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 514 * </p> 515 * <p> 516 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 517 * </p> 518 * <p> 519 * Else if the XML prolog had a charset encoding that encoding is used. 520 * </p> 521 * <p> 522 * Else if the content type had a charset encoding that encoding is used. 523 * </p> 524 * <p> 525 * Else 'UTF-8' is used. 526 * </p> 527 * <p> 528 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 529 * </p> 530 * 531 * @param inputStream InputStream to create the reader from. 532 * @param httpContentType content-type header to use for the resolution of the charset encoding. 533 * @param lenient indicates if the charset encoding detection should be relaxed. 534 * @throws NullPointerException if the input stream is {@code null}. 535 * @throws IOException thrown if there is a problem reading the file. 536 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 537 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 538 */ 539 @Deprecated 540 public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient) throws IOException { 541 this(inputStream, httpContentType, lenient, null); 542 } 543 544 /** 545 * Constructs a Reader using an InputStream and the associated content-type header. This constructor is lenient regarding the encoding detection. 546 * <p> 547 * First it checks if the stream has BOM. If there is not BOM checks the content-type encoding. If there is not content-type encoding checks the XML prolog 548 * encoding. If there is not XML prolog encoding uses the default encoding mandated by the content-type MIME type. 549 * </p> 550 * <p> 551 * If lenient detection is indicated and the detection above fails as per specifications it then attempts the following: 552 * </p> 553 * <p> 554 * If the content type was 'text/html' it replaces it with 'text/xml' and tries the detection again. 555 * </p> 556 * <p> 557 * Else if the XML prolog had a charset encoding that encoding is used. 558 * </p> 559 * <p> 560 * Else if the content type had a charset encoding that encoding is used. 561 * </p> 562 * <p> 563 * Else 'UTF-8' is used. 564 * </p> 565 * <p> 566 * If lenient detection is indicated an XmlStreamReaderException is never thrown. 567 * </p> 568 * 569 * @param inputStream InputStream to create the reader from. 570 * @param httpContentType content-type header to use for the resolution of the charset encoding. 571 * @param lenient indicates if the charset encoding detection should be relaxed. 572 * @param defaultEncoding The default encoding 573 * @throws NullPointerException if the input stream is {@code null}. 574 * @throws IOException thrown if there is a problem reading the file. 575 * @throws XmlStreamReaderException thrown if the charset encoding could not be determined according to the specification. 576 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 577 */ 578 @Deprecated 579 @SuppressWarnings("resource") // InputStream is managed through a InputStreamReader in this instance. 580 public XmlStreamReader(final InputStream inputStream, final String httpContentType, final boolean lenient, final String defaultEncoding) 581 throws IOException { 582 Objects.requireNonNull(inputStream, "inputStream"); 583 this.defaultEncoding = defaultEncoding; 584 final BOMInputStream bom = new BOMInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE), false, BOMS); 585 final BOMInputStream pis = new BOMInputStream(bom, true, XML_GUESS_BYTES); 586 this.encoding = processHttpStream(bom, pis, httpContentType, lenient); 587 this.reader = new InputStreamReader(pis, encoding); 588 } 589 590 /** 591 * Constructs a Reader for a File. 592 * <p> 593 * It looks for the UTF-8 BOM first, if none sniffs the XML prolog charset, if this is also missing defaults to UTF-8. 594 * </p> 595 * <p> 596 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 597 * </p> 598 * 599 * @param file File to create a Reader from. 600 * @throws NullPointerException if the input is {@code null}. 601 * @throws IOException thrown if there is a problem reading the file. 602 * @since 2.11.0 603 * @deprecated Use {@link #builder()}, {@link Builder}, and {@link Builder#get()} 604 */ 605 @Deprecated 606 @SuppressWarnings("resource") // InputStream is managed through another reader in this instance. 607 public XmlStreamReader(final Path file) throws IOException { 608 this(Files.newInputStream(Objects.requireNonNull(file, "file"))); 609 } 610 611 /** 612 * Constructs a Reader using the InputStream of a URL. 613 * <p> 614 * If the URL is not of type HTTP and there is not 'content-type' header in the fetched data it uses the same logic used for Files. 615 * </p> 616 * <p> 617 * If the URL is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with content-type. 618 * </p> 619 * <p> 620 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 621 * </p> 622 * 623 * @param url URL to create a Reader from. 624 * @throws NullPointerException if the input is {@code null}. 625 * @throws IOException thrown if there is a problem reading the stream of the URL. 626 */ 627 public XmlStreamReader(final URL url) throws IOException { 628 this(Objects.requireNonNull(url, "url").openConnection(), null); 629 } 630 631 /** 632 * Constructs a Reader using the InputStream of a URLConnection. 633 * <p> 634 * If the URLConnection is not of type HttpURLConnection and there is not 'content-type' header in the fetched data it uses the same logic used for files. 635 * </p> 636 * <p> 637 * If the URLConnection is a HTTP Url or there is a 'content-type' header in the fetched data it uses the same logic used for an InputStream with 638 * content-type. 639 * </p> 640 * <p> 641 * It does a lenient charset encoding detection, check the constructor with the lenient parameter for details. 642 * </p> 643 * 644 * @param urlConnection URLConnection to create a Reader from. 645 * @param defaultEncoding The default encoding 646 * @throws NullPointerException if the input is {@code null}. 647 * @throws IOException thrown if there is a problem reading the stream of the URLConnection. 648 */ 649 public XmlStreamReader(final URLConnection urlConnection, final String defaultEncoding) throws IOException { 650 Objects.requireNonNull(urlConnection, "urlConnection"); 651 this.defaultEncoding = defaultEncoding; 652 final boolean lenient = true; 653 final String contentType = urlConnection.getContentType(); 654 final InputStream inputStream = urlConnection.getInputStream(); 655 @SuppressWarnings("resource") // managed by the InputStreamReader tracked by this instance 656 // @formatter:off 657 final BOMInputStream bomInput = BOMInputStream.builder() 658 .setInputStream(new BufferedInputStream(inputStream, IOUtils.DEFAULT_BUFFER_SIZE)) 659 .setInclude(false) 660 .setByteOrderMarks(BOMS) 661 .get(); 662 @SuppressWarnings("resource") 663 final BOMInputStream piInput = BOMInputStream.builder() 664 .setInputStream(new BufferedInputStream(bomInput, IOUtils.DEFAULT_BUFFER_SIZE)) 665 .setInclude(true) 666 .setByteOrderMarks(XML_GUESS_BYTES) 667 .get(); 668 // @formatter:on 669 if (urlConnection instanceof HttpURLConnection || contentType != null) { 670 this.encoding = processHttpStream(bomInput, piInput, contentType, lenient); 671 } else { 672 this.encoding = doRawStream(bomInput, piInput, lenient); 673 } 674 this.reader = new InputStreamReader(piInput, encoding); 675 } 676 677 /** 678 * Calculates the HTTP encoding. 679 * 680 * @param httpContentType The HTTP content type 681 * @param bomEnc BOM encoding 682 * @param xmlGuessEnc XML Guess encoding 683 * @param xmlEnc XML encoding 684 * @param lenient indicates if the charset encoding detection should be relaxed. 685 * @return the HTTP encoding 686 * @throws IOException thrown if there is a problem reading the stream. 687 */ 688 String calculateHttpEncoding(final String httpContentType, final String bomEnc, final String xmlGuessEnc, final String xmlEnc, final boolean lenient) 689 throws IOException { 690 691 // Lenient and has XML encoding 692 if (lenient && xmlEnc != null) { 693 return xmlEnc; 694 } 695 696 // Determine mime/encoding content types from HTTP Content Type 697 final String cTMime = getContentTypeMime(httpContentType); 698 final String cTEnc = getContentTypeEncoding(httpContentType); 699 final boolean appXml = isAppXml(cTMime); 700 final boolean textXml = isTextXml(cTMime); 701 702 // Mime type NOT "application/xml" or "text/xml" 703 if (!appXml && !textXml) { 704 final String msg = MessageFormat.format(HTTP_EX_3, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 705 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 706 } 707 708 // No content type encoding 709 if (cTEnc == null) { 710 if (appXml) { 711 return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc); 712 } 713 return defaultEncoding == null ? US_ASCII : defaultEncoding; 714 } 715 716 // UTF-16BE or UTF-16LE content type encoding 717 if (cTEnc.equals(UTF_16BE) || cTEnc.equals(UTF_16LE)) { 718 if (bomEnc != null) { 719 final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 720 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 721 } 722 return cTEnc; 723 } 724 725 // UTF-16 content type encoding 726 if (cTEnc.equals(UTF_16)) { 727 if (bomEnc != null && bomEnc.startsWith(UTF_16)) { 728 return bomEnc; 729 } 730 final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 731 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 732 } 733 734 // UTF-32BE or UTF-132E content type encoding 735 if (cTEnc.equals(UTF_32BE) || cTEnc.equals(UTF_32LE)) { 736 if (bomEnc != null) { 737 final String msg = MessageFormat.format(HTTP_EX_1, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 738 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 739 } 740 return cTEnc; 741 } 742 743 // UTF-32 content type encoding 744 if (cTEnc.equals(UTF_32)) { 745 if (bomEnc != null && bomEnc.startsWith(UTF_32)) { 746 return bomEnc; 747 } 748 final String msg = MessageFormat.format(HTTP_EX_2, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 749 throw new XmlStreamReaderException(msg, cTMime, cTEnc, bomEnc, xmlGuessEnc, xmlEnc); 750 } 751 752 return cTEnc; 753 } 754 755 /** 756 * Calculate the raw encoding. 757 * 758 * @param bomEnc BOM encoding 759 * @param xmlGuessEnc XML Guess encoding 760 * @param xmlEnc XML encoding 761 * @return the raw encoding 762 * @throws IOException thrown if there is a problem reading the stream. 763 */ 764 String calculateRawEncoding(final String bomEnc, final String xmlGuessEnc, final String xmlEnc) throws IOException { 765 766 // BOM is Null 767 if (bomEnc == null) { 768 if (xmlGuessEnc == null || xmlEnc == null) { 769 return defaultEncoding == null ? UTF_8 : defaultEncoding; 770 } 771 if (xmlEnc.equals(UTF_16) && (xmlGuessEnc.equals(UTF_16BE) || xmlGuessEnc.equals(UTF_16LE))) { 772 return xmlGuessEnc; 773 } 774 return xmlEnc; 775 } 776 777 // BOM is UTF-8 778 if (bomEnc.equals(UTF_8)) { 779 if (xmlGuessEnc != null && !xmlGuessEnc.equals(UTF_8)) { 780 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 781 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 782 } 783 if (xmlEnc != null && !xmlEnc.equals(UTF_8)) { 784 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 785 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 786 } 787 return bomEnc; 788 } 789 790 // BOM is UTF-16BE or UTF-16LE 791 if (bomEnc.equals(UTF_16BE) || bomEnc.equals(UTF_16LE)) { 792 if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) { 793 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 794 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 795 } 796 if (xmlEnc != null && !xmlEnc.equals(UTF_16) && !xmlEnc.equals(bomEnc)) { 797 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 798 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 799 } 800 return bomEnc; 801 } 802 803 // BOM is UTF-32BE or UTF-32LE 804 if (bomEnc.equals(UTF_32BE) || bomEnc.equals(UTF_32LE)) { 805 if (xmlGuessEnc != null && !xmlGuessEnc.equals(bomEnc)) { 806 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 807 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 808 } 809 if (xmlEnc != null && !xmlEnc.equals(UTF_32) && !xmlEnc.equals(bomEnc)) { 810 final String msg = MessageFormat.format(RAW_EX_1, bomEnc, xmlGuessEnc, xmlEnc); 811 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 812 } 813 return bomEnc; 814 } 815 816 // BOM is something else 817 final String msg = MessageFormat.format(RAW_EX_2, bomEnc, xmlGuessEnc, xmlEnc); 818 throw new XmlStreamReaderException(msg, bomEnc, xmlGuessEnc, xmlEnc); 819 } 820 821 /** 822 * Closes the XmlStreamReader stream. 823 * 824 * @throws IOException thrown if there was a problem closing the stream. 825 */ 826 @Override 827 public void close() throws IOException { 828 reader.close(); 829 } 830 831 /** 832 * Does lenient detection. 833 * 834 * @param httpContentType content-type header to use for the resolution of the charset encoding. 835 * @param ex The thrown exception 836 * @return the encoding 837 * @throws IOException thrown if there is a problem reading the stream. 838 */ 839 private String doLenientDetection(String httpContentType, XmlStreamReaderException ex) throws IOException { 840 if (httpContentType != null && httpContentType.startsWith("text/html")) { 841 httpContentType = httpContentType.substring("text/html".length()); 842 httpContentType = "text/xml" + httpContentType; 843 try { 844 return calculateHttpEncoding(httpContentType, ex.getBomEncoding(), ex.getXmlGuessEncoding(), ex.getXmlEncoding(), true); 845 } catch (final XmlStreamReaderException ex2) { 846 ex = ex2; 847 } 848 } 849 String encoding = ex.getXmlEncoding(); 850 if (encoding == null) { 851 encoding = ex.getContentTypeEncoding(); 852 } 853 if (encoding == null) { 854 encoding = defaultEncoding == null ? UTF_8 : defaultEncoding; 855 } 856 return encoding; 857 } 858 859 /** 860 * Process the raw stream. 861 * 862 * @param bom BOMInputStream to detect byte order marks 863 * @param pis BOMInputStream to guess XML encoding 864 * @param lenient indicates if the charset encoding detection should be relaxed. 865 * @return the encoding to be used 866 * @throws IOException thrown if there is a problem reading the stream. 867 */ 868 private String doRawStream(final BOMInputStream bom, final BOMInputStream pis, final boolean lenient) throws IOException { 869 final String bomEnc = bom.getBOMCharsetName(); 870 final String xmlGuessEnc = pis.getBOMCharsetName(); 871 final String xmlEnc = getXmlProlog(pis, xmlGuessEnc); 872 try { 873 return calculateRawEncoding(bomEnc, xmlGuessEnc, xmlEnc); 874 } catch (final XmlStreamReaderException ex) { 875 if (lenient) { 876 return doLenientDetection(null, ex); 877 } 878 throw ex; 879 } 880 } 881 882 /** 883 * Gets the default encoding to use if none is set in HTTP content-type, XML prolog and the rules based on content-type are not adequate. 884 * <p> 885 * If it is {@code null} the content-type based rules are used. 886 * </p> 887 * 888 * @return the default encoding to use. 889 */ 890 public String getDefaultEncoding() { 891 return defaultEncoding; 892 } 893 894 /** 895 * Gets the charset encoding of the XmlStreamReader. 896 * 897 * @return charset encoding. 898 */ 899 public String getEncoding() { 900 return encoding; 901 } 902 903 /** 904 * Processes an HTTP stream. 905 * 906 * @param bomInput BOMInputStream to detect byte order marks 907 * @param piInput BOMInputStream to guess XML encoding 908 * @param httpContentType The HTTP content type 909 * @param lenient indicates if the charset encoding detection should be relaxed. 910 * @return the encoding to be used 911 * @throws IOException thrown if there is a problem reading the stream. 912 */ 913 private String processHttpStream(final BOMInputStream bomInput, final BOMInputStream piInput, final String httpContentType, final boolean lenient) 914 throws IOException { 915 final String bomEnc = bomInput.getBOMCharsetName(); 916 final String xmlGuessEnc = piInput.getBOMCharsetName(); 917 final String xmlEnc = getXmlProlog(piInput, xmlGuessEnc); 918 try { 919 return calculateHttpEncoding(httpContentType, bomEnc, xmlGuessEnc, xmlEnc, lenient); 920 } catch (final XmlStreamReaderException ex) { 921 if (lenient) { 922 return doLenientDetection(httpContentType, ex); 923 } 924 throw ex; 925 } 926 } 927 928 /** 929 * Reads the underlying reader's {@code read(char[], int, int)} method. 930 * 931 * @param buf the buffer to read the characters into 932 * @param offset The start offset 933 * @param len The number of bytes to read 934 * @return the number of characters read or -1 if the end of stream 935 * @throws IOException if an I/O error occurs. 936 */ 937 @Override 938 public int read(final char[] buf, final int offset, final int len) throws IOException { 939 return reader.read(buf, offset, len); 940 } 941 942}