diff options
| author | Johnny1984 <johhny.html@o2.pl> | 2026-01-22 22:04:14 +0100 |
|---|---|---|
| committer | Johnny1984 <johhny.html@o2.pl> | 2026-01-22 22:04:14 +0100 |
| commit | 6006b19ae770cee1474e004f44ffd57b645354ef (patch) | |
| tree | 62f3f57c92f6b50ad72c6003a7e820dd4c1b6207 /source | |
| parent | da6410cfcedc1c6b7f34c2692c11a746d97adac8 (diff) | |
HTML5 <meta charset="...> support
Fixes mojibake when crawling some websites.
Diffstat (limited to 'source')
| -rw-r--r-- | source/net/yacy/document/parser/html/ScraperInputStream.java | 10 |
1 files changed, 5 insertions, 5 deletions
diff --git a/source/net/yacy/document/parser/html/ScraperInputStream.java b/source/net/yacy/document/parser/html/ScraperInputStream.java index e722f1f93..2041aec75 100644 --- a/source/net/yacy/document/parser/html/ScraperInputStream.java +++ b/source/net/yacy/document/parser/html/ScraperInputStream.java @@ -109,7 +109,11 @@ public class ScraperInputStream extends InputStream implements ScraperListener { if (tagname == null || tagname.isEmpty()) return;
if (tagname.equalsIgnoreCase("meta")) {
- if (tagopts.containsKey("http-equiv")) {
+ if (tagopts.containsKey("charset")) {
+ // parse lines like <meta charset="UTF-8">
+ this.detectedCharset = tagopts.getProperty("charset");
+ this.charsetChanged = true;
+ } else if (tagopts.containsKey("http-equiv")) {
final String value = tagopts.getProperty("http-equiv");
if (value.equalsIgnoreCase("Content-Type")) {
// parse lines like <meta http-equiv="content-type" content="text/html; charset=ISO-8859-1">
@@ -117,10 +121,6 @@ public class ScraperInputStream extends InputStream implements ScraperListener { this.detectedCharset = extractCharsetFromMimetypeHeader(contentType);
if (this.detectedCharset != null && this.detectedCharset.length() > 0) {
this.charsetChanged = true;
- } else if (tagopts.containsKey("charset")) {
- // sometimes the charset property is configured as extra attribut. try it ...
- this.detectedCharset = tagopts.getProperty("charset");
- this.charsetChanged = true;
}
}
}
|