1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
package net.yacy.document.parser;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.net.MalformedURLException;
import java.nio.charset.Charset;
import java.util.List;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
import junit.framework.TestCase;
import net.yacy.cora.document.id.AnchorURL;
import net.yacy.document.Document;
import net.yacy.document.Parser;
import net.yacy.document.parser.html.ContentScraper;
import net.yacy.document.parser.html.ImageEntry;
import static net.yacy.document.parser.htmlParser.parseToScraper;
import org.junit.Test;
public class htmlParserTest extends TestCase {
@Test
public void testGetRealCharsetEncoding() {
String[][] testStrings = new String[][] {
new String[]{null,null},
new String[]{"windows1250","windows-1250"},
new String[]{"windows_1250","windows-1250"},
new String[]{"ISO-8859-1","ISO-8859-1"},
new String[]{"ISO8859-1","ISO-8859-1"},
new String[]{"ISO-88591","ISO-8859-1"},
new String[]{"ISO88591","ISO-8859-1"},
new String[]{"iso_8859_1","ISO-8859-1"},
new String[]{"cp-1252","windows-1252"},
new String[]{"gb_2312","gb2312"}, // was: x-EUC-CN
new String[]{"gb_2312-80","gb2312"}, // was: x-EUC-CN
new String[]{"UTF-8;","UTF-8"}
};
for (int i=0; i < testStrings.length; i++) {
// desired conversion result
String shouldBe = testStrings[i][1];
shouldBe = shouldBe!=null ? shouldBe.toLowerCase() : null;
// conversion result
String charset = htmlParser.patchCharsetEncoding(testStrings[i][0]);
// test if equal
assertEquals(shouldBe, charset!=null ? charset.toLowerCase() : null);
System.out.println("testGetRealCharsetEncoding: " + (testStrings[i][0]!=null?testStrings[i][0]:"null") + " -> " + (charset!=null?charset:"null") + " | Supported: " + (charset!=null?Charset.isSupported(charset):false));
}
}
/**
* Test of parse method, of class htmlParser.
* - test getCharset
*/
@Test
public void testParse() throws MalformedURLException, Parser.Failure, InterruptedException, FileNotFoundException {
System.out.println("htmlParser.parse");
String[] testFiles = {
"umlaute_html_iso.html",
"umlaute_html_utf8.html",
"umlaute_html_namedentities.html"};
final String mimetype = "text/html";
//final String resulttxt = "In München steht ein Hofbräuhaus. Dort gibt es Bier aus Maßkrügen.";
for (String testfile : testFiles) {
final String filename = "test/parsertest/" + testfile;
final File file = new File(filename);
final AnchorURL url = new AnchorURL("http://localhost/" + filename);
System.out.println("parse file: " + filename);
htmlParser p = new htmlParser();
final Document[] docs = p.parse(url, mimetype, null, new FileInputStream(file));
Document doc = docs[0];
String txt = doc.getCharset();
assertTrue("get Charset", txt != null);
System.out.println("detected charset = " + txt);
}
}
/**
* Test of parseToScraper method, of class htmlParser.
*/
@Test
public void testParseToScraper_4args() throws Exception {
// test link with inline html in text
// expectation to deliver pure text as it is possibly indexed in outboundlinks_anchortext_txt/inboundlinks_anchortext_txt
final AnchorURL url = new AnchorURL("http://localhost/");
final String mimetype = "text/html";
final String testhtml = "<html><body>"
+ "<a href='x1.html'><span>testtext</span></a>" // "testtext"
+ "<a href=\"http://localhost/x2.html\"> <i id=\"home-icon\" class=\"img-sprite\"></i>Start</a>" // "Start"
+ "<a href='x1.html'><span class='button'><img src='pic.gif'/></span></a>" // "" + image
+ "<figure><img width=\"550px\" title=\"image as exemple\" alt=\"image as exemple\" src=\"./img/my_image.png\"></figrue>" // + img width 550 (+html5 figure)
+ "</body></html>";
ContentScraper scraper = parseToScraper(url, mimetype, testhtml, 10);
List<AnchorURL> anchorlist = scraper.getAnchors();
String linktxt = anchorlist.get(0).getTextProperty();
assertEquals("testtext", linktxt);
linktxt = anchorlist.get(1).getTextProperty();
assertEquals("Start", linktxt);
linktxt = anchorlist.get(2).getTextProperty();
assertEquals("", linktxt);
int cnt = scraper.getImages().size();
assertEquals(2,cnt);
ImageEntry img = scraper.getImages().get(1);
assertEquals(550,img.width());
}
}
|
