From e41d046a9d4cdbcfbbd297da817a853b8415f058 Mon Sep 17 00:00:00 2001 From: luccioman Date: Mon, 21 Aug 2017 09:38:20 +0200 Subject: Improved parsing support for OOXML spreadsheets (.xlsx) As reported edycop in mantis 765 ( http://mantis.tokeek.de/view.php?id=765 ), parsing of xlsx files was quite incomplete. Now properly support "Shared String Table" entry in Office Open XML spreadsheets, an also detect embedded URLs. Integrating the Apache poi-ooxml library could be an option for finer OOXML formats support, but their SAX style parsing example ( http://poi.apache.org/spreadsheet/how-to.html#xssf_sax_api ) tends to show that a custom SAX handler is still efficient for lightweight and low memory footprint processing. --- test/java/net/yacy/document/ParserTest.java | 57 -------- .../net/yacy/document/parser/ooxmlParserTest.java | 160 +++++++++++++++++++++ test/parsertest/umlaute_linux.ppsx | Bin 0 -> 150435 bytes test/parsertest/umlaute_linux.xlsx | Bin 0 -> 6461 bytes 4 files changed, 160 insertions(+), 57 deletions(-) create mode 100644 test/java/net/yacy/document/parser/ooxmlParserTest.java create mode 100644 test/parsertest/umlaute_linux.ppsx create mode 100644 test/parsertest/umlaute_linux.xlsx (limited to 'test') diff --git a/test/java/net/yacy/document/ParserTest.java b/test/java/net/yacy/document/ParserTest.java index 9c2e40879..c78ca12f9 100644 --- a/test/java/net/yacy/document/ParserTest.java +++ b/test/java/net/yacy/document/ParserTest.java @@ -11,7 +11,6 @@ import java.net.MalformedURLException; import net.yacy.cora.document.id.AnchorURL; import net.yacy.document.parser.docParser; import net.yacy.document.parser.odtParser; -import net.yacy.document.parser.ooxmlParser; import net.yacy.document.parser.pdfParser; import net.yacy.document.parser.pptParser; import static org.hamcrest.CoreMatchers.containsString; @@ -21,62 +20,6 @@ import org.junit.Test; public class ParserTest { - @Test public void testooxmlParsers() throws FileNotFoundException, Parser.Failure, MalformedURLException, UnsupportedEncodingException, IOException { - final String[][] testFiles = new String[][] { - // meaning: filename in test/parsertest, mimetype, title, creator, description, - new String[]{"umlaute_windows.docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen", "", ""}, - new String[]{"umlaute_windows.pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "Folie 1", "", ""}, - }; - - for (final String[] testFile : testFiles) { - FileInputStream inStream = null; - final String filename = "test/parsertest/" + testFile[0]; - try { - final File file = new File(filename); - final String mimetype = testFile[1]; - final AnchorURL url = new AnchorURL("http://localhost/"+filename); - - AbstractParser p = new ooxmlParser(); - inStream = new FileInputStream(file); - final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, inStream); - for (final Document doc: docs) { - Reader content = null; - try { - content = new InputStreamReader(doc.getTextStream(), doc.getCharset()); - final StringBuilder str = new StringBuilder(); - int c; - while( (c = content.read()) != -1 ) - str.append((char)c); - - System.out.println("Parsed " + filename + ": " + str); - assertThat(str.toString(), containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen")); - assertThat(doc.dc_title(), containsString(testFile[2])); - assertThat(doc.dc_creator(), containsString(testFile[3])); - if (testFile[4].length() > 0) assertThat(doc.dc_description()[0], containsString(testFile[4])); - } finally { - if(content != null) { - try { - content.close(); - } catch(IOException ioe) { - System.out.println("Could not close text input stream"); - } - } - } - } - } catch (final InterruptedException ex) { - - } finally { - if(inStream != null) { - try { - inStream.close(); - } catch(IOException ioe) { - System.out.println("Could not close input stream on file " + filename); - } - } - } - } - } - @Test public void testodtParsers() throws FileNotFoundException, Parser.Failure, MalformedURLException, UnsupportedEncodingException, IOException { final String[][] testFiles = new String[][] { // meaning: filename in test/parsertest, mimetype, title, creator, description, diff --git a/test/java/net/yacy/document/parser/ooxmlParserTest.java b/test/java/net/yacy/document/parser/ooxmlParserTest.java new file mode 100644 index 000000000..1839f2fc9 --- /dev/null +++ b/test/java/net/yacy/document/parser/ooxmlParserTest.java @@ -0,0 +1,160 @@ +// ooxmlParserTest.java +// --------------------------- +// Copyright 2017 by luccioman; https://github.com/luccioman +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.document.parser; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.Collection; + +import org.junit.Test; + +import net.yacy.cora.document.id.AnchorURL; +import net.yacy.document.AbstractParser; +import net.yacy.document.Document; +import net.yacy.document.VocabularyScraper; + +/** + * Unit tests for the {@link ooxmlParser} class + * + * @author luccioman + * + */ +public class ooxmlParserTest { + + /** + * Unit test for the ooxmlParser.parse() function with some small tests + * documents. + * + * @throws Exception + * when an unexpected error occurred + */ + @Test + public void testParse() throws Exception { + final String[][] testFiles = new String[][] { + // meaning: filename in test/parsertest, mimetype, title, creator, description + new String[] { "umlaute_windows.docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen", "", "" }, + new String[] { "umlaute_mac.docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "", "", "" }, + new String[] { "umlaute_windows.pptx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", "Folie 1", "", + "" }, + new String[] { "umlaute_mac.pptx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", "Slide 1", "", + "" }, + new String[] { "umlaute_linux.ppsx", + "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "Office Open XML test slideshow from LibreOffice on Linux", "", "" }, + new String[] { "umlaute_mac.xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "", "", "" }, + new String[] { "umlaute_windows.xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "", "", "" }, + new String[] { "umlaute_linux.xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "Office Open XML Spreadsheet test document from LibreOffice Calc on Linux", "", + "Test spreadsheet document for YaCy ooxml parser" } }; + + for (final String[] testFile : testFiles) { + FileInputStream inStream = null; + final String filename = testFile[0]; + try { + final File file = new File("test" + File.separator + "parsertest" + File.separator + filename); + final String mimetype = testFile[1]; + final AnchorURL url = new AnchorURL("http://localhost/" + filename); + + AbstractParser p = new ooxmlParser(); + inStream = new FileInputStream(file); + final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, inStream); + for (final Document doc : docs) { + Reader content = null; + try { + content = new InputStreamReader(doc.getTextStream(), doc.getCharset()); + final StringBuilder str = new StringBuilder(); + int c; + while ((c = content.read()) != -1) + str.append((char) c); + + System.out.println("Parsed " + filename + ": " + str); + assertThat(str.toString(), + containsString("In München steht ein Hofbräuhaus, dort gibt es Bier in Maßkrügen")); + assertThat(doc.dc_title(), containsString(testFile[2])); + assertThat(doc.dc_creator(), containsString(testFile[3])); + if (testFile[4].length() > 0) + assertThat(doc.dc_description()[0], containsString(testFile[4])); + } finally { + if (content != null) { + try { + content.close(); + } catch (IOException ioe) { + System.out.println("Could not close text input stream"); + } + } + } + } + } finally { + if (inStream != null) { + try { + inStream.close(); + } catch (IOException ioe) { + System.out.println("Could not close input stream on file " + filename); + } + } + } + } + } + + /** + * Test URLs detection on the ooxmlParser.parse() function. + * @throws Exception when an unexpected error occurred + */ + @Test + public void testParseURLs() throws Exception { + final String fileName = "umlaute_linux.xlsx"; + final File file = new File("test" + File.separator + "parsertest" + File.separator + fileName); + final String mimetype = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + final AnchorURL url = new AnchorURL("http://localhost/" + fileName); + + AbstractParser p = new ooxmlParser(); + try(InputStream inStream = new FileInputStream(file);) { + final Document[] docs = p.parse(url, mimetype, null, new VocabularyScraper(), 0, inStream); + assertNotNull("Documents result must not be null", docs); + final Collection anchors = docs[0].getAnchors(); + assertNotNull("Detected URLs must not be null", anchors); + assertEquals("2 URLs should be detected", 2, anchors.size()); + assertTrue("YaCy home page URL should have been parsed", anchors.contains(new AnchorURL("http://yacy.net"))); + assertTrue("YaCy forum URL should have been parsed", anchors.contains(new AnchorURL("http://forum.yacy-websuche.de/"))); + } + } + +} diff --git a/test/parsertest/umlaute_linux.ppsx b/test/parsertest/umlaute_linux.ppsx new file mode 100644 index 000000000..16d253a65 Binary files /dev/null and b/test/parsertest/umlaute_linux.ppsx differ diff --git a/test/parsertest/umlaute_linux.xlsx b/test/parsertest/umlaute_linux.xlsx new file mode 100644 index 000000000..909a6a213 Binary files /dev/null and b/test/parsertest/umlaute_linux.xlsx differ -- cgit v1.2.3