From e115e57cc7a3bb34406656a701f33f013380691c Mon Sep 17 00:00:00 2001
From: luccioman <luccioman@users.noreply.github.com>
Date: Fri, 11 May 2018 15:42:53 +0200
Subject: Reduced text snippet extraction processing time.

By not generating MD5 hashes on all words of indexed texts, processing
time is reduced by 30 to 50% on indexed documents with more than 1Mbytes
of plain text.
---
 .../net/yacy/search/snippet/TextSnippetTest.java   | 90 +++++++++++++++++++---
 1 file changed, 81 insertions(+), 9 deletions(-)

(limited to 'test')

diff --git a/test/java/net/yacy/search/snippet/TextSnippetTest.java b/test/java/net/yacy/search/snippet/TextSnippetTest.java
index 0a708e186..f572ee0e2 100644
--- a/test/java/net/yacy/search/snippet/TextSnippetTest.java
+++ b/test/java/net/yacy/search/snippet/TextSnippetTest.java
@@ -1,20 +1,31 @@
 
 package net.yacy.search.snippet;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
 import java.net.MalformedURLException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.solr.common.SolrDocument;
+import org.junit.Before;
+import org.junit.Test;
+
 import net.yacy.cora.document.encoding.ASCII;
 import net.yacy.cora.document.id.DigestURL;
 import net.yacy.cora.federate.yacy.CacheStrategy;
-import net.yacy.cora.storage.HandleSet;
+import net.yacy.cora.protocol.Domains;
 import net.yacy.cora.util.CommonPattern;
+import net.yacy.cora.util.ConcurrentLog;
 import net.yacy.kelondro.data.meta.URIMetadataNode;
 import net.yacy.search.query.QueryGoal;
 import net.yacy.search.query.SearchEvent;
 import net.yacy.search.schema.CollectionSchema;
-import org.apache.solr.common.SolrDocument;
-import static org.junit.Assert.*;
-import org.junit.Before;
-import org.junit.Test;
 
 
 public class TextSnippetTest {
@@ -55,12 +66,12 @@ public class TextSnippetTest {
 
         String querywords = "testcase line";
         QueryGoal qg = new QueryGoal(querywords);
-        HandleSet queryhashes = qg.getIncludeHashes();
 
         TextSnippet ts = new TextSnippet(
                 null,
                 testpage,
-                queryhashes,
+                qg.getIncludeWordsSet(),
+                qg.getIncludeHashes(),
                 cacheStrategy,
                 pre,
                 snippetMaxLength,
@@ -95,12 +106,12 @@ public class TextSnippetTest {
 
         String querywords = "testcase line";
         QueryGoal qg = new QueryGoal(querywords);
-        HandleSet queryhashes = qg.getIncludeHashes();
 
         TextSnippet ts = new TextSnippet(
                 null,
                 testpage,
-                queryhashes,
+                qg.getIncludeWordsSet(),
+                qg.getIncludeHashes(),
                 cacheStrategy,
                 pre,
                 snippetMaxLength,
@@ -166,4 +177,65 @@ public class TextSnippetTest {
         assertTrue ("number (.) broken up",sniptxt.contains("1.83"));
         assertTrue ("number (,) broken up",sniptxt.contains("3,14"));
     }
+    
+	/**
+	 * Run text snippet extraction from a given plain text file.
+	 * @param args <ol><li>first element : the plain text file path. When not specified, "test/parsertest/umlaute_linux.txt" is used as default.</li>
+	 * <li>other elements : the search terms. When not specified, "Maßkrügen" is used as default</li>
+	 * </ol>
+	 * @throws IOException when a read/write error occurred
+	 */
+	public static void main(final String args[]) throws IOException {
+		try {
+			final SolrDocument doc = new SolrDocument();
+			final DigestURL url = new DigestURL("http://localhost/page.html");
+			doc.addField(CollectionSchema.id.name(), ASCII.String(url.hash()));
+			doc.addField(CollectionSchema.sku.name(), url.toNormalform(false));
+
+			final URIMetadataNode urlEntry = new URIMetadataNode(doc);
+			urlEntry.addField(CollectionSchema.title.name(), "New test case");
+			urlEntry.addField(CollectionSchema.keywords.name(), "junit");
+			urlEntry.addField(CollectionSchema.author.name(), "test author");
+			
+			final Path testFilePath;
+			if(args.length > 0) {
+				testFilePath = Paths.get(args[0]);
+			} else {
+				testFilePath = Paths.get("test/parsertest/umlaute_linux.txt");
+			}
+			
+			urlEntry.addField(CollectionSchema.text_t.name(), new String(Files.readAllBytes(testFilePath),
+					StandardCharsets.UTF_8));
+			
+			final StringBuilder queryWords = new StringBuilder();
+			if(args.length > 1) {
+				for(int i = 1; i < args.length; i++) {
+					if(queryWords.length() > 0) {
+						queryWords.append(" ");
+					}
+					queryWords.append(args[i]);	
+				}
+			} else {
+				queryWords.append("Maßkrügen");
+			}
+
+			final QueryGoal goal = new QueryGoal(queryWords.toString());
+			
+			System.out.println("Extracting text snippet for terms \"" + queryWords + "\" from file " + testFilePath);
+			
+			TextSnippet.statistics.setEnabled(true);
+			final TextSnippet snippet = new TextSnippet(null, urlEntry, goal.getIncludeWordsSet(), goal.getIncludeHashes(),
+					CacheStrategy.CACHEONLY, false, SearchEvent.SNIPPET_MAX_LENGTH, false);
+			System.out.println("Snippet initialized in " + TextSnippet.statistics.getMaxInitTime() + "ms");
+			System.out.println("Snippet status : " + snippet.getErrorCode());
+			System.out.println("Snippet : " + snippet.descriptionline(goal));
+		} finally {
+			/* Shutdown running threads */
+			try {
+				Domains.close();
+			} finally {
+				ConcurrentLog.shutdown();
+			}
+		}
+	}
 }
-- 
cgit v1.2.3