diff options
38 files changed, 949 insertions, 1554 deletions
diff --git a/build.properties b/build.properties index 5c6e0b5d4..ebaa67364 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.4 javacTarget=1.4
# Release Configuration
-releaseVersion=0.542
+releaseVersion=0.543
releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseFileParentDir=yacy
diff --git a/htroot/IndexControl_p.java b/htroot/IndexControl_p.java index 1fa294789..c8543a8aa 100644 --- a/htroot/IndexControl_p.java +++ b/htroot/IndexControl_p.java @@ -123,7 +123,7 @@ public class IndexControl_p { if (delurl || delurlref) {
// generate an urlx array
indexContainer index = null;
- index = switchboard.wordIndex.getContainer(keyhash, null, -1);
+ index = switchboard.wordIndex.getContainer(keyhash, null);
Iterator en = index.entries();
int i = 0;
urlx = new String[index.size()];
@@ -250,7 +250,7 @@ public class IndexControl_p { indexContainer index;
String result;
long starttime = System.currentTimeMillis();
- index = switchboard.wordIndex.getContainer(keyhash, null, -1);
+ index = switchboard.wordIndex.getContainer(keyhash, null);
// built urlCache
Iterator urlIter = index.entries();
HashMap knownURLs = new HashMap();
@@ -490,7 +490,7 @@ public class IndexControl_p { serverObjects prop = new serverObjects();
indexContainer index = null;
try {
- index = switchboard.wordIndex.getContainer(keyhash, null, -1);
+ index = switchboard.wordIndex.getContainer(keyhash, null);
prop.put("genUrlList_keyHash", keyhash);
diff --git a/htroot/PerformanceSearch_p.html b/htroot/PerformanceSearch_p.html index b67663370..5df066fb4 100644 --- a/htroot/PerformanceSearch_p.html +++ b/htroot/PerformanceSearch_p.html @@ -7,105 +7,24 @@ <body id="PerformanceSearch">
#%env/templates/header.template%#
#%env/templates/submenuConfig.template%#
- <h2>Timing Settings of Search Sequence</h2>
-
- <p>
- <strong>Settings for local search profile:</strong>
- </p>
- <form action="PerformanceSearch_p.html" method="post" enctype="multipart/form-data">
- <table border="0" cellpadding="2" cellspacing="1">
- <tr class="TableHeader" valign="bottom">
- <td>Entity</td>
- <td>Collection</td>
- <td>Join</td>
- <td>Pre-Sort</td>
- <td>URL Fetch</td>
- <td>Post-Sort</td>
- <td>Filter</td>
- <td>Snippet-Fetch</td>
- </tr>
- <tr class="TableCellDark">
- <td align="left">execution Time (percentage; sum of this must be 100)</td>
- <td align="right"><input name="searchProcessLocalTime_c" type="text" size="8" maxlength="8" value="#[searchProcessLocalTime_c]#" /></td>
- <td align="right"><input name="searchProcessLocalTime_j" type="text" size="8" maxlength="8" value="#[searchProcessLocalTime_j]#" /></td>
- <td align="right"><input name="searchProcessLocalTime_r" type="text" size="8" maxlength="8" value="#[searchProcessLocalTime_r]#" /></td>
- <td align="right"><input name="searchProcessLocalTime_u" type="text" size="8" maxlength="8" value="#[searchProcessLocalTime_u]#" /></td>
- <td align="right"><input name="searchProcessLocalTime_o" type="text" size="8" maxlength="8" value="#[searchProcessLocalTime_o]#" /></td>
- <td align="right"><input name="searchProcessLocalTime_f" type="text" size="8" maxlength="8" value="#[searchProcessLocalTime_f]#" /></td>
- <td align="right"><input name="searchProcessLocalTime_s" type="text" size="8" maxlength="8" value="#[searchProcessLocalTime_s]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td align="left">result count (percentage of requested amount)</td>
- <td align="right"><input name="searchProcessLocalCount_c" type="text" size="8" maxlength="8" value="#[searchProcessLocalCount_c]#" /></td>
- <td align="right"><input name="searchProcessLocalCount_j" type="text" size="8" maxlength="8" value="#[searchProcessLocalCount_j]#" /></td>
- <td align="right"><input name="searchProcessLocalCount_r" type="text" size="8" maxlength="8" value="#[searchProcessLocalCount_r]#" /></td>
- <td align="right"><input name="searchProcessLocalCount_u" type="text" size="8" maxlength="8" value="#[searchProcessLocalCount_u]#" /></td>
- <td align="right"><input name="searchProcessLocalCount_o" type="text" size="8" maxlength="8" value="#[searchProcessLocalCount_o]#" /></td>
- <td align="right"><input name="searchProcessLocalCount_f" type="text" size="8" maxlength="8" value="#[searchProcessLocalCount_f]#" /></td>
- <td align="right"><input name="searchProcessLocalCount_s" type="text" size="8" maxlength="8" value="#[searchProcessLocalCount_s]#" /></td>
- </tr>
- <tr class="TableCellLight">
- <td align="left" colspan="8">
- <input type="submit" name="submitlocalprofilecustom" value="Submit New Profile Values" />
- <input type="submit" name="submitlocalprofiledefault" value="Reset To Default Values" />
- </td>
- </tr>
- </table>
- </form>
-
- #(submitlocalrespond)#
- ::
- <p>Your settings are valid and will be used for next search.</p>
- ::
- <p>Reset to default settings done.</p>
- ::
- <p>Your settings cannot be accepted: sum of execution time percentage is not 100</p>
- #(/submitlocalrespond)#
+ <h2>Search Sequence Timing</h2>
<p>
<strong>Timing results of latest search request:</strong>
</p>
- <table border="0" cellpadding="2" cellspacing="1">
- <tr class="TableHeader" valign="bottom">
- <td>Entity</td>
- <td>Collection</td>
- <td>Join</td>
- <td>Pre-Sort</td>
- <td>URL Fetch</td>
- <td>Post-Sort</td>
- <td>Filter</td>
- <td>Snippet-Fetch</td>
- </tr>
- <tr class="TableCellDark">
- <td align="left">execution Time (absolute milliseconds)</td>
- <td align="right">#[latestLocalTimeAbs_c]#</td>
- <td align="right">#[latestLocalTimeAbs_j]#</td>
- <td align="right">#[latestLocalTimeAbs_r]#</td>
- <td align="right">#[latestLocalTimeAbs_u]#</td>
- <td align="right">#[latestLocalTimeAbs_o]#</td>
- <td align="right">#[latestLocalTimeAbs_f]#</td>
- <td align="right">#[latestLocalTimeAbs_s]#</td>
- </tr>
- <tr class="TableCellDark">
- <td align="left">execution Time (relative)</td>
- <td align="right">#[latestLocalTimeRel_c]#</td>
- <td align="right">#[latestLocalTimeRel_j]#</td>
- <td align="right">#[latestLocalTimeRel_r]#</td>
- <td align="right">#[latestLocalTimeRel_u]#</td>
- <td align="right">#[latestLocalTimeRel_o]#</td>
- <td align="right">#[latestLocalTimeRel_f]#</td>
- <td align="right">#[latestLocalTimeRel_s]#</td>
- </tr>
- <tr class="TableCellDark">
- <td align="left">result count (absolute amount)</td>
- <td align="right">#[latestLocalCountAbs_c]#</td>
- <td align="right">#[latestLocalCountAbs_j]#</td>
- <td align="right">#[latestLocalCountAbs_r]#</td>
- <td align="right">#[latestLocalCountAbs_u]#</td>
- <td align="right">#[latestLocalCountAbs_o]#</td>
- <td align="right">#[latestLocalCountAbs_f]#</td>
- <td align="right">#[latestLocalCountAbs_s]#</td>
- </tr>
+ <table border="0" cellpadding="2" cellspacing="1"> + <tr class="TableHeader" valign="bottom"> + <td>Event</td> + <td>Time (milliseconds)</td> + <td>Result-Count</td> + </tr> + #{table}#
+ <tr class="TableCellDark" valign="bottom">
+ <td>#[event]#</td> + <td>#[time]#</td>
+ <td>#[count]#</td>
+ </tr> + #{/table}#
</table>
<p>
diff --git a/htroot/PerformanceSearch_p.java b/htroot/PerformanceSearch_p.java index 6e9221ed4..8307a31db 100644 --- a/htroot/PerformanceSearch_p.java +++ b/htroot/PerformanceSearch_p.java @@ -1,56 +1,35 @@ -//PerformaceSearch_p.java
-//-----------------------
-//part of YaCy
-//(C) by Michael Peter Christen; mc@anomic.de
-//first published on http://www.anomic.de
-//Frankfurt, Germany, 2004, 2005
-//last major change: 16.02.2005
+// PerformaceSearch_p.java
+// (C) 2004, 2005 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// first published 16.02.2005 on http://yacy.net
//
-//This program is free software; you can redistribute it and/or modify
-//it under the terms of the GNU General Public License as published by
-//the Free Software Foundation; either version 2 of the License, or
-//(at your option) any later version.
+// This is a part of YaCy, a peer-to-peer based web search engine
//
-//This program is distributed in the hope that it will be useful,
-//but WITHOUT ANY WARRANTY; without even the implied warranty of
-//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//GNU General Public License for more details.
+// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedRevision: 1986 $
+// $LastChangedBy: orbiter $
//
-//You should have received a copy of the GNU General Public License
-//along with this program; if not, write to the Free Software
-//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
//
-//Using this software in any meaning (reading, learning, copying, compiling,
-//running) means that you agree that the Author(s) is (are) not responsible
-//for cost, loss of data or any harm that may be caused directly or indirectly
-//by usage of this softare or this documentation. The usage of this software
-//is on your own risk. The installation and usage (starting/running) of this
-//software may allow other people or application to access your computer and
-//any attached devices and is highly dependent on the configuration of the
-//software which must be done by the user of the software; the author(s) is
-//(are) also not responsible for proper configuration and usage of the
-//software, even if provoked by documentation provided together with
-//the software.
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
//
-//Any changes to this file according to the GPL as documented in the file
-//gpl.txt aside this file in the shipment you received can be done to the
-//lines that follows this copyright notice here, but changes must not be
-//done inside the copyright notive above. A re-distribution must contain
-//the intact and unchanged copyright notice.
-//Contributions and changes to the program code must be marked as such.
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//You must compile this file with
-//javac -classpath .:../classes Network.java
-//if the shell's current path is HTROOT
-import java.io.File;
-import java.util.Map;
+import java.util.Iterator;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSearchEvent;
import de.anomic.plasma.plasmaSearchProcessing;
-import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@@ -58,69 +37,25 @@ public class PerformanceSearch_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) {
// return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) sb;
serverObjects prop = new serverObjects();
- File defaultSettingsFile = new File(switchboard.getRootPath(), "yacy.init");
- Map defaultSettings = ((post == null) || (!(post.containsKey("submitlocalprofiledefault")))) ? null : serverFileUtils.loadHashMap(defaultSettingsFile);
-
- prop.put("submitlocalrespond", 0);
-
- // execute commands
- if (post != null) {
- if (post.containsKey("submitlocalprofilecustom")) {
- // first count percentages
- int c = 0;
- for (int i = 0; i < plasmaSearchProcessing.sequence.length; i++) {
- c += post.getInt("searchProcessLocalTime_" + plasmaSearchProcessing.sequence[i], 0);
- }
- // if check is ok set new values
- if (c == 100) {
- for (int i = 0; i < plasmaSearchProcessing.sequence.length; i++) {
- sb.setConfig("searchProcessLocalTime_" + plasmaSearchProcessing.sequence[i], post.get("searchProcessLocalTime_" + plasmaSearchProcessing.sequence[i], ""));
- sb.setConfig("searchProcessLocalCount_" + plasmaSearchProcessing.sequence[i], post.get("searchProcessLocalCount_" + plasmaSearchProcessing.sequence[i], ""));
- }
- prop.put("submitlocalrespond", 1);
- } else {
- prop.put("submitlocalrespond", 3);
- }
- }
- if (post.containsKey("submitlocalprofiledefault")) {
- for (int i = 0; i < plasmaSearchProcessing.sequence.length; i++) {
- sb.setConfig("searchProcessLocalTime_" + plasmaSearchProcessing.sequence[i], (String) defaultSettings.get("searchProcessLocalTime_" + plasmaSearchProcessing.sequence[i]));
- sb.setConfig("searchProcessLocalCount_" + plasmaSearchProcessing.sequence[i], (String) defaultSettings.get("searchProcessLocalCount_" + plasmaSearchProcessing.sequence[i]));
- }
- prop.put("submitlocalrespond", 2);
- }
- }
-
- // prepare values
plasmaSearchEvent se = plasmaSearchEvent.getEvent(plasmaSearchEvent.lastEventID);
- // count complete execution time
- long time = 0;
- long t;
- int c;
- char sequence;
- if (se != null) for (int i = 0; i < plasmaSearchProcessing.sequence.length; i++) {
- t = se.getLocalTiming().getYieldTime(plasmaSearchProcessing.sequence[i]);
- if (t > 0) time += t;
+ if (se == null) {
+ prop.put("table", 0);
+ return prop;
}
- for (int i = 0; i < plasmaSearchProcessing.sequence.length; i++) {
- sequence = plasmaSearchProcessing.sequence[i];
- prop.put("searchProcessLocalTime_" + sequence, sb.getConfig("searchProcessLocalTime_" + sequence, ""));
- prop.put("searchProcessLocalCount_" + sequence, sb.getConfig("searchProcessLocalCount_" + sequence, ""));
- if (se == null) {
- prop.put("latestLocalTimeAbs_" + sequence, "-");
- prop.put("latestLocalTimeRel_" + sequence, "-");
- prop.put("latestLocalCountAbs_" + sequence, "-");
- } else {
- t = se.getLocalTiming().getYieldTime(sequence);
- prop.put("latestLocalTimeAbs_" + sequence, (t < 0) ? "-" : Long.toString(t));
- prop.put("latestLocalTimeRel_" + sequence, ((t < 0 || time == 0) ? 0 : (t * 100 / time)) + "%");
- c = se.getLocalTiming().getYieldCount(sequence);
- prop.put("latestLocalCountAbs_" + sequence, (c < 0) ? "-" : Integer.toString(c));
- }
+
+ Iterator events = se.getProcess().events();
+ int c = 0;
+ plasmaSearchProcessing.Entry event;
+ while (events.hasNext()) {
+ event = (plasmaSearchProcessing.Entry) events.next();
+ prop.put("table_" + c + "_event", event.process);
+ prop.put("table_" + c + "_count", event.count);
+ prop.put("table_" + c + "_time", event.time);
+ c++;
}
+ prop.put("table", c);
return prop;
}
diff --git a/htroot/index.html b/htroot/index.html index 5e1b263c1..d3960e2e9 100644 --- a/htroot/index.html +++ b/htroot/index.html @@ -33,6 +33,7 @@ <input type="radio" id="app" name="contentdom" value="app" #(contentdomCheckApp)#::checked="checked"#(/contentdomCheckApp)# /><label for="app">Applications</label> #(searchoptions)#<!-- default values are hard-coded <input type="hidden" name="count" value="10" /> + <input type="hidden" name="offset" value="0" /> <input type="hidden" name="resource" value="global" /> <input type="hidden" name="time" value="6" /> <input type="hidden" name="urlmaskfilter" value=".*" /> diff --git a/htroot/js/yacysearch.js b/htroot/js/yacysearch.js index f93c4e43c..b55c951a2 100644 --- a/htroot/js/yacysearch.js +++ b/htroot/js/yacysearch.js @@ -51,64 +51,6 @@ function Progressbar(length, parent) { parent.appendChild(this.element); } -function AllTextSnippets(query, eventID) { - var span = document.getElementsByTagName("span"); - for(var x=0;x<span.length;x++) { - if (span[x].className == 'snippetLoading') { - var url = document.getElementById("url" + span[x].id.substring(1)); - requestTextSnippet(url, query, eventID); - } - } -} - -function AllMediaSnippets(urls, query, mediatype) { - document.getElementById("linkcount").innerHTML = 0; - var container = document.getElementById("results"); - var progressbar = new Progressbar(urls.length, container); - for (url in urls) { - requestMediaSnippet(urls[url],query,mediatype,progressbar); - } -} - -function AllImageSnippets(urls, query) { - document.getElementById("linkcount").innerHTML = 0; - var container = document.getElementById("results"); - var progressbar = new Progressbar(urls.length, container); - for(url in urls) { - requestImageSnippet(urls[url],query,progressbar); - } -} - -function requestTextSnippet(url, query, eventID){ - var request=createRequestObject(); - request.open('get', '/xml/snippet.xml?url=' + escape(url) + '&remove=true&media=text&search=' + escape(query) + '&eventID=' + eventID,true); - request.onreadystatechange = function () {handleTextState(request)}; - request.send(null); -} - -function requestMediaSnippet(url, query, mediatype, progressbar){ - var request=createRequestObject(); - request.open('get', '/xml/snippet.xml?url=' + escape(url) + '&remove=true&media=' + escape(mediatype) + '&search=' + escape(query),true); - request.onreadystatechange = function () {handleMediaState(request, progressbar)}; - request.send(null); -} - -function requestImageSnippet(url, query, progressbar){ - var request=createRequestObject(); - request.open('get', '/xml/snippet.xml?url=' + escape(url) + '&remove=true&media=image&search=' + escape(query),true); - request.onreadystatechange = function () {handleImageState(request, progressbar)}; - request.send(null); -} - -function show_hidden_results(){ - var results = document.getElementsByTagName("div"); - for (var i = 0; i < results.length; i++) { - var result = results[i]; - if (result.className == "searchresults hidden") - result.className = "searchresults"; - } - document.getElementById("hidden_results").innerHTML = ""; -} function handleTextState(req) { if(req.readyState != 4){ return; @@ -284,3 +226,10 @@ function addHover() { } } } + +function statistics(offset, items, global, total) { + document.getElementById("offset").firstChild.nodeValue = offset; + document.getElementById("itemscount").firstChild.nodeValue = items; + document.getElementById("globalcount").firstChild.nodeValue = global; + document.getElementById("totalcount").firstChild.nodeValue = total; +}
\ No newline at end of file diff --git a/htroot/xml/snippet.java b/htroot/xml/snippet.java deleted file mode 100644 index 33856265f..000000000 --- a/htroot/xml/snippet.java +++ /dev/null @@ -1,103 +0,0 @@ -package xml;
-
-import java.net.MalformedURLException;
-import java.util.ArrayList;
-import java.util.Set;
-import java.util.TreeSet;
-
-import de.anomic.http.httpHeader;
-import de.anomic.plasma.plasmaURL;
-import de.anomic.kelondro.kelondroMSetTools;
-import de.anomic.net.URL;
-import de.anomic.plasma.plasmaCondenser;
-import de.anomic.plasma.plasmaSearchQuery;
-import de.anomic.plasma.plasmaSnippetCache;
-import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.server.serverObjects;
-import de.anomic.server.serverSwitch;
-
-public class snippet {
- public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) throws MalformedURLException {
- // return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
- serverObjects prop = new serverObjects();
-
- //get the timeout for snippet-fetching
- int mediasnippet_timeout = 15000;
- int textsnippet_timeout = 10000;
- mediasnippet_timeout = Integer.parseInt(env.getConfig("timeout_text", "15000"));
- textsnippet_timeout = Integer.parseInt(env.getConfig("timeout_media", "10000"));
-
- // getting url
- String urlString = post.get("url", "");
- URL url = new URL(urlString);
- prop.put("urlHash",plasmaURL.urlHash(url));
-
- // if 'remove' is set to true, then RWI references to URLs that do not have the snippet are removed
- boolean remove = post.get("remove", "false").equals("true");
-
- // boolean line_end_with_punctuation
- boolean pre = post.get("pre", "false").equals("true");
-
- // type of media
- String media = post.get("media", "text");
-
- String querystring = post.get("search", "").trim();
- String eventID = post.get("eventID", "").trim();
- if ((querystring.length() > 2) && (querystring.charAt(0) == '"') && (querystring.charAt(querystring.length() - 1) == '"')) {
- querystring = querystring.substring(1, querystring.length() - 1).trim();
- }
- final TreeSet[] query = plasmaSearchQuery.cleanQuery(querystring);
- Set queryHashes = plasmaCondenser.words2hashes(query[0]);
-
- // filter out stopwords
- final TreeSet filtered = kelondroMSetTools.joinConstructive(query[0], plasmaSwitchboard.stopwords);
- if (filtered.size() > 0) {
- kelondroMSetTools.excludeDestructive(query[0], plasmaSwitchboard.stopwords);
- }
-
- // find snippet
- if (media.equals("text")) {
- // attach text snippet
- plasmaSnippetCache.TextSnippet snippet = plasmaSnippetCache.retrieveTextSnippet(url, queryHashes, true, pre, 260, textsnippet_timeout);
- prop.put("status",snippet.getErrorCode());
- if (snippet.getErrorCode() < 11) {
- // no problems occurred
- //prop.put("text", (snippet.exists()) ? snippet.getLineMarked(queryHashes) : "unknown");
- prop.putASIS("text", (snippet.exists()) ? snippet.getLineMarked(queryHashes) : "unknown"); //FIXME: the ASIS should not be needed, but we have still htmlcode in .java files
- } else {
- // problems with snippet fetch
- prop.put("text", (remove) ? plasmaSnippetCache.failConsequences(snippet, eventID) : snippet.getError());
- }
- prop.put("link", 0);
- prop.put("links", 0);
- prop.put("favicon",snippet.getFavicon()==null?"":snippet.getFavicon().toString());
- } else {
- // attach media information
- ArrayList mediaSnippets = plasmaSnippetCache.retrieveMediaSnippets(url, queryHashes, media, true, mediasnippet_timeout);
- plasmaSnippetCache.MediaSnippet ms;
- for (int i = 0; i < mediaSnippets.size(); i++) {
- ms = (plasmaSnippetCache.MediaSnippet) mediaSnippets.get(i);
- try {
- url = new URL(ms.href);
- } catch (MalformedURLException e) {
- continue;
- }
- prop.put("link_" + i + "_type", ms.type);
- prop.put("link_" + i + "_href", ms.href);
- prop.put("link_" + i + "_code", switchboard.licensedURLs.aquireLicense(url));
- prop.put("link_" + i + "_name", ms.name);
- prop.put("link_" + i + "_attr", ms.attr);
- }
- //System.out.println("DEBUG: " + mediaSnippets.size() + " ENTRIES IN MEDIA SNIPPET LINKS for url " + urlString);
- prop.put("text", "");
- prop.put("link", mediaSnippets.size());
- prop.put("links", mediaSnippets.size());
- prop.put("favicon","");
- }
-
-
- // return rewrite properties
- return prop;
- }
-}
diff --git a/htroot/xml/snippet.xml b/htroot/xml/snippet.xml deleted file mode 100644 index bcf10bc7e..000000000 --- a/htroot/xml/snippet.xml +++ /dev/null @@ -1,17 +0,0 @@ -<?xml version="1.0"?>
-<snippet>
- <text><![CDATA[#[text]#]]></text>
- <status>#[status]#</status>
- <urlHash>#[urlHash]#</urlHash>
- <links>#[links]#</links>
- <favicon>#[favicon]#</favicon>
- #{link}#
- <link>
- <type>#[type]#</type>
- <href>#[href]#</href> - <code>#[code]#</code>
- <name>#[name]#</name>
- <attr>#[attr]#</attr>
- </link>
- #{/link}#
-</snippet>
diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index d1b7ee6af..b0af8ff05 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -128,12 +128,12 @@ public final class search { ArrayList accu = null;
if ((query.length() == 0) && (abstractSet != null)) {
// this is _not_ a normal search, only a request for index abstracts
- theQuery = new plasmaSearchQuery(abstractSet, new TreeSet(kelondroBase64Order.enhancedCoder), maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), count, duetime, filter, plasmaSearchQuery.catchall_constraint);
+ theQuery = new plasmaSearchQuery(null, abstractSet, new TreeSet(kelondroBase64Order.enhancedCoder), maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), false, count, 0, duetime, filter, plasmaSearchQuery.SEARCHDOM_LOCAL, null, -1, plasmaSearchQuery.catchall_constraint);
theQuery.domType = plasmaSearchQuery.SEARCHDOM_LOCAL;
- yacyCore.log.logInfo("INIT HASH SEARCH (abstracts only): " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.wantedResults + " links");
+ yacyCore.log.logInfo("INIT HASH SEARCH (abstracts only): " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
// prepare a search profile
- plasmaSearchProcessing localTiming = new plasmaSearchProcessing(theQuery.maximumTime, theQuery.wantedResults);
+ plasmaSearchProcessing localTiming = new plasmaSearchProcessing(theQuery.maximumTime, theQuery.displayResults());
//theSearch = new plasmaSearchEvent(squery, rankingProfile, localTiming, remoteTiming, true, sb.wordIndex, null);
Map[] containers = localTiming.localSearchContainers(theQuery, sb.wordIndex, plasmaSearchQuery.hashes2Set(urls));
@@ -156,16 +156,16 @@ public final class search { } else {
// retrieve index containers from search request
- theQuery = new plasmaSearchQuery(queryhashes, excludehashes, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), count, duetime, filter, constraint);
+ theQuery = new plasmaSearchQuery(null, queryhashes, excludehashes, maxdist, prefer, plasmaSearchQuery.contentdomParser(contentdom), false, count, 0, duetime, filter, plasmaSearchQuery.SEARCHDOM_LOCAL, null, -1, constraint);
theQuery.domType = plasmaSearchQuery.SEARCHDOM_LOCAL;
- yacyCore.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.wantedResults + " links");
+ yacyCore.log.logInfo("INIT HASH SEARCH (query-" + abstracts + "): " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
// prepare a search profile
plasmaSearchRankingProfile rankingProfile = (profile.length() == 0) ? new plasmaSearchRankingProfile(plasmaSearchQuery.contentdomParser(contentdom)) : new plasmaSearchRankingProfile("", profile);
- plasmaSearchProcessing localProcess = new plasmaSearchProcessing(theQuery.maximumTime, theQuery.wantedResults);
+ plasmaSearchProcessing localProcess = new plasmaSearchProcessing(theQuery.maximumTime, theQuery.displayResults());
//plasmaSearchProcessing remoteProcess = null;
- plasmaSearchEvent theSearch = plasmaSearchEvent.getEvent(theQuery, rankingProfile, localProcess, null, sb.wordIndex, null, true, abstractSet);
+ plasmaSearchEvent theSearch = plasmaSearchEvent.getEvent(theQuery, rankingProfile, localProcess, sb.wordIndex, null, true, abstractSet);
//Map[] containers = localProcess.localSearchContainers(theQuery, sb.wordIndex, plasmaSearchQuery.hashes2Set(urls));
// set statistic details of search result and find best result index set
if (theSearch.getLocalCount() == 0) {
@@ -198,7 +198,7 @@ public final class search { } else {
joincount = theSearch.getLocalCount();
prop.putASIS("joincount", Integer.toString(joincount));
- accu = theSearch.computeResults(plasmaSwitchboard.blueList, false);
+ accu = theSearch.completeResults(duetime);
}
// generate compressed index for maxcounthash
@@ -234,7 +234,7 @@ public final class search { // prepare search statistics
Long trackerHandle = new Long(System.currentTimeMillis());
String client = (String) header.get("CLIENTIP");
- HashMap searchProfile = theQuery.resultProfile((accu == null) ? 0 : accu.size(), System.currentTimeMillis() - timestamp);
+ HashMap searchProfile = theQuery.resultProfile(System.currentTimeMillis() - timestamp);
searchProfile.put("host", client);
yacySeed remotepeer = yacyCore.seedDB.lookupByIP(natLib.getInetAddress(client), true, false, false);
searchProfile.put("peername", (remotepeer == null) ? "unknown" : remotepeer.getName());
@@ -258,9 +258,9 @@ public final class search { // result is a List of urlEntry elements
StringBuffer links = new StringBuffer();
String resource = null;
- plasmaSearchEvent.Entry entry;
+ plasmaSearchEvent.ResultEntry entry;
for (int i = 0; i < accu.size(); i++) {
- entry = (plasmaSearchEvent.Entry) accu.get(i);
+ entry = (plasmaSearchEvent.ResultEntry) accu.get(i);
resource = entry.resource();
if (resource != null) {
links.append("resource").append(i).append('=').append(resource).append(serverCore.crlfString);
diff --git a/htroot/yacysearch.html b/htroot/yacysearch.html index 0bac60667..2ebd6fb15 100644 --- a/htroot/yacysearch.html +++ b/htroot/yacysearch.html @@ -29,6 +29,7 @@ <input type="hidden" name="contentdom" value="#[contentdom]#" /> <input type="hidden" name="former" value="#[former]#" /> <input type="hidden" name="count" value="#[count]#" /> + <input type="hidden" name="offset" value="#[offset]#" /> <input type="hidden" name="resource" value="#[resource]#" /> <input type="hidden" name="time" value="#[time]#" /> <input type="hidden" name="urlmaskfilter" value="#[urlmaskfilter]#" /> @@ -43,7 +44,7 @@ </form> :: <form class="search small" action="yacysearch.html" method="get" accept-charset="UTF-8"> - <p class="yacylogo"><a href="http://yacy.net/" class="yacylogo"><img src="/env/grafics/YaCyLogo_60ppi.png" alt="yacy" /></a></p> + <p class="yacylogo"><a href="http://yacy.net/yacy/" class="yacylogo"><img src="/env/grafics/YaCyLogo_60ppi.png" alt="yacy" /></a></p> <h2>#[promoteSearchPageGreeting]#</h2> <fieldset class="yacys"> <input id="search" name="search" type="text" size="50" maxlength="80" value="#[former]#" onclick="document.getElementById('Enter').value = 'Search'" /> @@ -63,6 +64,7 @@ </div> <input type="hidden" name="former" value="#[former]#" /> <input type="hidden" name="count" value="#[count]#" /> + <input type="hidden" name="offset" value="#[offset]#" /> <input type="hidden" name="resource" value="#[resource]#" /> <input type="hidden" name="time" value="#[time]#" /> <input type="hidden" name="urlmaskfilter" value="#[urlmaskfilter]#" /> @@ -97,26 +99,12 @@ document.getElementById("Enter").value = "search again - catch up more links"; :: <p>No Results. (length of search words must be at least 3 characters)</p> :: - <p>No Results. If you think this is unsatisfactory then you may consider to support the global index by running your own proxy/peer. - If everybody contributes, the results will get better.</p> - <p>Other possible reasons for no result:<p> - <ul> - <li>The search time was too short. Search again with same query to catch up 'late peers'</li> - <li>There is currently no support for german umlaute. Please use ae/oe/ue instead</li> - <li>Words of length < 3 are not indexed. Please omit such words</li> - <li>YaCy tries to index singular instead of plural words. Please use the singular form</li> - <li>Only complete words are indexed, not parts of words</li> - <li>Don't use stopwords as search words</li> - <li>During this test phase the reaction time of remote peers is unknown. - Please repeat your search to see if there are late-responses from remote peers</li> - </ul> - <p>If you think the information you searched should exist in the global index, - then please run your own peer and start a crawl of your wanted information to make it - available for everyone. Then stay online to support crawls from other peers. Thank you!</p> + <p>No Results. Please repeat your search to see if there are late-responses from remote peers.</p> :: <p>No Results. "<strong>#[wrong_regex]#</strong>" is no valid regular expression. Please go back to the previous page and make sure to enter a valid regular expressions for URL mask and Prefer mask.</p> :: - <p><strong id="linkcount">#[linkcount]#</strong> results from <strong>#[orderedcount]#</strong> ordered links from <strong>#[filteredcount]#</strong> filtered links of a total number of <strong>#[totalcount]#</strong> known#(globalresults)#.::, <strong>#[globalcount]#</strong> links from other YaCy peers.#(/globalresults)#</p> + <p><strong id="offset">#[offset]#</strong>-<strong id="itemscount">#[linkcount]#</strong> results from a total number of <strong id="totalcount">#[totalcount]#</strong> known#(globalresults)#.::, <strong id="globalcount">#[globalcount]#</strong> links from other YaCy peers.#(/globalresults)#</p> + <p>Search Result Pages: <span id="pagenav">#[pagenav]#</span></p> :: <p>Searching the web with this peer is disabled for unauthorized users. Please <a href="Status.html?login=">log in</a> as administrator to use the search function</p> #(/num-results)# @@ -126,44 +114,19 @@ document.getElementById("Enter").value = "search again - catch up more links"; <!-- type 0: text search --> #(combine)# :: - <p><strong>Refine your search with these topwords</strong>:</p> + <p><strong>Refine with</strong>:</p> <p> #{words}# - <a href="yacysearch.html?search=#[newsearch]#&Enter=Search&count=#[count]#&resource=#[resource]#&time=#[time]#">#[word]#</a> + <a href="yacysearch.html?search=#[newsearch]#&Enter=Search&count=#[count]#&offset=#[offset]#&resource=#[resource]#&time=#[time]#">#[word]#</a> #{/words}# </p> #(/combine)# <!-- linklist begin --> <div id="hidden_results"></div> #{results}# - <!-- link begin --> - <div class="searchresults"> - <img src="ViewImage.png?width=16&height=16&code=#[faviconCode]#" id="f#[urlhash]#" class="favicon" width="16" height="16" alt="favicon of #[url]#" /> - #(authorized)#:: - <div class="urlactions"> - <a href="/Bookmarks.html?edit=#[urlhash]#" class="bookmarklink" title="bookmark"><img src="/env/grafics/empty.gif" title="bookmark" alt="bookmark" class="bookmarkIcon" /></a> - #(recommend)# - <img src="/env/grafics/empty.gif" title="" alt="" class="recommendIcon" /> - <img src="/env/grafics/empty.gif" title="" alt="" class="deleteIcon" /> - :: - <a href="#[recommendlink]#" class="recommendlink" title="recommend"><img src="/env/grafics/empty.gif" title="recommend" alt="recommend" class="recommendIcon" /></a> - <a href="#[deletelink]#" title="delete" class="deletelink" ><img src="/env/grafics/empty.gif" title="delete" alt="delete" class="deleteIcon" /></a> - #(/recommend)# - </div> - #(/authorized)# - <h4 class="linktitle"><a href="#[url]#" target="_parent">#[description]#</a></h4> - <p class="snippet"><span class="#(snippet)#snippetLoading::snippetLoaded#(/snippet)#" id="h#[urlhash]#">#(snippet)#loading snippet ...::#[text]##(/snippet)#</span></p> - <p class="url"><a href="#[url]#" id="url#[urlhash]#" target="_parent">#[urlname]#</a></p> - <p class="urlinfo">#[date]# | YBR-#[ybr]# | <a href="ViewFile.html?urlHash=#[urlhash]#&words=#[words]#">Info</a> | <a href="yacysearch.html?cat=image&url=#[url]#&search=#[former]#">Pictures</a></p> - </div> - <!-- link end --> + <!--#include virtual="yacysearchitem.html?item=#[item]#&eventID=#[eventID]#&display=#[display]#" --> #{/results}# - <script type="text/javascript"> - AllTextSnippets("#[former]#", "#[eventID]#"); - addHover(); - </script> - <!-- linklist end --> ::<!-- type 1: media search --> @@ -219,6 +182,7 @@ document.getElementById("Enter").value = "search again - catch up more links"; } </script> <br style="clear:left;" /> + ::<!-- type 3: image thumbnail list for one single url --> <table border="0" cellspacing="16" cellpadding="0"> #{results}# @@ -232,27 +196,6 @@ document.getElementById("Enter").value = "search again - catch up more links"; #{/results}# </table> - <!-- type 4: experimental new text search --> - #(combine)# - :: - <p><strong>Refine your search with these topwords</strong>:</p> - <p> - #{words}# - <a href="yacysearch.html?search=#[newsearch]#&Enter=Search&count=#[count]#&resource=#[resource]#&time=#[time]#">#[word]#</a> - #{/words}# - </p> - #(/combine)# - <!-- linklist begin --> - <div id="hidden_results"></div> - #{results}# - <!-- link begin --> - <div class="searchresults"> - <!--#include virtual="yacysearchitem.html?item=#[item]#&eventID=#[eventID]#" --> - </div> - <!-- link end --> - #{/results}# - - #(/type)# </body> diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 76823fd20..f2027917d 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -47,10 +47,7 @@ // if the shell's current path is HTROOT
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
-import java.net.URLEncoder;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.TreeSet;
@@ -66,13 +63,11 @@ import de.anomic.plasma.plasmaCondenser; import de.anomic.plasma.plasmaParserDocument;
import de.anomic.plasma.plasmaSearchEvent;
import de.anomic.plasma.plasmaSearchImages;
-import de.anomic.plasma.plasmaSearchPreOrder;
import de.anomic.plasma.plasmaSearchQuery;
import de.anomic.plasma.plasmaSearchRankingProfile;
import de.anomic.plasma.plasmaSearchProcessing;
import de.anomic.plasma.plasmaSnippetCache;
import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.plasma.plasmaURL;
import de.anomic.server.serverCore;
import de.anomic.server.serverDomains;
import de.anomic.server.serverDate;
@@ -80,11 +75,9 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.crypt;
-import de.anomic.tools.nxTools;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
-import de.anomic.yacy.yacySeed;
public class yacysearch {
@@ -134,6 +127,7 @@ public class yacysearch { prop.put("input_former", "");
prop.put("former", "");
prop.put("input_count", 10);
+ prop.put("input_offset", 0);
prop.put("input_resource", "global");
prop.put("input_time", 6);
prop.put("input_urlmaskfilter", ".*");
@@ -166,10 +160,11 @@ public class yacysearch { }
if (sb.facilityDB != null) try { sb.facilityDB.update("zeitgeist", querystring, post); } catch (Exception e) {}
- int count = Integer.parseInt(post.get("count", "10"));
+ int count = post.getInt("count", 10);
+ int offset = post.getInt("offset", 0);
boolean global = (post == null) ? true : post.get("resource", "global").equals("global");
final boolean indexof = post.get("indexof","").equals("on");
- final long searchtime = 1000 * Long.parseLong(post.get("time", "10"));
+ final long searchtime = 1000 * post.getLong("time", 6);
String urlmask = "";
if (post.containsKey("urlmask") && post.get("urlmask").equals("no")) {
urlmask = ".*";
@@ -269,7 +264,9 @@ public class yacysearch { maxDistance,
prefermask,
contentdomCode,
+ true,
count,
+ offset,
searchtime,
urlmask,
(clustersearch && globalsearch) ? plasmaSearchQuery.SEARCHDOM_CLUSTERALL :
@@ -278,11 +275,8 @@ public class yacysearch { 20,
constraint);
plasmaSearchRankingProfile ranking = (sb.getConfig("rankingProfile", "").length() == 0) ? new plasmaSearchRankingProfile(contentdomCode) : new plasmaSearchRankingProfile("", crypt.simpleDecode(sb.getConfig("rankingProfile", ""), null));
- plasmaSearchProcessing localTiming = new plasmaSearchProcessing(4 * theQuery.maximumTime / 10, theQuery.wantedResults);
- plasmaSearchProcessing remoteTiming = new plasmaSearchProcessing(6 * theQuery.maximumTime / 10, theQuery.wantedResults);
+ plasmaSearchProcessing localTiming = new plasmaSearchProcessing(4 * theQuery.maximumTime / 10, theQuery.displayResults());
- //**
- //searchFromLocal(thisSearch, ranking, localTiming, remoteTiming, true, (String) header.get("CLIENTIP"))
String client = (String) header.get("CLIENTIP"); // the search client who initiated the search
// tell all threads to do nothing for a specific time
@@ -292,139 +286,83 @@ public class yacysearch { theQuery.filterOut(plasmaSwitchboard.blueList);
// log
- serverLog.logInfo("LOCAL_SEARCH", "INIT WORD SEARCH: " + theQuery.queryString + ":" + theQuery.queryHashes + " - " + theQuery.wantedResults + " links, " + (theQuery.maximumTime / 1000) + " seconds");
+ serverLog.logInfo("LOCAL_SEARCH", "INIT WORD SEARCH: " + theQuery.queryString + ":" + theQuery.queryHashes + " - " + theQuery.neededResults() + " links to be computed, " + theQuery.displayResults() + " lines to be displayed, " + (theQuery.maximumTime / 1000) + " seconds");
long timestamp = System.currentTimeMillis();
// create a new search event
String wrongregex = null;
- plasmaSearchEvent theSearch = plasmaSearchEvent.getEvent(theQuery, ranking, localTiming, remoteTiming, sb.wordIndex, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false, null);
+ if (plasmaSearchEvent.getEvent(theQuery.id()) == null) {
+ theQuery.setOffset(0); // in case that this is a new search, always start without a offset
+ offset = 0;
+ }
+ plasmaSearchEvent theSearch = plasmaSearchEvent.getEvent(theQuery, ranking, localTiming, sb.wordIndex, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false, null);
// generate result object
serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER ORDERING OF SEARCH RESULTS: " + ((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
- ArrayList accu = theSearch.computeResults(plasmaSwitchboard.blueList, true);
serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER RESULT PREPARATION: " + ((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
// calc some more cross-reference
long remainingTime = theQuery.maximumTime - (System.currentTimeMillis() - timestamp);
if (remainingTime < 0) remainingTime = 1000;
- //Object[] ws = acc.getReferences(16);
serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER XREF PREPARATION: " + ((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
// log
serverLog.logInfo("LOCAL_SEARCH", "EXIT WORD SEARCH: " + theQuery.queryString + " - " +
(theSearch.getLocalCount() + theSearch.getGlobalCount()) + " links found, " +
- theSearch.filteredCount() + " links filtered, " +
- accu.size() + " links ordered, " +
((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
// prepare search statistics
Long trackerHandle = new Long(System.currentTimeMillis());
- HashMap searchProfile = theQuery.resultProfile(accu.size(), System.currentTimeMillis() - timestamp);
+ HashMap searchProfile = theQuery.resultProfile(System.currentTimeMillis() - timestamp);
searchProfile.put("querystring", theQuery.queryString);
searchProfile.put("time", trackerHandle);
searchProfile.put("host", client);
searchProfile.put("offset", new Integer(0));
- searchProfile.put("results", accu);
sb.localSearches.add(searchProfile);
TreeSet handles = (TreeSet) sb.localSearchTracker.get(client);
if (handles == null) handles = new TreeSet();
handles.add(trackerHandle);
sb.localSearchTracker.put(client, handles);
- //prop=sb.searchFromLocal(thisSearch, ranking, localTiming, remoteTiming, true, (String) header.get("CLIENTIP"));
prop = new serverObjects();
- //prop.put("references", 0);
prop.put("num-results_totalcount", theSearch.getLocalCount() + theSearch.getGlobalCount());
- prop.put("num-results_filteredcount", theSearch.filteredCount());
- prop.put("num-results_orderedcount", accu.size());
- prop.put("num-results_globalresults", (theSearch.getGlobalCount() == 0) ? 0 : 1);
+ prop.put("num-results_globalresults", 1);
prop.put("num-results_globalresults_globalcount", theSearch.getGlobalCount());
+ prop.put("num-results_offset", 0);
prop.put("num-results_linkcount", 0);
+
+ // compose page navigation
+ StringBuffer pagenav = new StringBuffer();
+ int thispage = offset / theQuery.displayResults();
+ if (thispage == 0) pagenav.append("< "); else {
+ pagenav.append(navurla(thispage - 1, display, theQuery));
+ pagenav.append("<strong><</strong></a> ");
+ }
+ int numberofpages = Math.min(10, Math.min(thispage + 2, (theSearch.getGlobalCount() + theSearch.getLocalCount()) / theQuery.displayResults()));
+ for (int i = 0; i < numberofpages; i++) {
+ if (i == thispage) {
+ pagenav.append("<strong>");
+ pagenav.append(i + 1);
+ pagenav.append("</strong> ");
+ } else {
+ pagenav.append(navurla(i, display, theQuery));
+ pagenav.append(i + 1);
+ pagenav.append("</a> ");
+ }
+ }
+ if (thispage >= numberofpages) pagenav.append(">"); else {
+ pagenav.append(navurla(thispage + 1, display, theQuery));
+ pagenav.append("<strong>></strong></a>");
+ }
+ prop.putASIS("num-results_pagenav", pagenav.toString());
- /*
- for (int i = 0; i < theQuery.wantedResults; i++) {
- prop.put("type_results_" + i + "_item", i);
+ // generate the search result lines; they will be produced by another servlet
+ for (int i = 0; i < theQuery.displayResults(); i++) {
+ prop.put("type_results_" + i + "_item", offset + i);
prop.put("type_results_" + i + "_eventID", theQuery.id());
+ prop.put("type_results_" + i + "_display", display);
}
- prop.put("type_results", theQuery.wantedResults);
- */
- //------------------------
-
- prop.put("type_results", 0);
- URL wordURL=null;
- for (int i = 0; i < accu.size(); i++) {
- plasmaSearchEvent.Entry result = (plasmaSearchEvent.Entry) accu.get(i);
- prop.put("type_results_" + i + "_authorized_recommend", (yacyCore.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? 1 : 0);
- prop.put("type_results_" + i + "_authorized_recommend_deletelink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.wantedResults + "&order=" + crypt.simpleEncode(ranking.toExternalString()) + "&resource=local&time=3&deleteref=" + result.hash() + "&urlmaskfilter=.*");
- prop.put("type_results_" + i + "_authorized_recommend_recommendlink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.wantedResults + "&order=" + crypt.simpleEncode(ranking.toExternalString()) + "&resource=local&time=3&recommendref=" + result.hash() + "&urlmaskfilter=.*");
- prop.put("type_results_" + i + "_authorized_urlhash", result.hash());
- prop.put("type_results_" + i + "_description", result.title());
- prop.put("type_results_" + i + "_url", result.urlstring());
- int port=result.url().getPort();
- URL faviconURL;
- try {
- faviconURL = new URL(result.url().getProtocol() + "://" + result.url().getHost() + ((port != -1) ? (":" + String.valueOf(port)) : "") + "/favicon.ico");
- } catch (MalformedURLException e1) {
- continue;
- }
- prop.put("type_results_" + i + "_faviconCode", sb.licensedURLs.aquireLicense(faviconURL)); // aquire license for favicon url loading
- prop.put("type_results_" + i + "_urlhash", result.hash());
- prop.put("type_results_" + i + "_urlhexhash", yacySeed.b64Hash2hexHash(result.hash()));
- prop.put("type_results_" + i + "_urlname", nxTools.shortenURLString(result.urlname(), 120));
- prop.put("type_results_" + i + "_date", plasmaSwitchboard.dateString(result.modified()));
- prop.put("type_results_" + i + "_ybr", plasmaSearchPreOrder.ybr(result.hash()));
- prop.put("type_results_" + i + "_size", Long.toString(result.filesize()));
- try {
- prop.put("type_results_" + i + "_words", URLEncoder.encode(query[0].toString(),"UTF-8"));
- } catch (UnsupportedEncodingException e) {}
- prop.put("type_results_" + i + "_former", querystring);
- prop.put("type_results_" + i + "_rankingprops", result.word().toPropertyForm() + ", domLengthEstimated=" + plasmaURL.domLengthEstimation(result.hash()) +
- ((plasmaURL.probablyRootURL(result.hash())) ? ", probablyRootURL" : "") +
- (((wordURL = plasmaURL.probablyWordURL(result.hash(), query[0])) != null) ? ", probablyWordURL=" + wordURL.toNormalform(false, true) : ""));
- // adding snippet if available
- if (result.hasSnippet()) {
- prop.put("type_results_" + i + "_snippet", 1);
- prop.putASIS("type_results_" + i + "_snippet_text", result.textSnippet().getLineMarked(theQuery.queryHashes));//FIXME: the ASIS should not be needed, if there is no html in .java
- } else {
- if (post.containsKey("fetchSnippet")) {
- // snippet fetch timeout
- int textsnippet_timeout = Integer.parseInt(env.getConfig("timeout_media", "10000"));
-
- // boolean line_end_with_punctuation
- boolean pre = post.get("pre", "false").equals("true");
-
- // if 'remove' is set to true, then RWI references to URLs that do not have the snippet are removed
- boolean remove = post.get("remove", "false").equals("true");
-
- plasmaSnippetCache.TextSnippet snippet = plasmaSnippetCache.retrieveTextSnippet(
- result.url(),
- queryHashes,
- true,
- pre,
- 260,
- textsnippet_timeout
- );
-
- if (snippet.getErrorCode() < 11) {
- // no problems occurred
- //prop.put("text", (snippet.exists()) ? snippet.getLineMarked(queryHashes) : "unknown");
- prop.putASIS("type_results_" + i + "_snippet_text", (snippet.exists()) ? snippet.getLineMarked(queryHashes) : "unknown");
- } else {
- // problems with snippet fetch
- prop.put("type_results_" + i + "_snippet_text", (remove) ? plasmaSnippetCache.failConsequences(snippet, theQuery.id()) : snippet.getError());
- }
- prop.put("type_results_" + i + "_snippet", 1);
- } else {
- // no snippet available (will be fetched later via ajax)
- prop.put("type_results_" + i + "_snippet", 0);
- prop.put("type_results_" + i + "_snippet_text", "");
- }
- }
- prop.put("type_results", accu.size());
- prop.put("num-results_linkcount", Integer.toString(accu.size()));
- }
-
- //------------------------
+ prop.put("type_results", theQuery.displayResults());
// process result of search
if (filtered.size() > 0) {
@@ -480,6 +418,7 @@ public class yacysearch { prop.put("type_combine_words_" + hintcount + "_word", word);
prop.put("type_combine_words_" + hintcount + "_newsearch", post.get("search", "").replace(' ', '+') + "+" + word);
prop.put("type_combine_words_" + hintcount + "_count", count);
+ prop.put("type_combine_words_" + hintcount + "_offset", offset);
prop.put("type_combine_words_" + hintcount + "_resource", ((global) ? "global" : "local"));
prop.put("type_combine_words_" + hintcount + "_time", (searchtime / 1000));
}
@@ -493,8 +432,6 @@ public class yacysearch { if (wrongregex != null) {
prop.put("num-results_wrong_regex", wrongregex);
prop.put("num-results", 4);
- } else if (totalcount == 0) {
- prop.put("num-results", 3); // long
} else {
prop.put("num-results", 5);
}
@@ -557,9 +494,10 @@ public class yacysearch { prop.put("input_input", input);
prop.put("input_display", display);
prop.putASIS("input_promoteSearchPageGreeting", promoteSearchPageGreeting);
- prop.put("input_former", post.get("search", ""));
+ prop.put("input_former", querystring);
prop.put("former", post.get("search", ""));
prop.put("input_count", count);
+ prop.put("input_offset", offset);
prop.put("input_resource", (global) ? "global" : "local");
prop.put("input_time", searchtime / 1000);
prop.put("input_urlmaskfilter", urlmask);
@@ -578,4 +516,7 @@ public class yacysearch { return prop;
}
+ private static String navurla(int page, int display, plasmaSearchQuery theQuery) {
+ return "<a href=\"yacysearch.html?display=" + display + "&search=" + theQuery.queryString() + "&count="+ theQuery.displayResults() + "&offset=" + (page * theQuery.displayResults()) + "&resource=" + theQuery.searchdom() + "&time=" + (theQuery.maximumTime / 1000) + "&urlmaskfilter=" + theQuery.urlMask + "&prefermaskfilter=" + theQuery.prefer + "&cat=href&constraint=" + theQuery.constraint.exportB64() + "&contentdom=" + theQuery.contentdom() + "&former=" + theQuery.queryString() + "\">";
+ }
}
diff --git a/htroot/yacysearchitem.html b/htroot/yacysearchitem.html index 98a3a76f4..d67474229 100644 --- a/htroot/yacysearchitem.html +++ b/htroot/yacysearchitem.html @@ -1,4 +1,8 @@ #(content)#:: + <div class="searchresults"> + <h4 class="linktitle"> + <img src="ViewImage.png?width=16&height=16&code=#[faviconCode]#" id="f#[urlhash]#" class="favicon" width="16" height="16" alt="favicon of #[url]#" /> + <a href="#[url]#" target="_parent">#[description]#</a></h4> #(authorized)#:: <div class="urlactions"> <a href="/Bookmarks.html?edit=#[urlhash]#" class="bookmarklink" title="bookmark"><img src="/env/grafics/empty.gif" title="bookmark" alt="bookmark" class="bookmarkIcon" /></a> @@ -11,8 +15,12 @@ #(/recommend)# </div> #(/authorized)# - <h4 class="linktitle"><img src="ViewImage.png?width=16&height=16&code=#[faviconCode]#" id="f#[urlhash]#" class="favicon" width="16" height="16" /><a href="#[url]#" target="_parent">#[description]#</a></h4> - <p class="snippet iconindented"><span class="snippetLoaded" id="h#[urlhash]#">#[snippet]#</span></p> - <p class="url iconindented"><a href="#[url]#" id="url#[urlhash]#" target="_parent">#[urlname]#</a></p> - <p class="urlinfo iconindented">#[date]# | YBR-#[ybr]# | <a href="ViewFile.html?urlHash=#[urlhash]#&words=#[words]#">Info</a> | <a href="yacysearch.html?cat=image&url=#[url]#&search=#[former]#">Pictures</a></p> + + <p class="snippet"><span class="snippetLoaded" id="h#[urlhash]#">#[snippet]#</span></p> + <p class="url"><a href="#[url]#" id="url#[urlhash]#" target="_parent">#[urlname]#</a></p> + <p class="urlinfo">#[date]# | YBR-#[ybr]# | <a href="ViewFile.html?urlHash=#[urlhash]#&words=#[words]#">Info</a> | <a href="yacysearch.html?cat=image&url=#[url]#&search=#[former]#">Pictures</a></p> + </div> #(/content)# +<script type="text/javascript"> +statistics("#[offset]#", "#[items]#", "#[global]#", "#[total]#"); +</script> diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index 13499209a..1dcb6240e 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -27,7 +27,6 @@ import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URLEncoder; -import java.util.ArrayList; import java.util.TreeSet; import de.anomic.http.httpHeader; @@ -55,6 +54,7 @@ public class yacysearchitem { String eventID = post.get("eventID", ""); int item = post.getInt("item", -1); + //int display = post.getInt("display", 0); boolean authenticated = sb.adminAuthenticated(header) >= 2; // find search event @@ -62,19 +62,25 @@ public class yacysearchitem { plasmaSearchQuery theQuery = theSearch.getQuery(); plasmaSearchRankingProfile ranking = theSearch.getRanking(); - long startprofiling = System.currentTimeMillis(); - // generate result object - ArrayList accu = theSearch.computeResults(plasmaSwitchboard.blueList, true); - - plasmaSearchEvent.Entry result = (plasmaSearchEvent.Entry) accu.get(item); - System.out.println("PROFILING_DEBUG: " + (System.currentTimeMillis() - startprofiling) + " millisekunden fuer item " + item); + plasmaSearchEvent.ResultEntry result = theSearch.oneResult(item); + + // dynamically update count values + prop.put("offset", theQuery.neededResults() - theQuery.displayResults() + 1); + prop.put("items", item + 1); + prop.put("global", theSearch.getGlobalCount()); + prop.put("total", theSearch.getGlobalCount() + theSearch.getLocalCount()); + if (result == null) { + prop.put("content", 0); // no content + return prop; + } + prop.put("content", 1); // switch on content prop.put("content_authorized", (authenticated) ? 1 : 0); prop.put("content_authorized_recommend", (yacyCore.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? 1 : 0); - prop.put("content_authorized_recommend_deletelink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.wantedResults + "&order=" + crypt.simpleEncode(ranking.toExternalString()) + "&resource=local&time=3&deleteref=" + result.hash() + "&urlmaskfilter=.*"); - prop.put("content_authorized_recommend_recommendlink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.wantedResults + "&order=" + crypt.simpleEncode(ranking.toExternalString()) + "&resource=local&time=3&recommendref=" + result.hash() + "&urlmaskfilter=.*"); + prop.put("content_authorized_recommend_deletelink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(ranking.toExternalString()) + "&resource=local&time=3&deleteref=" + result.hash() + "&urlmaskfilter=.*"); + prop.put("content_authorized_recommend_recommendlink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(ranking.toExternalString()) + "&resource=local&time=3&recommendref=" + result.hash() + "&urlmaskfilter=.*"); prop.put("content_authorized_urlhash", result.hash()); prop.put("content_description", result.title()); prop.put("content_url", result.urlstring()); @@ -105,40 +111,8 @@ public class yacysearchitem { ((plasmaURL.probablyRootURL(result.hash())) ? ", probablyRootURL" : "") + (((wordURL = plasmaURL.probablyWordURL(result.hash(), query[0])) != null) ? ", probablyWordURL=" + wordURL.toNormalform(false, true) : "")); - /* - // adding snippet if available - if (result.hasSnippet()) { - prop.put("content_snippet", result.textSnippet().getLineMarked(theQuery.queryHashes)); - } else { - // snippet fetch timeout - int textsnippet_timeout = Integer.parseInt(env.getConfig("timeout_media", "10000")); - - // boolean line_end_with_punctuation - boolean pre = post.get("pre", "false").equals("true"); - - // if 'remove' is set to true, then RWI references to URLs that do not have the snippet are removed - boolean remove = post.get("remove", "false").equals("true"); - - plasmaSnippetCache.TextSnippet snippet = plasmaSnippetCache.retrieveTextSnippet( - result.url(), - theQuery.queryHashes, - true, - pre, - 260, - textsnippet_timeout - ); - - if (snippet.getErrorCode() < 11) { - // no problems occurred - //prop.put("text", (snippet.exists()) ? snippet.getLineMarked(queryHashes) : "unknown"); - prop.putASIS("content_snippet", (snippet.exists()) ? snippet.getLineMarked(theQuery.queryHashes) : "unknown"); - } else { - // problems with snippet fetch - prop.put("content_snippet", (remove) ? plasmaSnippetCache.failConsequences(snippet, theQuery.id()) : snippet.getError()); - } - } - */ - prop.put("content_snippet","temporary no snippet computed"); + prop.putASIS("content_snippet", result.textSnippet().getLineMarked(theQuery.queryHashes)); + return prop; } diff --git a/source/de/anomic/http/httpSSI.java b/source/de/anomic/http/httpSSI.java index ce4451cb2..4b083e41b 100644 --- a/source/de/anomic/http/httpSSI.java +++ b/source/de/anomic/http/httpSSI.java @@ -35,11 +35,11 @@ import de.anomic.server.serverByteBuffer; public class httpSSI { - public static void writeSSI(serverByteBuffer in, OutputStream out) throws IOException { - writeSSI(in, 0, out); + public static void writeSSI(serverByteBuffer in, OutputStream out, String authorization) throws IOException { + writeSSI(in, 0, out, authorization); } - public static void writeSSI(serverByteBuffer in, int off, OutputStream out) throws IOException { + public static void writeSSI(serverByteBuffer in, int off, OutputStream out, String authorization) throws IOException { int p = in.indexOf("<!--#".getBytes(), off); if (p >= 0) { int q = in.indexOf("-->".getBytes(), p + 10); @@ -48,8 +48,8 @@ public class httpSSI { } else { out.write(in.getBytes(off, p - off)); } - parseSSI(in, p, q + 3 - p, out); - writeSSI(in, q + 3, out); + parseSSI(in, p, q + 3 - p, out, authorization); + writeSSI(in, q + 3, out, authorization); } else /* p < 0 */ { if (out instanceof httpChunkedOutputStream) { ((httpChunkedOutputStream) out).write(in, off, in.length() - off); @@ -59,17 +59,17 @@ public class httpSSI { } } - private static void parseSSI(serverByteBuffer in, int off, int len, OutputStream out) { + private static void parseSSI(serverByteBuffer in, int off, int len, OutputStream out, String authorization) { if (in.startsWith("<!--#include virtual=\"".getBytes(), off)) { int q = in.indexOf("\"".getBytes(), off + 22); if (q > 0) { String path = in.toString(off + 22, q); - writeContent(path, out); + writeContent(path, out, authorization); } } } - private static void writeContent(String path, OutputStream out) { + private static void writeContent(String path, OutputStream out, String authorization) { // check if there are arguments in path string String args = ""; int argpos = path.indexOf('?'); @@ -86,7 +86,7 @@ public class httpSSI { conProp.setProperty(httpHeader.CONNECTION_PROP_ARGS, args); conProp.setProperty(httpHeader.CONNECTION_PROP_HTTP_VER, httpHeader.HTTP_VERSION_0_9); conProp.setProperty("CLIENTIP", "127.0.0.1"); - + header.put(httpHeader.AUTHORIZATION, authorization); httpdFileHandler.doGet(conProp, header, out); } } diff --git a/source/de/anomic/http/httpdFileHandler.java b/source/de/anomic/http/httpdFileHandler.java index 680172bf3..051662f6e 100644 --- a/source/de/anomic/http/httpdFileHandler.java +++ b/source/de/anomic/http/httpdFileHandler.java @@ -283,7 +283,7 @@ public final class httpdFileHandler { String method = conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD);
path = conProp.getProperty(httpHeader.CONNECTION_PROP_PATH);
String argsString = conProp.getProperty(httpHeader.CONNECTION_PROP_ARGS); // is null if no args were given
- String httpVersion= conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER);
+ String httpVersion = conProp.getProperty(httpHeader.CONNECTION_PROP_HTTP_VER);
// check hack attacks in path
if (path.indexOf("..") >= 0) {
@@ -710,7 +710,7 @@ public final class httpdFileHandler { httpd.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, -1, targetDate, null, tp.getOutgoingHeader(), null, "chunked", nocache);
// send the content in chunked parts, see RFC 2616 section 3.6.1
httpChunkedOutputStream chos = new httpChunkedOutputStream(out);
- httpSSI.writeSSI(o, chos);
+ httpSSI.writeSSI(o, chos, authorization);
//chos.write(result);
chos.finish();
} else {
@@ -724,14 +724,14 @@ public final class httpdFileHandler { if (zipContent) {
GZIPOutputStream zippedOut = new GZIPOutputStream(o);
- httpSSI.writeSSI(o1, zippedOut);
+ httpSSI.writeSSI(o1, zippedOut, authorization);
//httpTemplate.writeTemplate(fis, zippedOut, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
zippedOut.finish();
zippedOut.flush();
zippedOut.close();
zippedOut = null;
} else {
- httpSSI.writeSSI(o1, o);
+ httpSSI.writeSSI(o1, o, authorization);
//httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
}
if (method.equals(httpHeader.METHOD_HEAD)) {
diff --git a/source/de/anomic/index/indexCachedRI.java b/source/de/anomic/index/indexCachedRI.java index 29353f78f..3579ea934 100644 --- a/source/de/anomic/index/indexCachedRI.java +++ b/source/de/anomic/index/indexCachedRI.java @@ -86,7 +86,7 @@ public class indexCachedRI implements indexRI { } public long getUpdateTime(String wordHash) { - indexContainer entries = getContainer(wordHash, null, -1); + indexContainer entries = getContainer(wordHash, null); if (entries == null) return 0; return entries.updated(); } @@ -140,25 +140,25 @@ public class indexCachedRI implements indexRI { return false; } - public indexContainer getContainer(String wordHash, Set urlselection, long maxTime) { + public indexContainer getContainer(String wordHash, Set urlselection) { // get from cache - indexContainer container = riExtern.getContainer(wordHash, urlselection, maxTime); + indexContainer container = riExtern.getContainer(wordHash, urlselection); if (container == null) { - container = riIntern.getContainer(wordHash, urlselection, maxTime); + container = riIntern.getContainer(wordHash, urlselection); } else { - container.addAllUnique(riIntern.getContainer(wordHash, urlselection, maxTime)); + container.addAllUnique(riIntern.getContainer(wordHash, urlselection)); } // get from collection index if (container == null) { - container = backend.getContainer(wordHash, urlselection, (maxTime < 0) ? -1 : maxTime); + container = backend.getContainer(wordHash, urlselection); } else { - container.addAllUnique(backend.getContainer(wordHash, urlselection, (maxTime < 0) ? -1 : maxTime)); + container.addAllUnique(backend.getContainer(wordHash, urlselection)); } return container; } - public Map getContainers(Set wordHashes, Set urlselection, boolean interruptIfEmpty, long maxTime) { + public Map getContainers(Set wordHashes, Set urlselection, boolean interruptIfEmpty) { // return map of wordhash:indexContainer // retrieve entities that belong to the hashes @@ -166,19 +166,12 @@ public class indexCachedRI implements indexRI { String singleHash; indexContainer singleContainer; Iterator i = wordHashes.iterator(); - long start = System.currentTimeMillis(); - long remaining; while (i.hasNext()) { - // check time - remaining = maxTime - (System.currentTimeMillis() - start); - //if ((maxTime > 0) && (remaining <= 0)) break; - if ((maxTime >= 0) && (remaining <= 0)) remaining = 100; - // get next word hash: singleHash = (String) i.next(); // retrieve index - singleContainer = getContainer(singleHash, urlselection, (maxTime < 0) ? -1 : remaining / (wordHashes.size() - containers.size())); + singleContainer = getContainer(singleHash, urlselection); // check result if (((singleContainer == null) || (singleContainer.size() == 0)) && (interruptIfEmpty)) return new HashMap(); diff --git a/source/de/anomic/index/indexCollectionRI.java b/source/de/anomic/index/indexCollectionRI.java index 072d6da15..543bae7a3 100644 --- a/source/de/anomic/index/indexCollectionRI.java +++ b/source/de/anomic/index/indexCollectionRI.java @@ -62,7 +62,7 @@ public class indexCollectionRI implements indexRI { } public long getUpdateTime(String wordHash) { - indexContainer entries = getContainer(wordHash, null, -1); + indexContainer entries = getContainer(wordHash, null); if (entries == null) return 0; return entries.updated(); } @@ -130,7 +130,7 @@ public class indexCollectionRI implements indexRI { } } - public synchronized indexContainer getContainer(String wordHash, Set urlselection, long maxtime) { + public synchronized indexContainer getContainer(String wordHash, Set urlselection) { try { kelondroRowSet collection = collectionIndex.get(wordHash.getBytes()); if (collection != null) collection.select(urlselection); diff --git a/source/de/anomic/index/indexContainer.java b/source/de/anomic/index/indexContainer.java index c90d329f4..25820cc77 100644 --- a/source/de/anomic/index/indexContainer.java +++ b/source/de/anomic/index/indexContainer.java @@ -1,6 +1,6 @@ // indexContainer.java
-// (C) 2006 by Michael Peter Christen; mc@anomic.de, Frankfurt a. M., Germany
-// first published 04.07.2006 on http://www.anomic.de
+// (C) 2006 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// first published 04.07.2006 on http://yacy.net
//
// This is a part of YaCy, a peer-to-peer based web search engine
//
@@ -154,28 +154,15 @@ public class indexContainer extends kelondroRowSet { return new indexRWIEntry(entry);
}
- public boolean removeEntry(String wordHash, String urlHash) {
- if (!wordHash.equals(this.wordHash)) return false;
- return remove(urlHash) != null;
- }
-
- public int removeEntries(String wordHash, Set urlHashes) {
- if (!wordHash.equals(this.wordHash)) return 0;
+ public int removeEntries(Set urlHashes) {
int count = 0;
Iterator i = urlHashes.iterator();
while (i.hasNext()) count += (remove((String) i.next()) == null) ? 0 : 1;
return count;
}
- public void removeEntriesMultiple(Set wordHashes, Set urlHashes) {
- Iterator i = wordHashes.iterator();
- while (i.hasNext()) {
- removeEntries((String) i.next(), urlHashes);
- }
- }
-
public Iterator entries() {
- // returns an iterator of indexEntry objects
+ // returns an iterator of indexRWIEntry objects
return new entryIterator();
}
@@ -220,9 +207,7 @@ public class indexContainer extends kelondroRowSet { }
}
- public static indexContainer joinContainers(Collection containers, long time, int maxDistance) {
-
- long stamp = System.currentTimeMillis();
+ public static indexContainer joinContainers(Collection containers, int maxDistance) {
// order entities by their size
TreeMap map = new TreeMap();
@@ -251,10 +236,9 @@ public class indexContainer extends kelondroRowSet { while ((map.size() > 0) && (searchResult.size() > 0)) {
// take the first element of map which is a result and combine it with result
k = (Long) map.firstKey(); // the next smallest...
- time -= (System.currentTimeMillis() - stamp); stamp = System.currentTimeMillis();
searchA = searchResult;
searchB = (indexContainer) map.remove(k);
- searchResult = indexContainer.joinConstructive(searchA, searchB, 2 * time / (map.size() + 1), maxDistance);
+ searchResult = indexContainer.joinConstructive(searchA, searchB, maxDistance);
// free resources
searchA = null;
searchB = null;
@@ -265,17 +249,14 @@ public class indexContainer extends kelondroRowSet { return searchResult;
}
- public static indexContainer excludeContainers(indexContainer pivot, Collection containers, long time) {
-
- long stamp = System.currentTimeMillis();
+ public static indexContainer excludeContainers(indexContainer pivot, Collection containers) {
// check if there is any result
if ((containers == null) || (containers.size() == 0)) return pivot; // no result, nothing found
Iterator i = containers.iterator();
while (i.hasNext()) {
- time -= (System.currentTimeMillis() - stamp); stamp = System.currentTimeMillis();
- pivot = excludeDestructive(pivot, (indexContainer) i.next(), time);
+ pivot = excludeDestructive(pivot, (indexContainer) i.next());
if ((pivot == null) || (pivot.size() == 0)) return null;
}
@@ -289,7 +270,7 @@ public class indexContainer extends kelondroRowSet { return l;
}
- public static indexContainer joinConstructive(indexContainer i1, indexContainer i2, long time, int maxDistance) {
+ public static indexContainer joinConstructive(indexContainer i1, indexContainer i2, int maxDistance) {
if ((i1 == null) || (i2 == null)) return null;
if ((i1.size() == 0) || (i2.size() == 0)) return null;
@@ -302,15 +283,15 @@ public class indexContainer extends kelondroRowSet { // start most efficient method
if (stepsEnum > stepsTest) {
if (i1.size() < i2.size())
- return joinConstructiveByTest(i1, i2, time, maxDistance);
+ return joinConstructiveByTest(i1, i2, maxDistance);
else
- return joinConstructiveByTest(i2, i1, time, maxDistance);
+ return joinConstructiveByTest(i2, i1, maxDistance);
} else {
- return joinConstructiveByEnumeration(i1, i2, time, maxDistance);
+ return joinConstructiveByEnumeration(i1, i2, maxDistance);
}
}
- private static indexContainer joinConstructiveByTest(indexContainer small, indexContainer large, long time, int maxDistance) {
+ private static indexContainer joinConstructiveByTest(indexContainer small, indexContainer large, int maxDistance) {
System.out.println("DEBUG: JOIN METHOD BY TEST");
assert small.rowdef.equals(large.rowdef) : "small = " + small.rowdef.toString() + "; large = " + large.rowdef.toString();
int keylength = small.rowdef.width(0);
@@ -318,8 +299,7 @@ public class indexContainer extends kelondroRowSet { indexContainer conj = new indexContainer(null, small.rowdef, 0); // start with empty search result
Iterator se = small.entries();
indexRWIEntry ie0, ie1;
- long stamp = System.currentTimeMillis();
- while ((se.hasNext()) && ((System.currentTimeMillis() - stamp) < time)) {
+ while (se.hasNext()) {
ie0 = (indexRWIEntry) se.next();
ie1 = large.get(ie0.urlHash());
if ((ie0 != null) && (ie1 != null)) {
@@ -333,7 +313,7 @@ public class indexContainer extends kelondroRowSet { return conj;
}
- private static indexContainer joinConstructiveByEnumeration(indexContainer i1, indexContainer i2, long time, int maxDistance) {
+ private static indexContainer joinConstructiveByEnumeration(indexContainer i1, indexContainer i2, int maxDistance) {
System.out.println("DEBUG: JOIN METHOD BY ENUMERATION");
assert i1.rowdef.equals(i2.rowdef) : "i1 = " + i1.rowdef.toString() + "; i2 = " + i2.rowdef.toString();
int keylength = i1.rowdef.width(0);
@@ -350,8 +330,7 @@ public class indexContainer extends kelondroRowSet { ie1 = (indexRWIEntry) e1.next();
ie2 = (indexRWIEntry) e2.next();
- long stamp = System.currentTimeMillis();
- while ((System.currentTimeMillis() - stamp) < time) {
+ while (true) {
assert (ie1.urlHash().length() == keylength) : "ie1.urlHash() = " + ie1.urlHash();
assert (ie2.urlHash().length() == keylength) : "ie2.urlHash() = " + ie2.urlHash();
c = i1.rowdef.getOrdering().compare(ie1.urlHash(), ie2.urlHash());
@@ -372,7 +351,7 @@ public class indexContainer extends kelondroRowSet { return conj;
}
- public static indexContainer excludeDestructive(indexContainer pivot, indexContainer excl, long time) {
+ public static indexContainer excludeDestructive(indexContainer pivot, indexContainer excl) {
if (pivot == null) return null;
if (excl == null) return pivot;
if (pivot.size() == 0) return null;
@@ -386,21 +365,20 @@ public class indexContainer extends kelondroRowSet { // start most efficient method
if (stepsEnum > stepsTest) {
- return excludeDestructiveByTest(pivot, excl, time);
+ return excludeDestructiveByTest(pivot, excl);
} else {
- return excludeDestructiveByEnumeration(pivot, excl, time);
+ return excludeDestructiveByEnumeration(pivot, excl);
}
}
- private static indexContainer excludeDestructiveByTest(indexContainer pivot, indexContainer excl, long time) {
+ private static indexContainer excludeDestructiveByTest(indexContainer pivot, indexContainer excl) {
assert pivot.rowdef.equals(excl.rowdef) : "small = " + pivot.rowdef.toString() + "; large = " + excl.rowdef.toString();
int keylength = pivot.rowdef.width(0);
assert (keylength == excl.rowdef.width(0));
boolean iterate_pivot = pivot.size() < excl.size();
Iterator se = (iterate_pivot) ? pivot.entries() : excl.entries();
indexRWIEntry ie0, ie1;
- long stamp = System.currentTimeMillis();
- while ((se.hasNext()) && ((System.currentTimeMillis() - stamp) < time)) {
+ while (se.hasNext()) {
ie0 = (indexRWIEntry) se.next();
ie1 = excl.get(ie0.urlHash());
if ((ie0 != null) && (ie1 != null)) {
@@ -412,7 +390,7 @@ public class indexContainer extends kelondroRowSet { return pivot;
}
- private static indexContainer excludeDestructiveByEnumeration(indexContainer pivot, indexContainer excl, long time) {
+ private static indexContainer excludeDestructiveByEnumeration(indexContainer pivot, indexContainer excl) {
assert pivot.rowdef.equals(excl.rowdef) : "i1 = " + pivot.rowdef.toString() + "; i2 = " + excl.rowdef.toString();
int keylength = pivot.rowdef.width(0);
assert (keylength == excl.rowdef.width(0));
@@ -427,8 +405,7 @@ public class indexContainer extends kelondroRowSet { ie1 = (indexRWIEntry) e1.next();
ie2 = (indexRWIEntry) e2.next();
- long stamp = System.currentTimeMillis();
- while ((System.currentTimeMillis() - stamp) < time) {
+ while (true) {
assert (ie1.urlHash().length() == keylength) : "ie1.urlHash() = " + ie1.urlHash();
assert (ie2.urlHash().length() == keylength) : "ie2.urlHash() = " + ie2.urlHash();
c = pivot.rowdef.getOrdering().compare(ie1.urlHash(), ie2.urlHash());
diff --git a/source/de/anomic/index/indexRAMRI.java b/source/de/anomic/index/indexRAMRI.java index 9b2631497..06d869567 100644 --- a/source/de/anomic/index/indexRAMRI.java +++ b/source/de/anomic/index/indexRAMRI.java @@ -101,7 +101,7 @@ public final class indexRAMRI implements indexRI { }
public synchronized long getUpdateTime(String wordHash) {
- indexContainer entries = getContainer(wordHash, null, -1);
+ indexContainer entries = getContainer(wordHash, null);
if (entries == null) return 0;
return entries.updated();
}
@@ -391,7 +391,7 @@ public final class indexRAMRI implements indexRI { return ((indexContainer) cache.get(wordHash)).size();
}
- public synchronized indexContainer getContainer(String wordHash, Set urlselection, long maxtime_dummy) {
+ public synchronized indexContainer getContainer(String wordHash, Set urlselection) {
// retrieve container
indexContainer container = (indexContainer) cache.get(wordHash);
@@ -418,7 +418,7 @@ public final class indexRAMRI implements indexRI { public synchronized boolean removeEntry(String wordHash, String urlHash) {
indexContainer c = (indexContainer) cache.get(wordHash);
- if ((c != null) && (c.removeEntry(wordHash, urlHash))) {
+ if ((c != null) && (c.remove(urlHash) != null)) {
// removal successful
if (c.size() == 0) {
deleteContainer(wordHash);
@@ -436,7 +436,7 @@ public final class indexRAMRI implements indexRI { if (urlHashes.size() == 0) return 0;
indexContainer c = (indexContainer) cache.get(wordHash);
int count;
- if ((c != null) && ((count = c.removeEntries(wordHash, urlHashes)) > 0)) {
+ if ((c != null) && ((count = c.removeEntries(urlHashes)) > 0)) {
// removal successful
if (c.size() == 0) {
deleteContainer(wordHash);
diff --git a/source/de/anomic/index/indexRI.java b/source/de/anomic/index/indexRI.java index 2b440c4e9..652c1ec47 100644 --- a/source/de/anomic/index/indexRI.java +++ b/source/de/anomic/index/indexRI.java @@ -42,7 +42,7 @@ public interface indexRI { public long getUpdateTime(String wordHash); public int indexSize(String wordHash); public boolean hasContainer(String wordHash); // should only be used if in case that true is returned the getContainer is NOT called - public indexContainer getContainer(String wordHash, Set urlselection, long maxtime); + public indexContainer getContainer(String wordHash, Set urlselection); public indexContainer deleteContainer(String wordHash); public boolean removeEntry(String wordHash, String urlHash); diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index a9432c4ba..4ac282a89 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -209,6 +209,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr }
public synchronized kelondroRow.Entry get(byte[] key) throws IOException {
+ if (index == null) return null; // case may happen during shutdown
int pos = index.geti(key);
assert this.size() == index.size() : "content.size() = " + this.size() + ", index.size() = " + index.size() + ", analysis: " + index.consistencyAnalysis();
if (pos < 0) return null;
@@ -259,6 +260,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr assert (!(serverLog.allZero(row.getColBytes(0))));
assert row.objectsize() <= this.rowdef.objectsize;
byte[] key = row.getColBytes(0);
+ if (index == null) return null; // case may appear during shutdown
int pos = index.geti(key);
if (pos < 0) {
pos = super.add(row);
diff --git a/source/de/anomic/kelondro/kelondroObjects.java b/source/de/anomic/kelondro/kelondroObjects.java index c9033b79e..84edd1d06 100644 --- a/source/de/anomic/kelondro/kelondroObjects.java +++ b/source/de/anomic/kelondro/kelondroObjects.java @@ -62,7 +62,8 @@ public class kelondroObjects { assert (key != null); assert (key.length() > 0); assert (newMap != null); - + if (cacheScore == null) return; // may appear during shutdown + // write entry kelondroRA kra = dyn.getRA(key); newMap.write(kra); @@ -95,8 +96,8 @@ public class kelondroObjects { protected synchronized kelondroObjectsEntry get(final String key, final boolean storeCache) throws IOException { // load map from cache - assert cache != null; assert key != null; + if (cache == null) return null; // case may appear during shutdown kelondroObjectsEntry map = (kelondroObjectsEntry) cache.get(key); if (map != null) return map; @@ -122,6 +123,7 @@ public class kelondroObjects { private synchronized void checkCacheSpace() { // check for space in cache + if (cache == null) return; // may appear during shutdown if (cache.size() >= cachesize) { // delete one entry final String delkey = (String) cacheScore.getMinObject(); diff --git a/source/de/anomic/kelondro/kelondroRowCollection.java b/source/de/anomic/kelondro/kelondroRowCollection.java index d71ed2604..5445e71fa 100644 --- a/source/de/anomic/kelondro/kelondroRowCollection.java +++ b/source/de/anomic/kelondro/kelondroRowCollection.java @@ -281,6 +281,7 @@ public class kelondroRowCollection { assert (index >= 0) : "get: access with index " + index + " is below zero"; assert (index < chunkcount) : "get: access with index " + index + " is above chunkcount " + chunkcount + "; sortBound = " + sortBound; assert (index * rowdef.objectsize < chunkcache.length); + if ((chunkcache == null) || (rowdef == null)) return null; // case may appear during shutdown if (index >= chunkcount) return null; if (index * rowdef.objectsize() >= chunkcache.length) return null; this.lastTimeRead = System.currentTimeMillis(); @@ -288,14 +289,27 @@ public class kelondroRowCollection { } public synchronized final void set(int index, kelondroRow.Entry a) { - assert (index >= 0) : "get: access with index " + index + " is below zero"; - assert (index < chunkcount) : "get: access with index " + index + " is above chunkcount " + chunkcount; - //assert (!(bugappearance(a, astart, alength))) : "a = " + serverLog.arrayList(a, astart, alength); - //if (bugappearance(a, astart, alength)) return; // TODO: this is temporary; remote peers may still submit bad entries + assert (index >= 0) : "set: access with index " + index + " is below zero"; + ensureSize(index + 1); a.writeToArray(chunkcache, index * rowdef.objectsize()); + if (index >= chunkcount) chunkcount = index + 1; this.lastTimeWrote = System.currentTimeMillis(); } + public final void insertUnique(int index, kelondroRow.Entry a) { + assert (a != null); + + if (index < chunkcount) { + // make room + ensureSize(chunkcount + 1); + System.arraycopy(chunkcache, rowdef.objectsize() * index, chunkcache, rowdef.objectsize() * (index + 1), (chunkcount - index) * rowdef.objectsize()); + chunkcount++; + } + // insert entry into gap + set(index, a); + } + + public synchronized void addUnique(kelondroRow.Entry row) { byte[] r = row.bytes(); addUnique(r, 0, r.length); diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 8a1d622ef..4492b8b59 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -297,6 +297,7 @@ public final class plasmaCrawlLURL { }
public synchronized boolean exists(String urlHash) {
+ if (urlIndexFile == null) return false; // case may happen during shutdown
try {
return urlIndexFile.has(urlHash.getBytes());
} catch (IOException e) {
diff --git a/source/de/anomic/plasma/plasmaSearchContainer.java b/source/de/anomic/plasma/plasmaSearchContainer.java new file mode 100644 index 000000000..760aec759 --- /dev/null +++ b/source/de/anomic/plasma/plasmaSearchContainer.java @@ -0,0 +1,176 @@ +// plasmaSearchContainer.java +// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 29.8.2007 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +package de.anomic.plasma; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; + +import de.anomic.htmlFilter.htmlFilterContentScraper; +import de.anomic.index.indexContainer; +import de.anomic.index.indexRWIEntry; +import de.anomic.kelondro.kelondroMScoreCluster; + +public class plasmaSearchContainer { + + private indexRWIEntry entryMin, entryMax; + private indexContainer container; + private plasmaSearchRankingProfile ranking; + private TreeSet searchedWords; + private int globalcount; + private HashSet urlhashes; // set for double-check + private kelondroMScoreCluster ref; // reference score computation for the commonSense heuristic + private plasmaSearchQuery query; + + + public plasmaSearchContainer(plasmaSearchQuery query, plasmaSearchRankingProfile ranking, TreeSet searchedWords) { + this(query, ranking, searchedWords, plasmaWordIndex.emptyContainer(null, 0)); + } + + public plasmaSearchContainer(plasmaSearchQuery query, plasmaSearchRankingProfile ranking, TreeSet searchedWords, indexContainer presortedContainer) { + // only for sorted containers + this.entryMin = null; + this.entryMax = null; + this.container = presortedContainer; + this.ranking = ranking; + this.searchedWords = searchedWords; + this.globalcount = 0; + this.urlhashes = new HashSet(); + this.ref = new kelondroMScoreCluster(); + this.query = query; + } + + public void insert(indexRWIEntry entry, boolean local) { + // add the entry to the container into a position in such a way, that the container stays sorted + assert (entry != null); + + // make a double-check: because different peers may have computed different ranking attributes, + // the double check cannot be made using the ranking and the insert position + if (urlhashes.contains(entry.urlHash())) return; + urlhashes.add(entry.urlHash()); + + // find new min/max borders + if (this.entryMin == null) this.entryMin = (indexRWIEntry) entry.clone(); else this.entryMin.min(entry); + if (this.entryMax == null) this.entryMax = (indexRWIEntry) entry.clone(); else this.entryMax.max(entry); + long pivot = this.ranking.preRanking(entry, this.entryMin, this.entryMax, this.searchedWords); + + // insert the entry + int insertPosition = insertPosition(pivot); + + // insert at found position + container.insertUnique(insertPosition, entry.toKelondroEntry()); + + // update counter + if (!local) this.globalcount++; + + } + + public void insert(indexContainer c, boolean local, boolean presorted) { + if ((this.container.size() == 0) && (presorted)) { + this.container = c; + if (!local) this.globalcount = c.size(); + } else { + Iterator i = c.entries(); + while (i.hasNext()) { + insert((indexRWIEntry) i.next(), local); + } + } + } + + private int insertPosition(long pivotRanking) { + return insertPosition(pivotRanking, 0, container.size()); + } + + private int insertPosition(long pivotRanking, int left /*including*/, int right /*excluding*/) { + if (right - left < 10) { + // do iterative search, less overhead + for (int i = left; i < right; i++) { + if (this.ranking.preRanking(new indexRWIEntry(container.get(i)), this.entryMin, this.entryMax, this.searchedWords) < pivotRanking) { + // we found the right insert position + return i; + } + } + return right; + } + // find recursively + int middle = (left + right) / 2; + if (this.ranking.preRanking(new indexRWIEntry(container.get(middle)), this.entryMin, this.entryMax, this.searchedWords) < pivotRanking) { + // must be on the left side + return insertPosition(pivotRanking, left, middle); + } else { + // must be on the right side + return insertPosition(pivotRanking, middle + 1, right); + } + } + + public indexRWIEntry remove(String urlHash) { + return this.container.remove(urlHash); + } + + public int removeEntries(Set urlHashes) { + return this.container.removeEntries(urlHashes); + } + + public indexContainer container() { + return this.container; + } + + public int getGlobalCount() { + return this.globalcount; + } + + public Object[] getReferences(int count) { + // create a list of words that had been computed by statistics over all + // words that appeared in the url or the description of all urls + return ref.getScores(count, false, 2, Integer.MAX_VALUE); + } + + public void addReferences(String[] words) { + String word; + for (int i = 0; i < words.length; i++) { + word = words[i].toLowerCase(); + if ((word.length() > 2) && + ("http_html_php_ftp_www_com_org_net_gov_edu_index_home_page_for_usage_the_and_".indexOf(word) < 0) && + (!(query.queryHashes.contains(plasmaCondenser.word2hash(word))))) + ref.incScore(word); + } + } + + protected void addReferences(plasmaSearchEvent.ResultEntry resultEntry) { + // take out relevant information for reference computation + if ((resultEntry.url() == null) || (resultEntry.title() == null)) return; + String[] urlcomps = htmlFilterContentScraper.urlComps(resultEntry.url().toNormalform(true, true)); // word components of the url + String[] descrcomps = resultEntry.title().toLowerCase().split(htmlFilterContentScraper.splitrex); // words in the description + + // add references + addReferences(urlcomps); + addReferences(descrcomps); + } + +} diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index 529e7be07..b43b9c005 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -30,7 +30,6 @@ import java.io.UnsupportedEncodingException; import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
@@ -40,8 +39,8 @@ import de.anomic.index.indexContainer; import de.anomic.index.indexRWIEntry;
import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroBitfield;
-import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMSetTools;
+import de.anomic.kelondro.kelondroRow;
import de.anomic.net.URL;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
@@ -51,7 +50,7 @@ import de.anomic.yacy.yacySeed; public final class plasmaSearchEvent {
- //public static plasmaSearchEvent lastEvent = null;
+ public static int workerThreadCount = 5;
public static String lastEventID = "";
private static HashMap lastEvents = new HashMap(); // a cache for objects from this class: re-use old search requests
public static final long eventLifetime = 600000; // the time an event will stay in the cache, 10 Minutes
@@ -60,25 +59,23 @@ public final class plasmaSearchEvent { private plasmaSearchQuery query;
private plasmaSearchRankingProfile ranking;
private plasmaWordIndex wordIndex;
- private indexContainer rcLocal; // cache for local results
- private indexContainer rcGlobal; // cache for global results
+ private plasmaSearchContainer rankedCache; // ordered search results, grows dynamically as all the query threads enrich this container
private Map rcAbstracts; // cache for index abstracts; word:TreeMap mapping where the embedded TreeMap is a urlhash:peerlist relation
- private plasmaSearchProcessing profileLocal, profileGlobal;
+ private plasmaSearchProcessing process;
private yacySearch[] primarySearchThreads, secondarySearchThreads;
private TreeMap preselectedPeerHashes;
- private int localcount, globalcount;
- private indexContainer sortedResults;
- private int lastglobal;
- private int filteredCount;
- private ArrayList display; // an array of url hashes of urls that had been displayed as search result after this search
private Object[] references;
public TreeMap IAResults, IACount;
public String IAmaxcounthash, IAneardhthash;
+ private int localcount;
+ private resultWorker[] workerThreads;
+ private ArrayList resultList; // list of this.Entry objects
+ private HashMap failedURLs; // a mapping from a urlhash to a fail reason string
+ TreeSet snippetFetchWordHashes; // a set of word hashes that are used to match with the snippets
private plasmaSearchEvent(plasmaSearchQuery query,
plasmaSearchRankingProfile ranking,
plasmaSearchProcessing localTiming,
- plasmaSearchProcessing remoteTiming,
plasmaWordIndex wordIndex,
TreeMap preselectedPeerHashes,
boolean generateAbstracts,
@@ -87,56 +84,62 @@ public final class plasmaSearchEvent { this.wordIndex = wordIndex;
this.query = query;
this.ranking = ranking;
- this.rcLocal = null;
- this.rcGlobal = plasmaWordIndex.emptyContainer(null, 0);;
this.rcAbstracts = (query.queryHashes.size() > 1) ? new TreeMap() : null; // generate abstracts only for combined searches
- this.profileLocal = localTiming;
- this.profileGlobal = remoteTiming;
+ this.process = localTiming;
this.primarySearchThreads = null;
this.secondarySearchThreads = null;
this.preselectedPeerHashes = preselectedPeerHashes;
- this.localcount = 0;
- this.globalcount = 0;
- this.sortedResults = null;
- this.lastglobal = 0;
- this.display = new ArrayList();
this.references = new String[0];
this.IAResults = new TreeMap();
this.IACount = new TreeMap();
this.IAmaxcounthash = null;
this.IAneardhthash = null;
+ this.localcount = 0;
+ this.workerThreads = null;
+ this.resultList = new ArrayList(10); // this is the result set which is filled up with search results, enriched with snippets
+ this.failedURLs = new HashMap(); // a map of urls to reason strings where a worker thread tried to work on, but failed.
+
+ // snippets do not need to match with the complete query hashes,
+ // only with the query minus the stopwords which had not been used for the search
+ final TreeSet filtered = kelondroMSetTools.joinConstructive(query.queryHashes, plasmaSwitchboard.stopwords);
+ this.snippetFetchWordHashes = (TreeSet) query.queryHashes.clone();
+ if ((filtered != null) && (filtered.size() > 0)) {
+ kelondroMSetTools.excludeDestructive(this.snippetFetchWordHashes, plasmaSwitchboard.stopwords);
+ }
long start = System.currentTimeMillis();
if ((query.domType == plasmaSearchQuery.SEARCHDOM_GLOBALDHT) ||
(query.domType == plasmaSearchQuery.SEARCHDOM_CLUSTERALL)) {
+ this.rankedCache = new plasmaSearchContainer(query, ranking, plasmaSearchQuery.cleanQuery(query.queryString)[0]);
+
int fetchpeers = (int) (query.maximumTime / 500L); // number of target peers; means 10 peers in 10 seconds
if (fetchpeers > 50) fetchpeers = 50;
if (fetchpeers < 30) fetchpeers = 30;
// do a global search
// the result of the fetch is then in the rcGlobal
- serverLog.logFine("SEARCH_EVENT", "STARTING " + fetchpeers + " THREADS TO CATCH EACH " + profileGlobal.getTargetCount(plasmaSearchProcessing.PROCESS_POSTSORT) + " URLs WITHIN " + (profileGlobal.duetime() / 1000) + " SECONDS");
- long secondaryTimeout = System.currentTimeMillis() + profileGlobal.duetime() / 3 * 2;
- long primaryTimeout = System.currentTimeMillis() + profileGlobal.duetime();
+ process.startTimer();
+ serverLog.logFine("SEARCH_EVENT", "STARTING " + fetchpeers + " THREADS TO CATCH EACH " + query.displayResults() + " URLs");
primarySearchThreads = yacySearch.primaryRemoteSearches(
plasmaSearchQuery.hashSet2hashString(query.queryHashes),
plasmaSearchQuery.hashSet2hashString(query.excludeHashes),
"",
query.prefer,
query.urlMask,
+ query.displayResults(),
query.maxDistance,
wordIndex,
- rcGlobal,
+ rankedCache,
rcAbstracts,
fetchpeers,
plasmaSwitchboard.urlBlacklist,
- profileGlobal,
ranking,
query.constraint,
(query.domType == plasmaSearchQuery.SEARCHDOM_GLOBALDHT) ? null : preselectedPeerHashes);
-
+ process.yield("remote search thread start", primarySearchThreads.length);
+
// meanwhile do a local search
- Map[] searchContainerMaps = profileLocal.localSearchContainers(query, wordIndex, null);
+ Map[] searchContainerMaps = process.localSearchContainers(query, wordIndex, null);
// use the search containers to fill up rcAbstracts locally
/*
@@ -164,71 +167,31 @@ public final class plasmaSearchEvent { */
// join and exlcude the local result
- this.rcLocal =
+ indexContainer rcLocal =
(searchContainerMaps == null) ?
plasmaWordIndex.emptyContainer(null, 0) :
- profileLocal.localSearchJoinExclude(
+ process.localSearchJoinExclude(
searchContainerMaps[0].values(),
searchContainerMaps[1].values(),
- (query.queryHashes.size() == 0) ?
- 0 :
- profileLocal.getTargetTime(plasmaSearchProcessing.PROCESS_JOIN) * query.queryHashes.size() / (query.queryHashes.size() + query.excludeHashes.size()),
query.maxDistance);
-
+ this.localcount = rcLocal.size();
+
// sort the local containers and truncate it to a limited count,
// so following sortings together with the global results will be fast
- plasmaSearchPreOrder firstsort = new plasmaSearchPreOrder(query, profileLocal, ranking, rcLocal);
+ plasmaSearchPreOrder firstsort = new plasmaSearchPreOrder(query, process, ranking, rcLocal);
rcLocal = firstsort.strippedContainer(200);
-
- int prefetchIndex = 0;
- HashSet unknownURLs = new HashSet();
- String urlhash;
-
- // while we wait for the first time-out for index abstracts, we fetch urls form the url-db
- while ((System.currentTimeMillis() < secondaryTimeout) && (prefetchIndex < rcLocal.size())) {
- if (yacySearch.remainingWaiting(primarySearchThreads) == 0) break; // all threads have finished
- urlhash = new String(rcLocal.get(prefetchIndex).getColBytes(0));
- if (wordIndex.loadedURL.load(urlhash, null) == null) unknownURLs.add(urlhash);
- prefetchIndex++;
- }
-
- // eventually wait some more time to retrieve index abstracts from primary search
- while (System.currentTimeMillis() < secondaryTimeout) {
- if (yacySearch.remainingWaiting(primarySearchThreads) == 0) break; // all threads have finished
- try {Thread.sleep(100);} catch (InterruptedException e) {}
- }
-
- // evaluate index abstracts and start a secondary search
- if (rcAbstracts != null) prepareSecondarySearch();
-
- // while we wait for the second time-out for index abstracts, we fetch more urls form the url-db
- while ((System.currentTimeMillis() < primaryTimeout) && (prefetchIndex < rcLocal.size())) {
- if (yacySearch.remainingWaiting(primarySearchThreads) == 0) break; // all threads have finished
- urlhash = new String(rcLocal.get(prefetchIndex).getColBytes(0));
- if (wordIndex.loadedURL.load(urlhash, null) == null) unknownURLs.add(urlhash);
- prefetchIndex++;
+ synchronized (rankedCache) {
+ this.rankedCache.insert(rcLocal, true, true);
}
-
- // when we have found some non-existing urls in the local collection, we delete them now
- wordIndex.removeEntriesMultiple(query.queryHashes, unknownURLs);
- rcLocal.removeEntriesMultiple(query.queryHashes, unknownURLs);
- localcount = rcLocal.size();
-
- // catch up global results:
- // wait until primary timeout passed
- while (System.currentTimeMillis() < primaryTimeout) {
- if ((yacySearch.remainingWaiting(primarySearchThreads) == 0) &&
- ((secondarySearchThreads == null) || (yacySearch.remainingWaiting(secondarySearchThreads) == 0))) break; // all threads have finished
- try {Thread.sleep(100);} catch (InterruptedException e) {}
- }
-
+
// finished searching
serverLog.logFine("SEARCH_EVENT", "SEARCH TIME AFTER GLOBAL-TRIGGER TO " + primarySearchThreads.length + " PEERS: " + ((System.currentTimeMillis() - start) / 1000) + " seconds");
} else {
- Map[] searchContainerMaps = profileLocal.localSearchContainers(query, wordIndex, null);
+ Map[] searchContainerMaps = process.localSearchContainers(query, wordIndex, null);
if (generateAbstracts) {
// compute index abstracts
+ process.startTimer();
Iterator ci = searchContainerMaps[0].entrySet().iterator();
Map.Entry entry;
int maxcount = -1;
@@ -252,26 +215,55 @@ public final class plasmaSearchEvent { IACount.put(wordhash, new Integer(container.size()));
IAResults.put(wordhash, plasmaURL.compressIndex(container, null, 1000).toString());
}
+ process.yield("abstract generation", searchContainerMaps[0].size());
}
- rcLocal =
+ indexContainer rcLocal =
(searchContainerMaps == null) ?
plasmaWordIndex.emptyContainer(null, 0) :
- profileLocal.localSearchJoinExclude(
+ process.localSearchJoinExclude(
searchContainerMaps[0].values(),
searchContainerMaps[1].values(),
- (query.queryHashes.size() == 0) ?
- 0 :
- profileLocal.getTargetTime(plasmaSearchProcessing.PROCESS_JOIN) * query.queryHashes.size() / (query.queryHashes.size() + query.excludeHashes.size()),
query.maxDistance);
this.localcount = rcLocal.size();
+ plasmaSearchPreOrder sort = new plasmaSearchPreOrder(query, process, ranking, rcLocal);
+
+ process.startTimer();
+ rcLocal = sort.strippedContainer(200);
+ process.yield("result strip", rcLocal.size());
+
+ this.rankedCache = new plasmaSearchContainer(query, ranking, plasmaSearchQuery.cleanQuery(query.queryString)[0], rcLocal);
}
- // log the event
- serverLog.logFine("SEARCH_EVENT", "SEARCHRESULT: " + profileLocal.reportToString());
-
- // set link for statistic
- //lastEvent = this;
+ if (query.onlineSnippetFetch) {
+ // start worker threads to fetch urls and snippets
+ this.workerThreads = new resultWorker[workerThreadCount];
+ for (int i = 0; i < workerThreadCount; i++) {
+ this.workerThreads[i] = new resultWorker(i, process.getTargetTime() * 3);
+ this.workerThreads[i].start();
+ }
+ } else {
+ // prepare result vector directly without worker threads
+ int rankedIndex = 0;
+ while ((rankedIndex < rankedCache.container().size()) && (resultList.size() < (query.neededResults()))) {
+ // fetch next entry to work on
+ indexContainer c = rankedCache.container();
+ indexRWIEntry entry = new indexRWIEntry(c.get(rankedIndex++));
+
+ ResultEntry resultEntry = obtainResultEntry(entry, false);
+ if (resultEntry == null) continue; // the entry had some problems, cannot be used
+
+ // place the result to the result vector
+ synchronized (resultList) {
+ resultList.add(resultEntry);
+ }
+
+ // add references
+ synchronized (rankedCache) {
+ rankedCache.addReferences(resultEntry);
+ }
+ }
+ }
// remove old events in the event cache
Iterator i = lastEvents.entrySet().iterator();
@@ -283,6 +275,120 @@ public final class plasmaSearchEvent { lastEvents.put(query.id(), this);
lastEventID = query.id();
}
+
+ private ResultEntry obtainResultEntry(indexRWIEntry entry, boolean fetchSnippetOnline) {
+
+ // a search result entry needs some work to produce a result Entry:
+ // - check if url entry exists in LURL-db
+ // - check exclusions, constraints, masks, media-domains
+ // - load snippet (see if page exists) and check if snippet contains searched word
+
+ // load only urls if there was not yet a root url of that hash
+ // find the url entry
+ indexURLEntry page = wordIndex.loadedURL.load(entry.urlHash(), entry);
+
+ if (page == null) {
+ registerFailure(entry.urlHash(), "url does not exist in lurl-db");
+ return null;
+ }
+
+ indexURLEntry.Components comp = page.comp();
+ String pagetitle = comp.title().toLowerCase();
+ if (comp.url() == null) {
+ registerFailure(entry.urlHash(), "url corrupted (null)");
+ return null; // rare case where the url is corrupted
+ }
+ String pageurl = comp.url().toString().toLowerCase();
+ String pageauthor = comp.author().toLowerCase();
+
+ // check exclusion
+ if (plasmaSearchQuery.matches(pagetitle, query.excludeHashes)) {
+ registerFailure(entry.urlHash(), "query-exclusion matches title: " + pagetitle);
+ return null;
+ }
+ if (plasmaSearchQuery.matches(pageurl, query.excludeHashes)) {
+ registerFailure(entry.urlHash(), "query-exclusion matches title: " + pagetitle);
+ return null;
+ }
+ if (plasmaSearchQuery.matches(pageauthor, query.excludeHashes)) {
+ registerFailure(entry.urlHash(), "query-exclusion matches title: " + pagetitle);
+ return null;
+ }
+
+ // check url mask
+ if (!(pageurl.matches(query.urlMask))) {
+ registerFailure(entry.urlHash(), "url-exclusion matches urlMask: " + pageurl);
+ return null;
+ }
+
+ // check constraints
+ if ((!(query.constraint.equals(plasmaSearchQuery.catchall_constraint))) &&
+ (query.constraint.get(plasmaCondenser.flag_cat_indexof)) &&
+ (!(comp.title().startsWith("Index of")))) {
+ final Iterator wi = query.queryHashes.iterator();
+ while (wi.hasNext()) wordIndex.removeEntry((String) wi.next(), page.hash());
+ registerFailure(entry.urlHash(), "index-of constrained not fullfilled");
+ return null;
+ }
+
+ if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_AUDIO) && (page.laudio() == 0)) {
+ registerFailure(entry.urlHash(), "contentdom-audio constrained not fullfilled");
+ return null;
+ }
+ if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_VIDEO) && (page.lvideo() == 0)) {
+ registerFailure(entry.urlHash(), "contentdom-video constrained not fullfilled");
+ return null;
+ }
+ if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_IMAGE) && (page.limage() == 0)) {
+ registerFailure(entry.urlHash(), "contentdom-image constrained not fullfilled");
+ return null;
+ }
+ if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_APP) && (page.lapp() == 0)) {
+ registerFailure(entry.urlHash(), "contentdom-app constrained not fullfilled");
+ return null;
+ }
+
+ // load snippet
+ if (query.contentdom == plasmaSearchQuery.CONTENTDOM_TEXT) {
+ // attach text snippet
+ plasmaSnippetCache.TextSnippet snippet = plasmaSnippetCache.retrieveTextSnippet(comp.url(), snippetFetchWordHashes, fetchSnippetOnline, query.constraint.get(plasmaCondenser.flag_cat_indexof), 260, 6000);
+ if (snippet.getErrorCode() < 11) {
+ // we loaded the file and found the snippet
+ return new ResultEntry(page, wordIndex, snippet, null); // result with snippet attached
+ } else if (!fetchSnippetOnline) {
+ // we did not demand online loading, therefore a failure does not mean that the missing snippet causes a rejection of this result
+ // this may happen during a remote search, because snippet loading is omitted to retrieve results faster
+ return new ResultEntry(page, wordIndex, null, null); // result without snippet
+ } else {
+ // problems with snippet fetch
+ registerFailure(entry.urlHash(), "no text snippet for URL " + comp.url());
+ plasmaSnippetCache.failConsequences(snippet, query.id());
+ return null;
+ }
+ } else {
+ // attach media information
+ ArrayList mediaSnippets = plasmaSnippetCache.retrieveMediaSnippets(comp.url(), snippetFetchWordHashes, query.contentdom, fetchSnippetOnline, 6000);
+ if ((mediaSnippets != null) && (mediaSnippets.size() > 0)) {
+ // found media snippets, return entry
+ return new ResultEntry(page, wordIndex, null, mediaSnippets);
+ } else if (!fetchSnippetOnline) {
+ return new ResultEntry(page, wordIndex, null, null);
+ } else {
+ // problems with snippet fetch
+ registerFailure(entry.urlHash(), "no media snippet for URL " + comp.url());
+ return null;
+ }
+ }
+ // finished, no more actions possible here
+ }
+
+ private boolean anyWorkerAlive() {
+ if (this.workerThreads == null) return false;
+ for (int i = 0; i < workerThreadCount; i++) {
+ if ((this.workerThreads[i] != null) && (this.workerThreads[i].isAlive())) return true;
+ }
+ return false;
+ }
public plasmaSearchQuery getQuery() {
return query;
@@ -292,8 +398,8 @@ public final class plasmaSearchEvent { return ranking;
}
- public plasmaSearchProcessing getLocalTiming() {
- return profileLocal;
+ public plasmaSearchProcessing getProcess() {
+ return process;
}
public yacySearch[] getPrimarySearchThreads() {
@@ -308,7 +414,7 @@ public final class plasmaSearchEvent { }
public int getGlobalCount() {
- return this.globalcount;
+ return this.rankedCache.getGlobalCount();
}
public static plasmaSearchEvent getEvent(String eventID) {
@@ -318,128 +424,167 @@ public final class plasmaSearchEvent { public static plasmaSearchEvent getEvent(plasmaSearchQuery query,
plasmaSearchRankingProfile ranking,
plasmaSearchProcessing localTiming,
- plasmaSearchProcessing remoteTiming,
plasmaWordIndex wordIndex,
TreeMap preselectedPeerHashes,
boolean generateAbstracts,
TreeSet abstractSet) {
plasmaSearchEvent event = (plasmaSearchEvent) lastEvents.get(query.id());
if (event == null) {
- event = new plasmaSearchEvent(query, ranking, localTiming, remoteTiming, wordIndex, preselectedPeerHashes, generateAbstracts, abstractSet);
+ event = new plasmaSearchEvent(query, ranking, localTiming, wordIndex, preselectedPeerHashes, generateAbstracts, abstractSet);
} else {
//re-new the event time for this event, so it is not deleted next time too early
event.eventTime = System.currentTimeMillis();
+ // replace the query, because this contains the current result offset
+ event.query = query;
}
- return event;
- }
-
- private indexContainer search() {
- // combine the local and global (if any) result and order
- if ((rcGlobal != null) && (rcGlobal.size() > 0)) {
- globalcount = rcGlobal.size();
- if ((this.sortedResults == null) || (this.lastglobal != globalcount)) {
- indexContainer searchResult = plasmaWordIndex.emptyContainer(null, rcLocal.size() + rcGlobal.size());
- searchResult.addAllUnique(rcLocal);
- searchResult.addAllUnique(rcGlobal);
- searchResult.sort();
- searchResult.uniq(100);
- lastglobal = globalcount;
- plasmaSearchPreOrder pre = new plasmaSearchPreOrder(query, profileLocal, ranking, searchResult);
- this.filteredCount = pre.filteredCount();
- this.sortedResults = pre.strippedContainer(200);
- }
- } else {
- if (this.sortedResults == null) {
- plasmaSearchPreOrder pre = new plasmaSearchPreOrder(query, profileLocal, ranking, rcLocal);
- this.filteredCount = pre.filteredCount();
- this.sortedResults = pre.strippedContainer(200);
+
+ // if worker threads had been alive, but did not succeed, start them again to fetch missing links
+ if ((query.onlineSnippetFetch) &&
+ (!event.anyWorkerAlive()) &&
+ (event.resultList.size() < query.neededResults()) &&
+ ((event.getLocalCount() + event.getGlobalCount()) > event.resultList.size())) {
+ // set new timeout
+ event.eventTime = System.currentTimeMillis();
+ // start worker threads to fetch urls and snippets
+ event.workerThreads = new resultWorker[workerThreadCount];
+ for (int i = 0; i < workerThreadCount; i++) {
+ event.workerThreads[i] = event.deployWorker(i, 3 * event.process.getTargetTime());
}
}
- return this.sortedResults;
+ return event;
+ }
+
+ private resultWorker deployWorker(int id, long lifetime) {
+ resultWorker worker = new resultWorker(id, lifetime);
+ worker.start();
+ return worker;
}
- public ArrayList computeResults(
- TreeSet blueList,
- boolean overfetch) {
+ private class resultWorker extends Thread {
- indexContainer pre = search();
- final ArrayList hits = new ArrayList();
+ private indexRWIEntry entry; // entry this thread is working on
+ private long timeout; // the date until this thread should try to work
+ private long sleeptime; // the sleeptime of this thread at the beginning of its life
+ private int id;
- // start url-fetch
- final long postorderTime = this.profileLocal.getTargetTime(plasmaSearchProcessing.PROCESS_POSTSORT);
- //System.out.println("DEBUG: postorder-final (urlfetch) maxtime = " + postorderTime);
- final long postorderLimitTime = (postorderTime < 0) ? Long.MAX_VALUE : (System.currentTimeMillis() + postorderTime);
- this.profileLocal.startTimer();
- final plasmaSearchPostOrder acc = new plasmaSearchPostOrder(query, ranking);
+ public resultWorker(int id, long lifetime) {
+ this.id = id;
+ this.timeout = System.currentTimeMillis() + lifetime;
+ this.sleeptime = lifetime / 10 * id;
+ this.entry = null;
+ }
+
+ public void run() {
+
+ // sleep first to give remote loading threads a chance to fetch entries
+ try {Thread.sleep(this.sleeptime);} catch (InterruptedException e1) {}
+
+ // start fetching urls and snippets
+ while ((resultList.size() < query.neededResults() + query.displayResults()) && (System.currentTimeMillis() < this.timeout)) {
+
+ // try secondary search
+ prepareSecondarySearch();
+
+ // fetch next entry to work on
+ this.entry = null;
+ entry = nextOrder();
+ if (entry == null) {
+ // wait and try again
+ try {Thread.sleep(200);} catch (InterruptedException e) {}
+ continue;
+ }
+
+ ResultEntry resultEntry = obtainResultEntry(entry, true);
+ if (resultEntry == null) continue; // the entry had some problems, cannot be used
+
+ // place the result to the result vector
+ synchronized (resultList) {
+ resultList.add(resultEntry);
+ }
+
+ // add references
+ synchronized (rankedCache) {
+ rankedCache.addReferences(resultEntry);
+ }
+
+ System.out.println("DEBUG SNIPPET_LOADING: thread " + id + " got " + resultEntry.url());
+ }
+ System.out.println("DEBUG: resultWorker thread " + id + " terminated");
+ }
- indexRWIEntry rwientry;
- indexURLEntry page;
- indexURLEntry.Components comp;
- String pagetitle, pageurl, pageauthor;
- final int minEntries = this.profileLocal.getTargetCount(plasmaSearchProcessing.PROCESS_POSTSORT);
- try {
- ordering: for (int i = 0; i < pre.size(); i++) {
- if ((System.currentTimeMillis() >= postorderLimitTime) || (acc.sizeFetched() >= ((overfetch) ? 4 : 1) * minEntries)) break;
- rwientry = new indexRWIEntry(pre.get(i));
- // load only urls if there was not yet a root url of that hash
- // find the url entry
- page = wordIndex.loadedURL.load(rwientry.urlHash(), rwientry);
- if (page != null) {
- comp = page.comp();
- pagetitle = comp.title().toLowerCase();
- if (comp.url() == null) continue ordering; // rare case where the url is corrupted
- pageurl = comp.url().toString().toLowerCase();
- pageauthor = comp.author().toLowerCase();
-
- // check exclusion
- if (plasmaSearchQuery.matches(pagetitle, query.excludeHashes)) continue ordering;
- if (plasmaSearchQuery.matches(pageurl, query.excludeHashes)) continue ordering;
- if (plasmaSearchQuery.matches(pageauthor, query.excludeHashes)) continue ordering;
-
- // check url mask
- if (!(pageurl.matches(query.urlMask))) continue ordering;
-
- // check constraints
- if ((!(query.constraint.equals(plasmaSearchQuery.catchall_constraint))) &&
- (query.constraint.get(plasmaCondenser.flag_cat_indexof)) &&
- (!(comp.title().startsWith("Index of")))) {
- serverLog.logFine("PLASMA", "filtered out " + comp.url().toString());
- // filter out bad results
- final Iterator wi = query.queryHashes.iterator();
- while (wi.hasNext()) wordIndex.removeEntry((String) wi.next(), page.hash());
- } else if (query.contentdom != plasmaSearchQuery.CONTENTDOM_TEXT) {
- if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_AUDIO) && (page.laudio() > 0)) acc.addPage(page);
- else if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_VIDEO) && (page.lvideo() > 0)) acc.addPage(page);
- else if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_IMAGE) && (page.limage() > 0)) acc.addPage(page);
- else if ((query.contentdom == plasmaSearchQuery.CONTENTDOM_APP) && (page.lapp() > 0)) acc.addPage(page);
- } else {
- acc.addPage(page);
- }
+ private indexRWIEntry nextOrder() {
+ synchronized (rankedCache) {
+ indexContainer c = rankedCache.container();
+ kelondroRow.Entry entry;
+ String urlhash;
+ for (int i = 0; i < c.size(); i++) {
+ entry = c.get(i);
+ urlhash = new String(entry.getColBytes(0));
+ if ((anyFailureWith(urlhash)) || (anyWorkerWith(urlhash)) || (anyResultWith(urlhash))) continue;
+ return new indexRWIEntry(entry);
}
}
- } catch (final kelondroException ee) {
- serverLog.logSevere("PLASMA", "Database Failure during plasmaSearch.order: " + ee.getMessage(), ee);
+ return null; // no more entries available
}
- this.profileLocal.setYieldTime(plasmaSearchProcessing.PROCESS_URLFETCH);
- this.profileLocal.setYieldCount(plasmaSearchProcessing.PROCESS_URLFETCH, acc.sizeFetched());
- // start postsorting
- this.profileLocal.startTimer();
- acc.sortPages(true);
- this.profileLocal.setYieldTime(plasmaSearchProcessing.PROCESS_POSTSORT);
- this.profileLocal.setYieldCount(plasmaSearchProcessing.PROCESS_POSTSORT, acc.sizeOrdered());
+ private boolean anyWorkerWith(String urlhash) {
+ for (int i = 0; i < workerThreadCount; i++) {
+ if ((workerThreads[i] == null) || (workerThreads[i] == this)) continue;
+ if ((workerThreads[i].entry != null) && (workerThreads[i].entry.urlHash().equals(urlhash))) return true;
+ }
+ return false;
+ }
+ private boolean anyResultWith(String urlhash) {
+ for (int i = 0; i < resultList.size(); i++) {
+ if (((ResultEntry) resultList.get(i)).urlentry.hash().equals(urlhash)) return true;
+ }
+ return false;
+ }
- // apply filter
- this.profileLocal.startTimer();
- acc.removeRedundant();
- this.profileLocal.setYieldTime(plasmaSearchProcessing.PROCESS_FILTER);
- this.profileLocal.setYieldCount(plasmaSearchProcessing.PROCESS_FILTER, acc.sizeOrdered());
+ private boolean anyFailureWith(String urlhash) {
+ return (failedURLs.get(urlhash) != null);
+ }
+ }
+
+ private void registerFailure(String urlhash, String reason) {
+ this.failedURLs.put(urlhash, reason);
+ serverLog.logInfo("search", "sorted out hash " + urlhash + " during search: " + reason);
+ }
+
+
+ public ResultEntry oneResult(int item) {
+ // first sleep a while to give accumulation threads a chance to work
+ long sleeptime = this.eventTime + (this.query.maximumTime / this.query.displayResults() * ((item % this.query.displayResults()) + 1)) - System.currentTimeMillis();
+ if ((query.domType == plasmaSearchQuery.SEARCHDOM_GLOBALDHT) &&
+ (anyWorkerAlive()) &&
+ (sleeptime > 0)) try {Thread.sleep(sleeptime);} catch (InterruptedException e) {}
- // generate references
- this.references = acc.getReferences(16);
+ // then sleep until a result is available
+ while ((this.resultList.size() <= item) && (anyWorkerAlive())) {
+ try {Thread.sleep(100);} catch (InterruptedException e) {}
+ }
+ // finally, if there is something, return the result
+ synchronized (this.resultList) {
+ if (this.resultList.size() <= item) return null;
+
+ // todo: fetch best result (switch) from item position to end of resultList
+
+ return (ResultEntry) this.resultList.get(item);
+ }
+ }
+
+ public ArrayList completeResults(long waitingtime) {
+ long timeout = System.currentTimeMillis() + waitingtime;
+ while ((this.resultList.size() < query.neededResults()) && (anyWorkerAlive()) && (System.currentTimeMillis() < timeout)) {
+ try {Thread.sleep(200);} catch (InterruptedException e) {}
+ }
+ return this.resultList;
+ }
+
+ /*
// generate Result.Entry objects and optionally fetch snippets
int i = 0;
Entry entry;
@@ -454,11 +599,7 @@ public final class plasmaSearchEvent { // bluelisted word
// appear either in url, url's description or search word
// the search word was sorted out earlier
- /*
- * String s = descr.toLowerCase() + url.toString().toLowerCase();
- * for (int c = 0; c < blueList.length; c++) { if
- * (s.indexOf(blueList[c]) >= 0) return; }
- */
+
if (includeSnippets) {
entry.setSnippet(plasmaSnippetCache.retrieveTextSnippet(
entry.url(), query.queryHashes, false,
@@ -476,29 +617,19 @@ public final class plasmaSearchEvent { i++;
hits.add(entry);
}
-
- /*
- * while ((acc.hasMoreElements()) && (((time + timestamp) <
- * System.currentTimeMillis()))) { urlentry = acc.nextElement();
- * urlstring = htmlFilterContentScraper.urlNormalform(urlentry.url());
- * descr = urlentry.descr();
- *
- * addScoreForked(ref, gs, descr.split(" ")); addScoreForked(ref, gs,
- * urlstring.split("/")); }
- */
- return hits;
- }
+ */
- public int filteredCount() {
- return this.filteredCount;
- }
+ boolean secondarySearchStartet = false;
private void prepareSecondarySearch() {
- // catch up index abstracts and join them; then call peers again to submit their urls
- System.out.println("DEBUG-INDEXABSTRACT: " + rcAbstracts.size() + " word references catched, " + query.queryHashes.size() + " needed");
+ if (secondarySearchStartet) return; // dont do this twice
- if (rcAbstracts.size() != query.queryHashes.size()) return; // secondary search not possible
+ if ((rcAbstracts == null) || (rcAbstracts.size() != query.queryHashes.size())) return; // secondary search not possible (yet)
+ this.secondarySearchStartet = true;
+ // catch up index abstracts and join them; then call peers again to submit their urls
+ System.out.println("DEBUG-INDEXABSTRACT: " + rcAbstracts.size() + " word references catched, " + query.queryHashes.size() + " needed");
+
Iterator i = rcAbstracts.entrySet().iterator();
Map.Entry entry;
while (i.hasNext()) {
@@ -550,8 +681,8 @@ public final class plasmaSearchEvent { System.out.println("DEBUG-INDEXABSTRACT ***: peer " + peer + " has urls: " + urls);
System.out.println("DEBUG-INDEXABSTRACT ***: peer " + peer + " from words: " + words);
secondarySearchThreads[c++] = yacySearch.secondaryRemoteSearch(
- words, "", urls, wordIndex, rcGlobal, peer, plasmaSwitchboard.urlBlacklist,
- profileGlobal, ranking, query.constraint, preselectedPeerHashes);
+ words, "", urls, wordIndex, this.rankedCache, peer, plasmaSwitchboard.urlBlacklist,
+ ranking, query.constraint, preselectedPeerHashes);
}
}
@@ -587,32 +718,29 @@ public final class plasmaSearchEvent { public void remove(String urlhash) {
// removes the url hash reference from last search result
- indexRWIEntry e = this.sortedResults.remove(urlhash);
- assert e != null;
- rcLocal.remove(urlhash);
- }
-
- public void displayed(String urlhash, int position) {
- this.display.set(position, urlhash);
+ /*indexRWIEntry e =*/ this.rankedCache.remove(urlhash);
+ //assert e != null;
}
public Object[] references() {
return this.references;
}
- public static class Entry {
+ public static class ResultEntry {
private indexURLEntry urlentry;
private indexURLEntry.Components urlcomps; // buffer for components
private String alternative_urlstring;
private String alternative_urlname;
- private plasmaSnippetCache.TextSnippet snippet;
+ private plasmaSnippetCache.TextSnippet textSnippet;
+ private ArrayList /* of plasmaSnippetCache.MediaSnippet */ mediaSnippets;
- public Entry(indexURLEntry urlentry, plasmaWordIndex wordIndex) {
+ public ResultEntry(indexURLEntry urlentry, plasmaWordIndex wordIndex, plasmaSnippetCache.TextSnippet textSnippet, ArrayList mediaSnippets) {
this.urlentry = urlentry;
this.urlcomps = urlentry.comp();
this.alternative_urlstring = null;
this.alternative_urlname = null;
- this.snippet = null;
+ this.textSnippet = textSnippet;
+ this.mediaSnippets = mediaSnippets;
String host = urlcomps.url().getHost();
if (host.endsWith(".yacyh")) {
// translate host into current IP
@@ -660,11 +788,11 @@ public final class plasmaSearchEvent { public String title() {
return urlcomps.title();
}
- public void setSnippet(plasmaSnippetCache.TextSnippet snippet) {
- this.snippet = snippet;
+ public plasmaSnippetCache.TextSnippet textSnippet() {
+ return this.textSnippet;
}
- public plasmaSnippetCache.TextSnippet snippet() {
- return this.snippet;
+ public ArrayList /* of plasmaSnippetCache.MediaSnippet */ mediaSnippets() {
+ return this.mediaSnippets;
}
public Date modified() {
return urlentry.moddate();
@@ -675,16 +803,16 @@ public final class plasmaSearchEvent { public indexRWIEntry word() {
return urlentry.word();
}
- public boolean hasSnippet() {
- return false;
+ public boolean hasTextSnippet() {
+ return (this.textSnippet != null) && (this.textSnippet.getErrorCode() < 11);
}
- public plasmaSnippetCache.TextSnippet textSnippet() {
- return null;
+ public boolean hasMediaSnippets() {
+ return (this.mediaSnippets != null) && (this.mediaSnippets.size() > 0);
}
public String resource() {
// generate transport resource
- if ((snippet != null) && (snippet.exists())) {
- return urlentry.toString(snippet.getLineRaw());
+ if ((textSnippet != null) && (textSnippet.exists())) {
+ return urlentry.toString(textSnippet.getLineRaw());
} else {
return urlentry.toString();
}
diff --git a/source/de/anomic/plasma/plasmaSearchImages.java b/source/de/anomic/plasma/plasmaSearchImages.java index 5078d44a8..398c99625 100644 --- a/source/de/anomic/plasma/plasmaSearchImages.java +++ b/source/de/anomic/plasma/plasmaSearchImages.java @@ -48,7 +48,6 @@ import java.util.Map; import java.util.TreeSet;
import de.anomic.htmlFilter.htmlFilterImageEntry;
-import de.anomic.index.indexURLEntry;
import de.anomic.net.URL;
import de.anomic.plasma.parser.ParserException;
import de.anomic.server.serverDate;
@@ -99,16 +98,6 @@ public final class plasmaSearchImages { }
}
- public plasmaSearchImages(long maxTime, plasmaSearchPostOrder sres, int depth) {
- long start = System.currentTimeMillis();
- this.images = new TreeSet();
- indexURLEntry urlentry;
- while (sres.hasMoreElements()) {
- urlentry = sres.nextElement();
- addAll(new plasmaSearchImages(serverDate.remainingTime(start, maxTime, 10), urlentry.comp().url(), depth));
- }
- }
-
public void addAll(plasmaSearchImages m) {
synchronized (m.images) {
addAll(m.images);
diff --git a/source/de/anomic/plasma/plasmaSearchPostOrder.java b/source/de/anomic/plasma/plasmaSearchPostOrder.java deleted file mode 100644 index 4e90742f4..000000000 --- a/source/de/anomic/plasma/plasmaSearchPostOrder.java +++ /dev/null @@ -1,271 +0,0 @@ -// plasmaSearchResult.java
-// -----------------------
-// part of YACY
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2005
-// Created: 10.10.2005
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
-
-
-package de.anomic.plasma;
-
-import java.net.MalformedURLException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-
-import de.anomic.htmlFilter.htmlFilterContentScraper;
-import de.anomic.plasma.plasmaURL;
-import de.anomic.index.indexURLEntry;
-import de.anomic.kelondro.kelondroMScoreCluster;
-import de.anomic.net.URL;
-import de.anomic.server.serverCodings;
-
-public final class plasmaSearchPostOrder {
-
- private TreeMap pageAcc; // key = order hash; value = plasmaLURL.entry
- private kelondroMScoreCluster ref; // reference score computation for the commonSense heuristic
- private ArrayList results; // this is a buffer for plasmaWordIndexEntry + plasmaCrawlLURL.entry - objects
- private plasmaSearchQuery query;
- private plasmaSearchRankingProfile ranking;
-
- public plasmaSearchPostOrder(plasmaSearchQuery query, plasmaSearchRankingProfile ranking) {
- this.pageAcc = new TreeMap();
- this.ref = new kelondroMScoreCluster();
- this.results = new ArrayList();
- this.query = query;
- this.ranking = ranking;
- }
-
- public plasmaSearchPostOrder cloneSmart() {
- // clones only the top structure
- plasmaSearchPostOrder theClone = new plasmaSearchPostOrder(this.query, this.ranking);
- theClone.pageAcc = (TreeMap) this.pageAcc.clone();
- theClone.ref = this.ref;
- theClone.results = this.results;
- return theClone;
- }
-
- public int sizeOrdered() {
- return pageAcc.size();
- }
-
- public int sizeFetched() {
- return results.size();
- }
-
- public boolean hasMoreElements() {
- return pageAcc.size() > 0;
- }
-
- public indexURLEntry nextElement() {
- Object top = pageAcc.firstKey();
- //System.out.println("postorder-key: " + ((String) top));
- return (indexURLEntry) pageAcc.remove(top);
- }
-
- protected void addPage(indexURLEntry page) {
-
- // take out relevant information for reference computation
- indexURLEntry.Components comp = page.comp();
- if ((comp.url() == null) || (comp.title() == null)) return;
- String[] urlcomps = htmlFilterContentScraper.urlComps(comp.url().toNormalform(true, true)); // word components of the url
- String[] descrcomps = comp.title().toLowerCase().split(htmlFilterContentScraper.splitrex); // words in the description
-
- // store everything
- results.add(new Object[] {page, urlcomps, descrcomps});
-
- // add references
- addScoreFiltered(urlcomps);
- addScoreFiltered(descrcomps);
- }
-
- protected void sortPages(boolean postsort) {
- // finally sort the results
-
- // create a commonSense - set that represents a set of words that is
- // treated as 'typical' for this search request
- Object[] references = getReferences(16);
- Set commonSense = new HashSet();
- for (int i = 0; i < references.length; i++) commonSense.add(references[i]);
-
- Object[] resultVector;
- indexURLEntry page;
- long ranking;
- for (int i = 0; i < results.size(); i++) {
- // take out values from result array
- resultVector = (Object[]) results.get(i);
- page = (indexURLEntry) resultVector[0];
-
- // calculate ranking
- if (postsort)
- ranking = this.ranking.postRanking(
- query,
- commonSense,
- (String[]) resultVector[1],
- (String[]) resultVector[2],
- page,
- i
- );
- else
- ranking = ((Long) resultVector[3]).longValue();
-
- // insert value
- //System.out.println("Ranking " + ranking + ", YBR-" + plasmaSearchPreOrder.ybr(indexEntry.getUrlHash()) + " for URL " + page.url());
- pageAcc.put(serverCodings.encodeHex(Long.MAX_VALUE - ranking, 16) + page.hash(), page);
- }
-
- // flush memory
- results = null;
- }
-
- public void removeRedundant() {
- // remove all urls from the pageAcc structure that occur double by specific redundancy rules
- // a link is redundant, if a sub-path of the url is cited before. redundant urls are removed
- // we find redundant urls by iteration over all elements in pageAcc
- Iterator i = pageAcc.entrySet().iterator();
- HashMap paths = new HashMap(); // a url-subpath to pageAcc-key relation
- Map.Entry entry;
-
- // first scan all entries and find all urls that are referenced
- while (i.hasNext()) {
- entry = (Map.Entry) i.next();
- paths.put(((indexURLEntry) entry.getValue()).comp().url().toNormalform(true, true), entry.getKey());
- //if (path != null) path = shortenPath(path);
- //if (path != null) paths.put(path, entry.getKey());
- }
-
- // now scan the pageAcc again and remove all redundant urls
- i = pageAcc.entrySet().iterator();
- String shorten;
- while (i.hasNext()) {
- entry = (Map.Entry) i.next();
- shorten = shortenPath(((indexURLEntry) entry.getValue()).comp().url().toNormalform(true, true));
- // scan all subpaths of the url
- while (shorten != null) {
- if (pageAcc.size() <= query.wantedResults) break;
- if (paths.containsKey(shorten)) {
- //System.out.println("deleting path from search result: " + path + " is redundant to " + shorten);
- try {
- i.remove();
- } catch (IllegalStateException e) {
-
- }
- }
- shorten = shortenPath(shorten);
- }
- }
- }
-
- private static String shortenPath(String path) {
- int pos = path.lastIndexOf('/');
- if (pos < 0) return null;
- return path.substring(0, pos);
- }
- /*
- private static String urlPath(URL url) {
- String port = ((url.getPort() < 0) ? "" : ":" + url.getPort());
- String path = url.getPath();
- if (path.endsWith("/")) path = path.substring(0, path.length() - 1);
- int pos = path.lastIndexOf('/');
- if ((pos >= 0) && (path.length() > pos + 5) && (path.substring(pos + 1).toLowerCase().startsWith("index"))) {
- path = path.substring(0, pos);
- }
- return url.getHost() + port + path;
- }
- */
- public Object[] getReferences(int count) {
- // create a list of words that had been computed by statistics over all
- // words that appeared in the url or the description of all urls
- return ref.getScores(count, false, 2, Integer.MAX_VALUE);
- }
-
- public void addScoreFiltered(String[] words) {
- String word;
- for (int i = 0; i < words.length; i++) {
- word = words[i].toLowerCase();
- if ((word.length() > 2) &&
- ("http_html_php_ftp_www_com_org_net_gov_edu_index_home_page_for_usage_the_and_".indexOf(word) < 0) &&
- (!(query.queryHashes.contains(plasmaCondenser.word2hash(word)))))
- ref.incScore(word);
- }
- }
-
- /*
- private void printSplitLog(String x, String[] y) {
- String s = "";
- for (int i = 0; i < y.length; i++) s = s + ", " + y[i];
- if (s.length() > 0) s = s.substring(2);
- System.out.println("Split '" + x + "' = {" + s + "}");
- }
- */
-
- public static void main(String[] args) {
- URL[] urls = new URL[10];
- try {
- urls[0] = new URL("http://www.yacy.net");
- urls[1] = new URL("http://www.yacy.de/");
- urls[2] = new URL("http://yacy.net/");
- urls[3] = new URL("http://www.yacy.net:80/");
- urls[4] = new URL("http://yacy.net:80/");
- urls[5] = new URL("http://www.yacy.net/index.html");
- urls[6] = new URL("http://www.yacy.net/yacy");
- urls[7] = new URL("http://www.yacy.net/yacy/");
- urls[8] = new URL("http://www.yacy.net/yacy/index.html");
- urls[9] = new URL("ftp://www.yacy.net/yacy/index.html");
- String hash, fill;
- String[] paths1 = new String[urls.length]; for (int i = 0; i < urls.length; i++) {
- fill = ""; for (int j = 0; j < 35 - urls[i].toString().length(); j++) fill +=" ";
- paths1[i] = urls[i].toNormalform(true, true);
- hash = plasmaURL.urlHash(urls[i]);
- System.out.println("paths1[" + urls[i] + fill +"] = " + hash + ", typeID=" + plasmaURL.flagTypeID(hash) + ", tldID=" + plasmaURL.flagTLDID(hash) + ", lengthID=" + plasmaURL.flagLengthID(hash) + " / " + paths1[i]);
- }
- String[] paths2 = new String[urls.length]; for (int i = 0; i < urls.length; i++) {
- fill = ""; for (int j = 0; j < 35 - urls[i].toString().length(); j++) fill +=" ";
- paths2[i] = shortenPath(paths1[i]);
- hash = plasmaURL.urlHash(urls[i]);
- System.out.println("paths2[" + urls[i] + fill + "] = " + hash + ", typeID=" + plasmaURL.flagTypeID(hash) + ", tldID=" + plasmaURL.flagTLDID(hash) + ", lengthID=" + plasmaURL.flagLengthID(hash) + " / " + paths2[i]);
- }
- } catch (MalformedURLException e) {
- e.printStackTrace();
- }
- }
-
-}
\ No newline at end of file diff --git a/source/de/anomic/plasma/plasmaSearchPreOrder.java b/source/de/anomic/plasma/plasmaSearchPreOrder.java index 16b4dfc67..5abdb1f61 100644 --- a/source/de/anomic/plasma/plasmaSearchPreOrder.java +++ b/source/de/anomic/plasma/plasmaSearchPreOrder.java @@ -85,12 +85,8 @@ public final class plasmaSearchPreOrder { assert (container != null);
- long maxTime = process.getTargetTime(plasmaSearchProcessing.PROCESS_PRESORT);
process.startTimer();
- // set limit time for interruption
- long limitTime = (maxTime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxTime;
-
// first pass: find min/max to obtain limits for normalization
Iterator i = container.entries();
int count = 0;
@@ -98,7 +94,6 @@ public final class plasmaSearchPreOrder { this.entryMax = null;
indexRWIEntry iEntry;
while (i.hasNext()) {
- if (System.currentTimeMillis() > limitTime) break;
iEntry = (indexRWIEntry) i.next();
if (this.entryMin == null) this.entryMin = (indexRWIEntry) iEntry.clone(); else this.entryMin.min(iEntry);
if (this.entryMax == null) this.entryMax = (indexRWIEntry) iEntry.clone(); else this.entryMax.max(iEntry);
@@ -110,7 +105,6 @@ public final class plasmaSearchPreOrder { this.pageAcc = new TreeMap();
TreeSet searchWords = plasmaSearchQuery.cleanQuery(query.queryString)[0];
for (int j = 0; j < count; j++) {
- if (System.currentTimeMillis() > limitTime) break;
iEntry = (indexRWIEntry) i.next();
if (iEntry.urlHash().length() != container.row().width(container.row().primaryKey())) continue;
if ((!(query.constraint.equals(plasmaSearchQuery.catchall_constraint))) && (!(iEntry.flags().allOf(query.constraint)))) continue; // filter out entries that do not match the search constraint
@@ -124,10 +118,9 @@ public final class plasmaSearchPreOrder { }
this.filteredCount = pageAcc.size();
- if (container.size() > query.wantedResults) remove(true, true);
+ if (container.size() > query.neededResults()) remove(true, true);
- process.setYieldTime(plasmaSearchProcessing.PROCESS_PRESORT);
- process.setYieldCount(plasmaSearchProcessing.PROCESS_PRESORT, container.size());
+ process.yield(plasmaSearchProcessing.PRESORT, container.size());
}
public int filteredCount() {
@@ -136,7 +129,7 @@ public final class plasmaSearchPreOrder { private void remove(boolean rootDomExt, boolean doubleDom) {
// this removes all refererences to urls that are extended paths of existing 'RootDom'-urls
- if (pageAcc.size() <= query.wantedResults) return;
+ if (pageAcc.size() <= query.neededResults()) return;
HashSet rootDoms = new HashSet();
HashSet doubleDoms = new HashSet();
Iterator i = pageAcc.entrySet().iterator();
@@ -146,7 +139,7 @@ public final class plasmaSearchPreOrder { boolean isWordRootURL;
TreeSet querywords = plasmaSearchQuery.cleanQuery(query.queryString())[0];
while (i.hasNext()) {
- if (pageAcc.size() <= query.wantedResults) break;
+ if (pageAcc.size() <= query.neededResults()) break;
entry = (Map.Entry) i.next();
iEntry = (indexRWIEntry) entry.getValue();
hashpart = iEntry.urlHash().substring(6);
diff --git a/source/de/anomic/plasma/plasmaSearchProcessing.java b/source/de/anomic/plasma/plasmaSearchProcessing.java index 82837b663..5a4496970 100644 --- a/source/de/anomic/plasma/plasmaSearchProcessing.java +++ b/source/de/anomic/plasma/plasmaSearchProcessing.java @@ -26,8 +26,10 @@ package de.anomic.plasma; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; import java.util.Set; @@ -66,211 +68,70 @@ public class plasmaSearchProcessing implements Cloneable { // time = time to fetch snippets for selected URLs // count = maximum number of snipptes to be fetched - public static final char PROCESS_COLLECTION = 'c'; - public static final char PROCESS_JOIN = 'j'; - public static final char PROCESS_PRESORT = 'r'; - public static final char PROCESS_URLFETCH = 'u'; - public static final char PROCESS_POSTSORT = 'o'; - public static final char PROCESS_FILTER = 'f'; - public static final char PROCESS_SNIPPETFETCH = 's'; + public static final String COLLECTION = "collection"; + public static final String JOIN = "join"; + public static final String PRESORT = "presort"; + public static final String URLFETCH = "urlfetch"; private static final long minimumTargetTime = 100; - - public static char[] sequence = new char[]{ - PROCESS_COLLECTION, - PROCESS_JOIN, - PROCESS_PRESORT, - PROCESS_URLFETCH, - PROCESS_POSTSORT, - PROCESS_FILTER, - PROCESS_SNIPPETFETCH - }; - private HashMap targetTime; - private HashMap targetCount; - private HashMap yieldTime; - private HashMap yieldCount; + private long targetTime; + private int targetCount; + private ArrayList yield; private long timer; private plasmaSearchProcessing() { - targetTime = new HashMap(); - targetCount = new HashMap(); - yieldTime = new HashMap(); - yieldCount = new HashMap(); + targetTime = minimumTargetTime; + targetCount = 10; + yield = new ArrayList(); timer = 0; } public plasmaSearchProcessing(long time, int count) { - this( - 3 * time / 12, 10 * count, - 1 * time / 12, 10 * count, - 1 * time / 12, 10 * count, - 2 * time / 12, 5 * count, - 3 * time / 12, count, - 1 * time / 12, count, - 1 * time / 12, 1 - ); - } - - public plasmaSearchProcessing( - long time_collection, int count_collection, - long time_join, int count_join, - long time_presort, int count_presort, - long time_urlfetch, int count_urlfetch, - long time_postsort, int count_postsort, - long time_filter, int count_filter, - long time_snippetfetch, int count_snippetfetch) { this(); - - targetTime.put(new Character(PROCESS_COLLECTION), new Long(time_collection)); - targetTime.put(new Character(PROCESS_JOIN), new Long(time_join)); - targetTime.put(new Character(PROCESS_PRESORT), new Long(time_presort)); - targetTime.put(new Character(PROCESS_URLFETCH), new Long(time_urlfetch)); - targetTime.put(new Character(PROCESS_POSTSORT), new Long(time_postsort)); - targetTime.put(new Character(PROCESS_FILTER), new Long(time_filter)); - targetTime.put(new Character(PROCESS_SNIPPETFETCH), new Long(time_snippetfetch)); - targetCount.put(new Character(PROCESS_COLLECTION), new Integer(count_collection)); - targetCount.put(new Character(PROCESS_JOIN), new Integer(count_join)); - targetCount.put(new Character(PROCESS_PRESORT), new Integer(count_presort)); - targetCount.put(new Character(PROCESS_URLFETCH), new Integer(count_urlfetch)); - targetCount.put(new Character(PROCESS_POSTSORT), new Integer(count_postsort)); - targetCount.put(new Character(PROCESS_FILTER), new Integer(count_filter)); - targetCount.put(new Character(PROCESS_SNIPPETFETCH), new Integer(count_snippetfetch)); - - } - - public Object clone() { - plasmaSearchProcessing p = new plasmaSearchProcessing(); - p.targetTime = (HashMap) this.targetTime.clone(); - p.targetCount = (HashMap) this.targetCount.clone(); - p.yieldTime = (HashMap) this.yieldTime.clone(); - p.yieldCount = (HashMap) this.yieldCount.clone(); - return p; - } - - public plasmaSearchProcessing(String s) { - targetTime = new HashMap(); - targetCount = new HashMap(); - yieldTime = new HashMap(); - yieldCount = new HashMap(); - - intoMap(s, targetTime, targetCount); - } - - public long duetime() { - // returns the old duetime value as sum of all waiting times - long d = 0; - for (int i = 0; i < sequence.length; i++) { - d += ((Long) targetTime.get(new Character(sequence[i]))).longValue(); + this.targetTime = time; + this.targetCount = count; + } + + public static class Entry { + public String process; + public int count; + public long time; + public Entry(String process, int count, long time) { + this.process = process; + this.count = count; + this.time = time; } - return d; - } - - public void putYield(String s) { - intoMap(s, yieldTime, yieldCount); } - public String yieldToString() { - return toString(yieldTime, yieldCount); - } - - public String targetToString() { - return toString(targetTime, targetCount); - } - - public long getTargetTime(char type) { - // sum up all time that was demanded and subtract all that had been wasted - long sum = 0; - Long t; - Character element; - for (int i = 0; i < sequence.length; i++) { - element = new Character(sequence[i]); - t = (Long) targetTime.get(element); - if (t != null) sum += t.longValue(); - if (type == sequence[i]) return (sum < 0) ? minimumTargetTime : sum; - t = (Long) yieldTime.get(element); - if (t != null) sum -= t.longValue(); - } - return minimumTargetTime; - } - - public int getTargetCount(char type) { - Integer i = (Integer) targetCount.get(new Character(type)); - if (i == null) return -1; else return i.intValue(); + public int getTargetCount() { + return this.targetCount; } - public long getYieldTime(char type) { - Long l = (Long) yieldTime.get(new Character(type)); - if (l == null) return -1; else return l.longValue(); + public long getTargetTime() { + return this.targetTime; } - - public int getYieldCount(char type) { - Integer i = (Integer) yieldCount.get(new Character(type)); - if (i == null) return -1; else return i.intValue(); - } - + public void startTimer() { this.timer = System.currentTimeMillis(); } - - public void setYieldTime(char type) { - // sets a time that is computed using the timer + + public void yield(String s, int count) { long t = System.currentTimeMillis() - this.timer; - yieldTime.put(new Character(type), new Long(t)); - } - - public void setYieldCount(char type, int count) { - yieldCount.put(new Character(type), new Integer(count)); + Entry e = new Entry(s, count, t); + yield.add(e); } - public String reportToString() { - return "target=" + toString(targetTime, targetCount) + "; yield=" + toString(yieldTime, yieldCount); + public Iterator events() { + // iteratese Entry-type Objects + return yield.iterator(); } - - public static String toString(HashMap time, HashMap count) { - // put this into a format in such a way that it can be send in a http header or post argument - // that means that no '=' or spaces are allowed - StringBuffer sb = new StringBuffer(sequence.length * 10); - Character element; - Integer xi; - Long xl; - for (int i = 0; i < sequence.length; i++) { - element = new Character(sequence[i]); - sb.append("t"); - sb.append(element); - xl = (Long) time.get(element); - sb.append((xl == null) ? "0" : xl.toString()); - sb.append("|"); - sb.append("c"); - sb.append(element); - xi = (Integer) count.get(element); - sb.append((xi == null) ? "0" : xi.toString()); - sb.append("|"); - } - return sb.toString(); - } - - public static void intoMap(String s, HashMap time, HashMap count) { - // this is the reverse method to toString - int p = 0; - char ct; - String elt; - String v; - int p1; - while ((p < s.length()) && ((p1 = s.indexOf('|', p)) > 0)) { - ct = s.charAt(p); - elt = s.substring(p + 1, p + 2); - v = s.substring(p + 2, p1); - if (ct == 't') { - time.put(elt, new Long(Long.parseLong(v))); - } else { - count.put(elt, new Integer(Integer.parseInt(v))); - } - } + + public int size() { + // returns number of events / Entry-Objects in yield array + return yield.size(); } - // the processes - // collection public Map[] localSearchContainers( plasmaSearchQuery query, @@ -280,23 +141,18 @@ public class plasmaSearchProcessing implements Cloneable { // retrieve entities that belong to the hashes startTimer(); - long start = System.currentTimeMillis(); Map inclusionContainers = (query.queryHashes.size() == 0) ? new HashMap() : wordIndex.getContainers( query.queryHashes, urlselection, true, - true, - getTargetTime(plasmaSearchProcessing.PROCESS_COLLECTION) * query.queryHashes.size() / (query.queryHashes.size() + query.excludeHashes.size())); + true); if ((inclusionContainers.size() != 0) && (inclusionContainers.size() < query.queryHashes.size())) inclusionContainers = new HashMap(); // prevent that only a subset is returned - long remaintime = getTargetTime(plasmaSearchProcessing.PROCESS_COLLECTION) - System.currentTimeMillis() + start; - Map exclusionContainers = ((inclusionContainers == null) || (inclusionContainers.size() == 0) || (remaintime <= 0)) ? new HashMap() : wordIndex.getContainers( + Map exclusionContainers = ((inclusionContainers == null) || (inclusionContainers.size() == 0)) ? new HashMap() : wordIndex.getContainers( query.excludeHashes, urlselection, true, - true, - remaintime); - setYieldTime(plasmaSearchProcessing.PROCESS_COLLECTION); - setYieldCount(plasmaSearchProcessing.PROCESS_COLLECTION, inclusionContainers.size()); + true); + yield(plasmaSearchProcessing.COLLECTION, inclusionContainers.size()); return new Map[]{inclusionContainers, exclusionContainers}; } @@ -305,7 +161,7 @@ public class plasmaSearchProcessing implements Cloneable { public indexContainer localSearchJoinExclude( Collection includeContainers, Collection excludeContainers, - long time, int maxDistance) { + int maxDistance) { // join a search result and return the joincount (number of pages after join) // since this is a conjunction we return an empty entity if any word is not known @@ -313,15 +169,12 @@ public class plasmaSearchProcessing implements Cloneable { // join the result startTimer(); - long start = System.currentTimeMillis(); - indexContainer rcLocal = indexContainer.joinContainers(includeContainers, time, maxDistance); - long remaining = getTargetTime(plasmaSearchProcessing.PROCESS_JOIN) - System.currentTimeMillis() + start; - if ((rcLocal != null) && (remaining > 0)) { - indexContainer.excludeContainers(rcLocal, excludeContainers, remaining); + indexContainer rcLocal = indexContainer.joinContainers(includeContainers, maxDistance); + if (rcLocal != null) { + indexContainer.excludeContainers(rcLocal, excludeContainers); } if (rcLocal == null) rcLocal = plasmaWordIndex.emptyContainer(null, 0); - setYieldTime(plasmaSearchProcessing.PROCESS_JOIN); - setYieldCount(plasmaSearchProcessing.PROCESS_JOIN, rcLocal.size()); + yield(plasmaSearchProcessing.JOIN, rcLocal.size()); return rcLocal; } diff --git a/source/de/anomic/plasma/plasmaSearchQuery.java b/source/de/anomic/plasma/plasmaSearchQuery.java index db7543773..5a298cbff 100644 --- a/source/de/anomic/plasma/plasmaSearchQuery.java +++ b/source/de/anomic/plasma/plasmaSearchQuery.java @@ -63,6 +63,7 @@ public final class plasmaSearchQuery { public static final int SEARCHDOM_GLOBALDHT = 3;
public static final int SEARCHDOM_GLOBALALL = 4;
+ public static final int CONTENTDOM_ALL = -1;
public static final int CONTENTDOM_TEXT = 0;
public static final int CONTENTDOM_IMAGE = 1;
public static final int CONTENTDOM_AUDIO = 2;
@@ -74,7 +75,7 @@ public final class plasmaSearchQuery { public String queryString;
public TreeSet queryHashes, excludeHashes;
- public int wantedResults;
+ private int linesPerPage, offset;
public String prefer;
public int contentdom;
public long maximumTime;
@@ -84,9 +85,11 @@ public final class plasmaSearchQuery { public int domMaxTargets;
public int maxDistance;
public kelondroBitfield constraint;
+ public boolean onlineSnippetFetch;
public plasmaSearchQuery(String queryString, TreeSet queryHashes, TreeSet excludeHashes, int maxDistance, String prefer, int contentdom,
- int wantedResults, long maximumTime, String urlMask,
+ boolean onlineSnippetFetch,
+ int lines, int offset, long maximumTime, String urlMask,
int domType, String domGroupName, int domMaxTargets,
kelondroBitfield constraint) {
this.queryString = queryString;
@@ -95,31 +98,29 @@ public final class plasmaSearchQuery { this.maxDistance = maxDistance;
this.prefer = prefer;
this.contentdom = contentdom;
- this.wantedResults = wantedResults;
+ this.linesPerPage = lines;
+ this.offset = offset;
this.maximumTime = maximumTime;
this.urlMask = urlMask;
this.domType = domType;
this.domGroupName = domGroupName;
this.domMaxTargets = domMaxTargets;
this.constraint = constraint;
+ this.onlineSnippetFetch = onlineSnippetFetch;
}
- public plasmaSearchQuery(TreeSet queryHashes, TreeSet excludeHashes, int maxDistance, String prefer, int contentdom,
- int wantedResults, long maximumTime, String urlMask,
- kelondroBitfield constraint) {
- this.queryString = null;
- this.excludeHashes = excludeHashes;
- this.maxDistance = maxDistance;
- this.prefer = prefer;
- this.contentdom = contentdom;
- this.queryHashes = queryHashes;
- this.wantedResults = wantedResults;
- this.maximumTime = maximumTime;
- this.urlMask = urlMask;
- this.domType = -1;
- this.domGroupName = null;
- this.domMaxTargets = -1;
- this.constraint = constraint;
+ public int neededResults() {
+ // the number of result lines that must be computed
+ return this.offset + this.linesPerPage;
+ }
+
+ public int displayResults() {
+ // the number if result lines that are displayed at once (size of result page)
+ return this.linesPerPage;
+ }
+
+ public void setOffset(int newOffset) {
+ this.offset = newOffset;
}
public static int contentdomParser(String dom) {
@@ -131,6 +132,19 @@ public final class plasmaSearchQuery { return CONTENTDOM_TEXT;
}
+ public String contentdom() {
+ if (this.contentdom == CONTENTDOM_TEXT) return "text";
+ else if (this.contentdom == CONTENTDOM_IMAGE) return "image";
+ else if (this.contentdom == CONTENTDOM_AUDIO) return "audio";
+ else if (this.contentdom == CONTENTDOM_VIDEO) return "video";
+ else if (this.contentdom == CONTENTDOM_APP) return "app";
+ return "text";
+ }
+
+ public String searchdom() {
+ return (this.domType == SEARCHDOM_LOCAL) ? "local" : "global";
+ }
+
public static TreeSet hashes2Set(String query) {
if (query == null) return new TreeSet(kelondroBase64Order.enhancedCoder);
final TreeSet keyhashes = new TreeSet(kelondroBase64Order.enhancedCoder);
@@ -225,14 +239,14 @@ public final class plasmaSearchQuery { return hashSet2hashString(this.queryHashes) + "-" + hashSet2hashString(this.excludeHashes) + ":" + this.contentdom;
}
- public HashMap resultProfile(int searchcount, long searchtime) {
+ public HashMap resultProfile(long searchtime) {
// generate statistics about search: query, time, etc
HashMap r = new HashMap();
r.put("queryhashes", queryHashes);
r.put("querystring", queryString);
- r.put("querycount", new Integer(wantedResults));
+ r.put("querycount", new Integer(linesPerPage));
r.put("querytime", new Long(maximumTime));
- r.put("resultcount", new Integer(searchcount));
+ //r.put("resultcount", new Integer(searchcount));
r.put("resulttime", new Long(searchtime));
return r;
}
diff --git a/source/de/anomic/plasma/plasmaSearchRankingProfile.java b/source/de/anomic/plasma/plasmaSearchRankingProfile.java index 43d5f20a6..57684572c 100644 --- a/source/de/anomic/plasma/plasmaSearchRankingProfile.java +++ b/source/de/anomic/plasma/plasmaSearchRankingProfile.java @@ -282,48 +282,11 @@ public class plasmaSearchRankingProfile { ranking += (flags.get(plasmaCondenser.flag_cat_hasapp)) ? 256 << coeff_cathasapp : 0; ranking += (plasmaURL.probablyRootURL(t.urlHash())) ? 16 << coeff_urllength : 0; - ranking += (plasmaURL.probablyWordURL(t.urlHash(), searchedWords) != null) ? 256 << coeff_appurl : 0; + if (searchedWords != null) ranking += (plasmaURL.probablyWordURL(t.urlHash(), searchedWords) != null) ? 256 << coeff_appurl : 0; return ranking; } - /* - public long preRanking(indexRWIEntry normalizedEntry, TreeSet searchedWords) { - // the normalizedEntry must be a normalized indexEntry - long ranking = 0; - ranking += (256 - plasmaURL.domLengthNormalized(normalizedEntry.urlHash())) << coeff_domlength; - ranking += plasmaSearchPreOrder.ybr_p(normalizedEntry.urlHash()) << coeff_ybr; - ranking += normalizedEntry.virtualAge() << coeff_date; - ranking += normalizedEntry.wordsintitle() << coeff_wordsintitle; - ranking += normalizedEntry.wordsintext() << coeff_wordsintext; - ranking += normalizedEntry.phrasesintext() << coeff_phrasesintext; - ranking += normalizedEntry.llocal() << coeff_llocal; - ranking += normalizedEntry.lother() << coeff_lother; - ranking += (normalizedEntry.urllength() == 0) ? 0 : (256 - normalizedEntry.urllength()) << coeff_urllength; - ranking += (normalizedEntry.urlcomps() == 0) ? 0 : (256 - normalizedEntry.urlcomps()) << coeff_urlcomps; - ranking += (normalizedEntry.hitcount() == 0) ? 0 : normalizedEntry.hitcount() << coeff_hitcount; - ranking += (normalizedEntry.posintext() == 0) ? 0 : (256 - normalizedEntry.posintext()) << coeff_posintext; - ranking += (normalizedEntry.posofphrase() == 0) ? 0 : (256 - normalizedEntry.posofphrase()) << coeff_posofphrase; - ranking += (normalizedEntry.worddistance() == 0) ? 0 : (256 - normalizedEntry.worddistance()) << coeff_worddistance; - - kelondroBitfield flags = normalizedEntry.flags(); - ranking += (flags.get(indexRWIEntry.flag_app_url)) ? 256 << coeff_appurl : 0; - ranking += (flags.get(indexRWIEntry.flag_app_descr)) ? 256 << coeff_appdescr : 0; - ranking += (flags.get(indexRWIEntry.flag_app_author)) ? 256 << coeff_appauthor : 0; - ranking += (flags.get(indexRWIEntry.flag_app_tags)) ? 256 << coeff_apptags : 0; - ranking += (flags.get(indexRWIEntry.flag_app_reference)) ? 256 << coeff_appref : 0; - ranking += (flags.get(indexRWIEntry.flag_app_emphasized)) ? 256 << coeff_appemph : 0; - ranking += (flags.get(plasmaCondenser.flag_cat_indexof)) ? 256 << coeff_catindexof : 0; - ranking += (flags.get(plasmaCondenser.flag_cat_hasimage)) ? 256 << coeff_cathasimage : 0; - ranking += (flags.get(plasmaCondenser.flag_cat_hasaudio)) ? 256 << coeff_cathasaudio : 0; - ranking += (flags.get(plasmaCondenser.flag_cat_hasvideo)) ? 256 << coeff_cathasvideo : 0; - ranking += (flags.get(plasmaCondenser.flag_cat_hasapp)) ? 256 << coeff_cathasapp : 0; - - ranking += (plasmaURL.probablyRootURL(normalizedEntry.urlHash())) ? 16 << coeff_urllength : 0; - ranking += (plasmaURL.probablyWordURL(normalizedEntry.urlHash(), searchedWords) != null) ? 256 << coeff_appurl : 0; - - return ranking; - } - */ + public long postRanking( plasmaSearchQuery query, Set topwords, diff --git a/source/de/anomic/plasma/plasmaSnippetCache.java b/source/de/anomic/plasma/plasmaSnippetCache.java index b4b02ac47..bda27351a 100644 --- a/source/de/anomic/plasma/plasmaSnippetCache.java +++ b/source/de/anomic/plasma/plasmaSnippetCache.java @@ -230,8 +230,9 @@ public class plasmaSnippetCache { } public static class MediaSnippet { - public String type, href, name, attr; - public MediaSnippet(String type, String href, String name, String attr) { + public int type; + public String href, name, attr; + public MediaSnippet(int type, String href, String name, String attr) { this.type = type; this.href = href; this.name = name; @@ -614,31 +615,30 @@ public class plasmaSnippetCache { } } - public static ArrayList retrieveMediaSnippets(URL url, Set queryhashes, String mediatype, boolean fetchOnline, int timeout) { + public static ArrayList retrieveMediaSnippets(URL url, Set queryhashes, int mediatype, boolean fetchOnline, int timeout) { if (queryhashes.size() == 0) { serverLog.logFine("snippet fetch", "no query hashes given for url " + url); return new ArrayList(); } - if (mediatype == null) mediatype = ""; plasmaParserDocument document = retrieveDocument(url, fetchOnline, timeout, false); ArrayList a = new ArrayList(); if (document != null) { - if ((mediatype.length() == 0) || (mediatype.equals("audio"))) a.addAll(computeMediaSnippets(document, queryhashes, "audio")); - if ((mediatype.length() == 0) || (mediatype.equals("video"))) a.addAll(computeMediaSnippets(document, queryhashes, "video")); - if ((mediatype.length() == 0) || (mediatype.equals("app" ))) a.addAll(computeMediaSnippets(document, queryhashes, "app")); - if ((mediatype.length() == 0) || (mediatype.equals("image"))) a.addAll(computeImageSnippets(document, queryhashes)); + if ((mediatype == plasmaSearchQuery.CONTENTDOM_ALL) || (mediatype == plasmaSearchQuery.CONTENTDOM_AUDIO)) a.addAll(computeMediaSnippets(document, queryhashes, plasmaSearchQuery.CONTENTDOM_AUDIO)); + if ((mediatype == plasmaSearchQuery.CONTENTDOM_ALL) || (mediatype == plasmaSearchQuery.CONTENTDOM_VIDEO)) a.addAll(computeMediaSnippets(document, queryhashes, plasmaSearchQuery.CONTENTDOM_VIDEO)); + if ((mediatype == plasmaSearchQuery.CONTENTDOM_ALL) || (mediatype == plasmaSearchQuery.CONTENTDOM_APP)) a.addAll(computeMediaSnippets(document, queryhashes, plasmaSearchQuery.CONTENTDOM_APP)); + if ((mediatype == plasmaSearchQuery.CONTENTDOM_ALL) || (mediatype == plasmaSearchQuery.CONTENTDOM_IMAGE)) a.addAll(computeImageSnippets(document, queryhashes)); } return a; } - public static ArrayList computeMediaSnippets(plasmaParserDocument document, Set queryhashes, String mediatype) { + public static ArrayList computeMediaSnippets(plasmaParserDocument document, Set queryhashes, int mediatype) { if (document == null) return new ArrayList(); Map media = null; - if (mediatype.equals("audio")) media = document.getAudiolinks(); - else if (mediatype.equals("video")) media = document.getVideolinks(); - else if (mediatype.equals("app")) media = document.getApplinks(); + if (mediatype == plasmaSearchQuery.CONTENTDOM_AUDIO) media = document.getAudiolinks(); + else if (mediatype == plasmaSearchQuery.CONTENTDOM_VIDEO) media = document.getVideolinks(); + else if (mediatype == plasmaSearchQuery.CONTENTDOM_APP) media = document.getApplinks(); if (media == null) return null; Iterator i = media.entrySet().iterator(); @@ -650,7 +650,6 @@ public class plasmaSnippetCache { entry = (Map.Entry) i.next(); url = (String) entry.getKey(); desc = (String) entry.getValue(); - //result.add(new MediaSnippet(mediatype, url, (desc.length() == 0) ? url : desc, null)); s = removeAppearanceHashes(url, queryhashes); if (s.size() == 0) { result.add(new MediaSnippet(mediatype, url, desc, null)); @@ -678,15 +677,14 @@ public class plasmaSnippetCache { ientry = (htmlFilterImageEntry) i.next(); url = (String) ientry.url().toNormalform(true, true); desc = (String) ientry.alt(); - //result.add(new MediaSnippet("image", url, (desc.length() == 0) ? url : desc, ientry.width() + " x " + ientry.height())); s = removeAppearanceHashes(url, queryhashes); if (s.size() == 0) { - result.add(new MediaSnippet("image", url, desc, ientry.width() + " x " + ientry.height())); + result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height())); continue; } s = removeAppearanceHashes(desc, s); if (s.size() == 0) { - result.add(new MediaSnippet("image", url, desc, ientry.width() + " x " + ientry.height())); + result.add(new MediaSnippet(plasmaSearchQuery.CONTENTDOM_IMAGE, url, desc, ientry.width() + " x " + ientry.height())); continue; } } diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index bb144f3a6..eba6fce1d 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -147,7 +147,7 @@ public final class plasmaWordIndex implements indexRI { }
public long getUpdateTime(String wordHash) {
- indexContainer entries = getContainer(wordHash, null, -1);
+ indexContainer entries = getContainer(wordHash, null);
if (entries == null) return 0;
return entries.updated();
}
@@ -203,7 +203,7 @@ public final class plasmaWordIndex implements indexRI { indexContainer c;
while (collectMax) {
wordHash = ram.maxScoreWordHash();
- c = ram.getContainer(wordHash, null, -1);
+ c = ram.getContainer(wordHash, null);
if ((c != null) && (c.size() > wCacheMaxChunk)) {
containerList.add(ram.deleteContainer(wordHash));
if (serverMemory.available() < collections.minMem()) break; // protect memory during flush
@@ -310,7 +310,7 @@ public final class plasmaWordIndex implements indexRI { return false;
}
- public indexContainer getContainer(String wordHash, Set urlselection, long maxTime) {
+ public indexContainer getContainer(String wordHash, Set urlselection) {
if ((wordHash == null) || (wordHash.length() != yacySeedDB.commonHashLength)) {
// wrong input
return null;
@@ -319,28 +319,28 @@ public final class plasmaWordIndex implements indexRI { // get from cache
indexContainer container;
synchronized (dhtOutCache) {
- container = dhtOutCache.getContainer(wordHash, urlselection, -1);
+ container = dhtOutCache.getContainer(wordHash, urlselection);
}
synchronized (dhtInCache) {
if (container == null) {
- container = dhtInCache.getContainer(wordHash, urlselection, -1);
+ container = dhtInCache.getContainer(wordHash, urlselection);
} else {
- container.addAllUnique(dhtInCache.getContainer(wordHash, urlselection, -1));
+ container.addAllUnique(dhtInCache.getContainer(wordHash, urlselection));
}
}
// get from collection index
synchronized (this) {
if (container == null) {
- container = collections.getContainer(wordHash, urlselection, (maxTime < 0) ? -1 : maxTime);
+ container = collections.getContainer(wordHash, urlselection);
} else {
- container.addAllUnique(collections.getContainer(wordHash, urlselection, (maxTime < 0) ? -1 : maxTime));
+ container.addAllUnique(collections.getContainer(wordHash, urlselection));
}
}
return container;
}
- public Map getContainers(Set wordHashes, Set urlselection, boolean deleteIfEmpty, boolean interruptIfEmpty, long maxTime) {
+ public Map getContainers(Set wordHashes, Set urlselection, boolean deleteIfEmpty, boolean interruptIfEmpty) {
// return map of wordhash:indexContainer
// retrieve entities that belong to the hashes
@@ -348,19 +348,13 @@ public final class plasmaWordIndex implements indexRI { String singleHash;
indexContainer singleContainer;
Iterator i = wordHashes.iterator();
- long start = System.currentTimeMillis();
- long remaining;
while (i.hasNext()) {
- // check time
- remaining = maxTime - (System.currentTimeMillis() - start);
- //if ((maxTime > 0) && (remaining <= 0)) break;
- if ((maxTime >= 0) && (remaining <= 0)) remaining = 100;
// get next word hash:
singleHash = (String) i.next();
// retrieve index
- singleContainer = getContainer(singleHash, urlselection, (maxTime < 0) ? -1 : remaining / (wordHashes.size() - containers.size()));
+ singleContainer = getContainer(singleHash, urlselection);
// check result
if (((singleContainer == null) || (singleContainer.size() == 0)) && (interruptIfEmpty)) return new HashMap();
diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 48795049c..ad1729aae 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -61,8 +61,8 @@ import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBitfield;
import de.anomic.net.URL;
import de.anomic.plasma.plasmaCondenser;
+import de.anomic.plasma.plasmaSearchContainer;
import de.anomic.plasma.plasmaSearchRankingProfile;
-import de.anomic.plasma.plasmaSearchProcessing;
import de.anomic.plasma.plasmaSnippetCache;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaWordIndex;
@@ -104,7 +104,7 @@ public final class yacyClient { salt = yacyNetwork.enrichRequestPost(obj, plasmaSwitchboard.getSwitchboard(), null);
obj.putASIS("count", "20");
obj.putASIS("seed", yacyCore.seedDB.mySeed.genSeedStr(salt));
-
+ yacyCore.log.logFine("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' contacting peer at " + address);
// send request
result = nxTools.table(
httpc.wput(new URL("http://" + address + "/yacy/hello.html"),
@@ -339,15 +339,15 @@ public final class yacyClient { String urlhashes,
String prefer,
String filter,
+ int count,
int maxDistance,
boolean global,
int partitions,
yacySeed target,
plasmaWordIndex wordIndex,
- indexContainer containerCache,
+ plasmaSearchContainer containerCache,
Map abstractCache,
plasmaURLPattern blacklist,
- plasmaSearchProcessing timingProfile,
plasmaSearchRankingProfile rankingProfile,
kelondroBitfield constraint
) {
@@ -370,9 +370,8 @@ public final class yacyClient { // prepare request
final serverObjects post = new serverObjects();
final String salt = yacyNetwork.enrichRequestPost(post, plasmaSwitchboard.getSwitchboard(), target.hash);
- long duetime = timingProfile.duetime();
post.putASIS("myseed", yacyCore.seedDB.mySeed.genSeedStr(salt));
- post.put("count", timingProfile.getTargetCount(plasmaSearchProcessing.PROCESS_POSTSORT));
+ post.put("count", Math.max(10, count));
post.putASIS("resource", ((global) ? "global" : "local"));
post.put("partitions", partitions);
post.putASIS("query", wordhashes);
@@ -381,8 +380,6 @@ public final class yacyClient { post.putASIS("prefer", prefer);
post.putASIS("filter", filter);
post.putASIS("ttl", "0");
- post.put("duetime", Long.toString(duetime));
- post.putASIS("timing", crypt.simpleEncode(timingProfile.targetToString())); // new duetimes splitted by specific search tasks
post.put("maxdist", maxDistance);
post.putASIS("profile", crypt.simpleEncode(rankingProfile.toExternalString()));
post.putASIS("constraint", constraint.exportB64());
@@ -424,8 +421,6 @@ public final class yacyClient { // compute all computation times
final long totalrequesttime = System.currentTimeMillis() - timestamp;
- String returnProfile = (String) result.get("profile");
- if (returnProfile != null) timingProfile.putYield(returnProfile);
// OUTPUT:
// version : application version of responder
@@ -449,7 +444,7 @@ public final class yacyClient { final int words = wordhashes.length() / yacySeedDB.commonHashLength;
indexContainer[] container = new indexContainer[words];
for (int i = 0; i < words; i++) {
- container[i] = plasmaWordIndex.emptyContainer(wordhashes.substring(i * yacySeedDB.commonHashLength, (i + 1) * yacySeedDB.commonHashLength), timingProfile.getTargetCount(plasmaSearchProcessing.PROCESS_POSTSORT));
+ container[i] = plasmaWordIndex.emptyContainer(wordhashes.substring(i * yacySeedDB.commonHashLength, (i + 1) * yacySeedDB.commonHashLength), count);
}
// insert results to containers
@@ -502,17 +497,28 @@ public final class yacyClient { // System.out.println("--- RECEIVED SNIPPET '" + link.snippet() + "'");
plasmaSnippetCache.storeToCache(wordhashes, urlEntry.hash(), urlEntry.snippet());
}
+
// add the url entry to the word indexes
for (int m = 0; m < words; m++) {
container[m].add(entry, System.currentTimeMillis());
}
+
// store url hash for statistics
urls[n] = urlEntry.hash();
}
+ // store remote result to local result container
+ synchronized (containerCache) {
+ // insert one container into the search result buffer
+ containerCache.insert(container[0], false, false); // one is enough
+
+ // integrate remote topwords
+ String references = (String) result.get("references");
+ if (references != null) containerCache.addReferences(references.split(","));
+ }
+
// insert the containers to the index
- containerCache.addAllUnique(container[0]); // one is enough
- for (int m = 0; m < words; m++) {
+ for (int m = 0; m < words; m++) {
wordIndex.addEntries(container[m], System.currentTimeMillis(), true);
}
@@ -557,8 +563,8 @@ public final class yacyClient { + ", DHTdist="
+ ((wordhashes.length() < 12) ? "void" : Double
.toString(yacyDHTAction.dhtDistance(target.hash,
- wordhashes.substring(0, 12)))) + ", duetime="
- + duetime + ", searchtime=" + searchtime + ", netdelay="
+ wordhashes.substring(0, 12))))
+ + ", searchtime=" + searchtime + ", netdelay="
+ (totalrequesttime - searchtime) + ", references="
+ result.get("references"));
return urls;
diff --git a/source/de/anomic/yacy/yacyDHTAction.java b/source/de/anomic/yacy/yacyDHTAction.java index 4003c4631..71b0e72ed 100644 --- a/source/de/anomic/yacy/yacyDHTAction.java +++ b/source/de/anomic/yacy/yacyDHTAction.java @@ -295,9 +295,11 @@ public class yacyDHTAction implements yacyPeerAction { assert yacyCore.seedDB != null;
assert yacyCore.seedDB.mySeed != null;
assert yacyCore.seedDB.mySeed.hash != null;
+ /*
assert
!(kelondroBase64Order.enhancedCoder.cardinal(firstKey.getBytes()) < kelondroBase64Order.enhancedCoder.cardinal(yacyCore.seedDB.mySeed.hash.getBytes()) &&
kelondroBase64Order.enhancedCoder.cardinal(lastKey.getBytes()) > kelondroBase64Order.enhancedCoder.cardinal(yacyCore.seedDB.mySeed.hash.getBytes()));
+ */
ArrayList seeds = new ArrayList();
yacySeed seed;
//double ownDistance = Math.min(yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, firstKey), yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, lastKey));
diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index 7f2021f13..c7f2cf36c 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -51,12 +51,11 @@ import java.util.Map; import java.util.Set;
import java.util.TreeMap;
-import de.anomic.index.indexContainer;
import de.anomic.kelondro.kelondroBitfield;
import de.anomic.kelondro.kelondroMScoreCluster;
+import de.anomic.plasma.plasmaSearchContainer;
import de.anomic.plasma.plasmaSearchQuery;
import de.anomic.plasma.plasmaSearchRankingProfile;
-import de.anomic.plasma.plasmaSearchProcessing;
import de.anomic.plasma.plasmaWordIndex;
import de.anomic.plasma.urlPattern.plasmaURLPattern;
import de.anomic.server.logging.serverLog;
@@ -67,22 +66,21 @@ public class yacySearch extends Thread { final private boolean global;
final private int partitions;
final private plasmaWordIndex wordIndex;
- final private indexContainer containerCache;
+ final private plasmaSearchContainer containerCache;
final private Map abstractCache;
final private plasmaURLPattern blacklist;
final private yacySeed targetPeer;
private String[] urls;
- private int maxDistance;
- final private plasmaSearchProcessing timingProfile;
+ private int count, maxDistance;
final private plasmaSearchRankingProfile rankingProfile;
final private String prefer, filter;
final private kelondroBitfield constraint;
- public yacySearch(String wordhashes, String excludehashes, String urlhashes, String prefer, String filter, int maxDistance,
+ public yacySearch(String wordhashes, String excludehashes, String urlhashes, String prefer, String filter, int count, int maxDistance,
boolean global, int partitions, yacySeed targetPeer, plasmaWordIndex wordIndex,
- indexContainer containerCache, Map abstractCache,
+ plasmaSearchContainer containerCache, Map abstractCache,
plasmaURLPattern blacklist,
- plasmaSearchProcessing timingProfile, plasmaSearchRankingProfile rankingProfile,
+ plasmaSearchRankingProfile rankingProfile,
kelondroBitfield constraint) {
super("yacySearch_" + targetPeer.getName());
//System.out.println("DEBUG - yacySearch thread " + this.getName() + " initialized " + ((urlhashes.length() == 0) ? "(primary)" : "(secondary)"));
@@ -99,17 +97,17 @@ public class yacySearch extends Thread { this.blacklist = blacklist;
this.targetPeer = targetPeer;
this.urls = null;
+ this.count = count;
this.maxDistance = maxDistance;
- this.timingProfile = (plasmaSearchProcessing) timingProfile.clone();
this.rankingProfile = rankingProfile;
this.constraint = constraint;
}
public void run() {
this.urls = yacyClient.search(
- wordhashes, excludehashes, urlhashes, prefer, filter, maxDistance, global, partitions,
+ wordhashes, excludehashes, urlhashes, prefer, filter, count, maxDistance, global, partitions,
targetPeer, wordIndex, containerCache, abstractCache,
- blacklist, timingProfile, rankingProfile, constraint);
+ blacklist, rankingProfile, constraint);
if (urls != null) {
// urls is an array of url hashes. this is only used for log output
StringBuffer urllist = new StringBuffer(this.urls.length * 13);
@@ -133,8 +131,8 @@ public class yacySearch extends Thread { return this.urls.length;
}
- public plasmaSearchProcessing timingProfile() {
- return this.timingProfile;
+ public int count() {
+ return this.count;
}
public yacySeed target() {
@@ -248,11 +246,11 @@ public class yacySearch extends Thread { public static yacySearch[] primaryRemoteSearches(
String wordhashes, String excludehashes, String urlhashes,
- String prefer, String filter, int maxDist,
+ String prefer, String filter, int count, int maxDist,
plasmaWordIndex wordIndex,
- indexContainer containerCache, Map abstractCache,
+ plasmaSearchContainer containerCache, Map abstractCache,
int targets, plasmaURLPattern blacklist,
- plasmaSearchProcessing timingProfile, plasmaSearchRankingProfile rankingProfile,
+ plasmaSearchRankingProfile rankingProfile,
kelondroBitfield constraint, TreeMap clusterselection) {
// check own peer status
if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getPublicAddress() == null) { return null; }
@@ -264,8 +262,8 @@ public class yacySearch extends Thread { if (targets == 0) return new yacySearch[0];
yacySearch[] searchThreads = new yacySearch[targets];
for (int i = 0; i < targets; i++) {
- searchThreads[i] = new yacySearch(wordhashes, excludehashes, urlhashes, prefer, filter, maxDist, true, targets, targetPeers[i],
- wordIndex, containerCache, abstractCache, blacklist, timingProfile, rankingProfile, constraint);
+ searchThreads[i] = new yacySearch(wordhashes, excludehashes, urlhashes, prefer, filter, count, maxDist, true, targets, targetPeers[i],
+ wordIndex, containerCache, abstractCache, blacklist, rankingProfile, constraint);
searchThreads[i].start();
//try {Thread.sleep(20);} catch (InterruptedException e) {}
}
@@ -274,9 +272,9 @@ public class yacySearch extends Thread { public static yacySearch secondaryRemoteSearch(String wordhashes, String excludehashes, String urlhashes,
plasmaWordIndex wordIndex,
- indexContainer containerCache,
+ plasmaSearchContainer containerCache,
String targethash, plasmaURLPattern blacklist,
- plasmaSearchProcessing timingProfile, plasmaSearchRankingProfile rankingProfile,
+ plasmaSearchRankingProfile rankingProfile,
kelondroBitfield constraint, TreeMap clusterselection) {
// check own peer status
if (yacyCore.seedDB.mySeed == null || yacyCore.seedDB.mySeed.getPublicAddress() == null) { return null; }
@@ -285,8 +283,8 @@ public class yacySearch extends Thread { final yacySeed targetPeer = yacyCore.seedDB.getConnected(targethash);
if (targetPeer == null) return null;
if (clusterselection != null) targetPeer.setAlternativeAddress((String) clusterselection.get(targetPeer.hash));
- yacySearch searchThread = new yacySearch(wordhashes, excludehashes, urlhashes, "", "", 9999, true, 0, targetPeer,
- wordIndex, containerCache, new TreeMap(), blacklist, timingProfile, rankingProfile, constraint);
+ yacySearch searchThread = new yacySearch(wordhashes, excludehashes, urlhashes, "", "", 0, 9999, true, 0, targetPeer,
+ wordIndex, containerCache, new TreeMap(), blacklist, rankingProfile, constraint);
searchThread.start();
return searchThread;
}
@@ -99,6 +99,18 @@ update.time.download = 0 # the deploy time when the last update was done; milliseconds since epoch
update.time.deploy = 0
+# restart-option
+# a peer can be re-started periodically
+# restart.process can be either 'off' (no automatic restart) or 'time' (time- rule-based, see below)
+restart.process = off
+# the restart.cycle is the number of hours that must pass bevore a restart is done
+restart.cycle = 20
+# the restart.hour is a pattern that must match with the hour string (two-digit, 24h)
+# when the restart should be performed
+restart.hour = 03
+# the following values are set automatically
+restart.time = 0
+
# clusters within a network:
# every network can have an unlimited number of clusters. Clusters may be also completely
# sealed and have no connection to other peers. When a cluster does not use the
@@ -635,7 +647,8 @@ javastart_Xms__pro=Xms512m # not for first startup of YaCy
# UNIX: corresponds to the nice-level
# WIN: -20=realtime;-15=high;-10=above;0=normal;10=below;20=low
-javastart_priority=0
+javastart_priority=10
+javastart_priority__pro=0
# performance properties for the word index cache
# wordCacheMaxLow/High is the number of word indexes that shall be held in the
|