diff options
| -rw-r--r-- | defaults/yacy.init | 56 | ||||
| -rw-r--r-- | htroot/IndexFederated_p.html | 9 | ||||
| -rw-r--r-- | source/net/yacy/htroot/IndexFederated_p.java | 15 | ||||
| -rw-r--r-- | source/net/yacy/htroot/yacy/transferRWI.java | 86 | ||||
| -rw-r--r-- | source/net/yacy/htroot/yacy/transferURL.java | 85 | ||||
| -rw-r--r-- | source/net/yacy/peers/Protocol.java | 113 | ||||
| -rw-r--r-- | source/net/yacy/search/SwitchboardConstants.java | 4 |
7 files changed, 320 insertions, 48 deletions
diff --git a/defaults/yacy.init b/defaults/yacy.init index 2274d4d59..dd0db4496 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -577,6 +577,20 @@ allowReceiveIndex=true allowReceiveIndex.search=true
indexReceiveBlockBlacklist=true
+# Proactive Error-URL Rejection: block distribution of known broken URLs via DHT
+# Set to false to disable this feature (opt-out)
+indexReceiveBlockErrors=true
+
+# How many days to block error URLs before allowing retry
+# Permanent errors (404, 410) are always blocked
+# Temporary errors (5xx, timeouts) are blocked for this many days
+indexReceiveBlockErrors.retryAfterDays=30
+
+# Comma-separated list of HTTP status codes considered permanent errors (always block)
+# -1 = DNS/Network errors (UnknownHost, connection failures)
+# 404 = Not Found, 410 = Gone
+indexReceiveBlockErrors.permanentStatus=404,410,-1
+
# the frequency is the number of links per minute, that the peer allowes
# _every_ other peer to send to this peer
defaultWordReceiveFrequency=100
@@ -1416,24 +1430,24 @@ crawler.userAgent.clienttimeout = 10000 # experiments with timeout requests
timeoutrequests = true
-# interface decorations -decoration.audio = false -decoration.grafics.linkstructure = true -decoration.hostanalysis = false -decoration.simpleheadernavbar = navbar-default - -# ai settings -ai.production_models = [] -ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies. -ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable. -ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n -ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array: -ai.shield.allow-nonlocalhost = false -ai.shield.show-chat-link = false -ai.shield.rate.per-minute = 4 -ai.shield.rate.per-hour = 60 -ai.shield.rate.per-day = 120 -ai.shield.limit-all = false -ai.shield.all.per-minute = 16 -ai.shield.all.per-hour = 240 -ai.shield.all.per-day = 480 +# interface decorations
+decoration.audio = false
+decoration.grafics.linkstructure = true
+decoration.hostanalysis = false
+decoration.simpleheadernavbar = navbar-default
+
+# ai settings
+ai.production_models = []
+ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies.
+ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.
+ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n
+ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array:
+ai.shield.allow-nonlocalhost = false
+ai.shield.show-chat-link = false
+ai.shield.rate.per-minute = 4
+ai.shield.rate.per-hour = 60
+ai.shield.rate.per-day = 120
+ai.shield.limit-all = false
+ai.shield.all.per-minute = 16
+ai.shield.all.per-hour = 240
+ai.shield.all.per-day = 480
diff --git a/htroot/IndexFederated_p.html b/htroot/IndexFederated_p.html index 3edd5ceb7..c50c95137 100644 --- a/htroot/IndexFederated_p.html +++ b/htroot/IndexFederated_p.html @@ -117,6 +117,15 @@ <dt><input type="checkbox" name="core.service.rwi" id="core_service_rwi" #(core.service.rwi.checked)#:: checked="checked"#(/core.service.rwi.checked)# /></dt> <dd>support peer-to-peer index transmission (DHT RWI index)</dd> + + <dt class="TableCellDark">Block known error URLs in DHT <input type="checkbox" name="indexReceiveBlockErrors" #(indexReceiveBlockErrors.checked)#:: checked="checked"#(/indexReceiveBlockErrors.checked)# /></dt> + <dd>Reject URLs/RWIs with known errors from peers. Disable to opt out.</dd> + + <dt class="TableCellDark">Retry after (days)</dt> + <dd><input type="text" name="indexReceiveBlockErrors.retryAfterDays" value="#[indexReceiveBlockErrors.retryAfterDays]#" size="4" /> for temporary errors; permanent errors stay blocked.</dd> + + <dt class="TableCellDark">Permanent error statuses</dt> + <dd><input type="text" name="indexReceiveBlockErrors.permanentStatus" value="#[indexReceiveBlockErrors.permanentStatus]#" size="15" /> comma-separated (default: 404,410,-1; -1=DNS/network errors)</dd> <dt><input type="submit" name="setrwi" class="btn btn-primary" value="Set" /></dt><dd></dd> </dl> diff --git a/source/net/yacy/htroot/IndexFederated_p.java b/source/net/yacy/htroot/IndexFederated_p.java index bbaba2b3a..4acf5d996 100644 --- a/source/net/yacy/htroot/IndexFederated_p.java +++ b/source/net/yacy/htroot/IndexFederated_p.java @@ -64,6 +64,16 @@ public class IndexFederated_p { final long fileSizeMax = (OS.isWindows) ? sb.getConfigLong("filesize.max.win", Integer.MAX_VALUE) : sb.getConfigLong( "filesize.max.other", Integer.MAX_VALUE); sb.index.connectRWI(wordCacheMaxCount, fileSizeMax); } catch (final IOException e) { ConcurrentLog.logException(e); } // switch on + + // DHT error URL blocking settings + final boolean blockErrors = post.getBoolean(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS); + env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, blockErrors); + + final int retryDays = post.getInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30); + env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, retryDays); + + final String permanent = post.get(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410,-1"); + env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, permanent); } if (post != null && post.containsKey("setcitation")) { @@ -222,6 +232,11 @@ public class IndexFederated_p { prop.put("core.service.rwi.checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false) ? 1 : 0); prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0); prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0); + + // DHT error URL blocking settings + prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS + ".checked", env.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true) ? 1 : 0); + prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, env.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30)); + prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, env.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410")); prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED_DEFAULT) ? 1 : 0); diff --git a/source/net/yacy/htroot/yacy/transferRWI.java b/source/net/yacy/htroot/yacy/transferRWI.java index 0981ff70e..38c0bbbc4 100644 --- a/source/net/yacy/htroot/yacy/transferRWI.java +++ b/source/net/yacy/htroot/yacy/transferRWI.java @@ -28,11 +28,15 @@ package net.yacy.htroot.yacy;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
+import org.apache.solr.common.SolrDocument;
+
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.feed.RSSMessage;
@@ -55,6 +59,7 @@ import net.yacy.peers.Seed; import net.yacy.repository.Blacklist.BlacklistType;
import net.yacy.search.Switchboard;
import net.yacy.search.SwitchboardConstants;
+import net.yacy.search.schema.CollectionSchema;
import net.yacy.server.serverObjects;
import net.yacy.server.serverSwitch;
@@ -75,6 +80,7 @@ public final class transferRWI { prop.put("unknownURL", "");
prop.put("pause", 60000);
String result = "";
+ final StringBuilder errorURLs = new StringBuilder(4000);
if ((post == null) || (env == null)) {
result = "post or env is null!";
logWarning(contentType, result);
@@ -127,7 +133,6 @@ public final class transferRWI { int pause = 0;
result = "ok";
final StringBuilder unknownURLs = new StringBuilder(6000);
-
final double load = Memory.getSystemLoadAverage();
final float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f);
if (load > maxload) {
@@ -188,6 +193,7 @@ public final class transferRWI { final ArrayList<String> wordhashes = new ArrayList<String>();
int received = 0;
int blocked = 0;
+ int blockedErrors = 0;
int count = 0;
final Set<String> testids = new HashSet<String>();
while (it.hasNext()) {
@@ -222,6 +228,75 @@ public final class transferRWI { continue;
}
+ // reject RWI entries for URLs we already know are broken
+ final String urlHashStr = ASCII.String(urlHash);
+ final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true);
+ if (blockErrors && sb.index.fulltext().exists(urlHashStr)) {
+ try {
+ final SolrDocument errorCheck = sb.index.fulltext().getDefaultConnector().getDocumentById(urlHashStr,
+ CollectionSchema.httpstatus_i.getSolrFieldName(), CollectionSchema.failreason_s.getSolrFieldName(),
+ CollectionSchema.load_date_dt.getSolrFieldName());
+ if (errorCheck != null) {
+ final Object httpstatus = errorCheck.getFieldValue(CollectionSchema.httpstatus_i.getSolrFieldName());
+ final Object failreason = errorCheck.getFieldValue(CollectionSchema.failreason_s.getSolrFieldName());
+ if (httpstatus != null && failreason != null && failreason.toString().length() > 0) {
+ int hs = (httpstatus instanceof Integer) ? (Integer) httpstatus : Integer.parseInt(httpstatus.toString());
+ if (hs != 200) {
+ boolean shouldBlock = false;
+
+ // Get configuration
+ final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30);
+ final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410,-1");
+ final Set<Integer> permanentStatus = new HashSet<Integer>();
+ for (String s : permanentStatusStr.split(",")) {
+ try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {}
+ }
+ final long retryAfterMillis = retryAfterDays * 24L * 60L * 60L * 1000L;
+ final long now = System.currentTimeMillis();
+
+ // Permanent errors (404, 410) - always block
+ if (permanentStatus.contains(hs)) {
+ shouldBlock = true;
+ if (Network.log.isFine()) Network.log.fine("transferRWI: rejected RWI for known permanent error URL hash '" + urlHashStr + "' (httpstatus=" + hs + ") from peer " + otherPeerName);
+ } else {
+ // Temporary errors - check age
+ final Object loadDate = errorCheck.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName());
+ if (loadDate != null) {
+ try {
+ final Date errorDate = (loadDate instanceof Date) ? (Date) loadDate :
+ new Date(Long.parseLong(loadDate.toString()));
+ final long errorAge = now - errorDate.getTime();
+
+ if (errorAge < retryAfterMillis) {
+ shouldBlock = true;
+ if (Network.log.isFine()) Network.log.fine("transferRWI: rejected RWI for known temporary error URL hash '" + urlHashStr + "' (httpstatus=" + hs + ", age=" + (errorAge / (24*60*60*1000)) + " days) from peer " + otherPeerName);
+ } else {
+ if (Network.log.isFine()) Network.log.fine("transferRWI: allowing retry for URL hash '" + urlHashStr + "' (error age=" + (errorAge / (24*60*60*1000)) + " days exceeds retry threshold) from peer " + otherPeerName);
+ }
+ } catch (Exception e) {
+ // If we can't parse the date, treat as permanent error to be safe
+ shouldBlock = true;
+ }
+ } else {
+ // No load_date available - treat as permanent error
+ shouldBlock = true;
+ }
+ }
+
+ if (shouldBlock) {
+ errorURLs.append(urlHashStr).append(',');
+ blocked++;
+ blockedErrors++;
+ continue;
+ }
+ }
+ }
+ }
+ } catch (final IOException e) {
+ // ignore Solr errors during error URL check
+ }
+ }
+
// learn entry
try {
sb.index.storeRWI(ASCII.getBytes(wordHash), iEntry);
@@ -251,14 +326,16 @@ public final class transferRWI { unknownURLs.append(UTF8.String(bit.next())).append(',');
}
if (unknownURLs.length() > 0) { unknownURLs.setLength(unknownURLs.length() - 1); }
+ if (errorURLs.length() > 0) { errorURLs.setLength(errorURLs.length() - 1); }
+
if (wordhashes.isEmpty() || received == 0) {
- sb.getLog().info("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");
+ sb.getLog().info("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs, reporting " + blockedErrors + " error URLs");
} else {
final String firstHash = wordhashes.get(0);
final String lastHash = wordhashes.get(wordhashes.size() - 1);
final long avdist = (Distribution.horizontalDHTDistance(firstHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash)) + Distribution.horizontalDHTDistance(lastHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash))) / 2;
- sb.getLog().info("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + received+ " URLs from " + otherPeerName);
- EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + received + " URLs from " + otherPeerName, "", otherPeer.hash));
+ sb.getLog().info("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + " (error " + blockedErrors + "), requesting " + unknownURL.size() + "/" + received+ " URLs, reporting " + blockedErrors + " error URLs from " + otherPeerName);
+ EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + " (error " + blockedErrors + "), requesting " + unknownURL.size() + "/" + received + " URLs, reporting " + blockedErrors + " error URLs from " + otherPeerName, "", otherPeer.hash));
}
result = "ok";
@@ -266,6 +343,7 @@ public final class transferRWI { }
prop.put("unknownURL", unknownURLs.toString());
+ prop.put("errorURL", errorURLs.toString());
prop.put("result", result);
prop.put("pause", pause);
diff --git a/source/net/yacy/htroot/yacy/transferURL.java b/source/net/yacy/htroot/yacy/transferURL.java index b9c9e2f82..07af06601 100644 --- a/source/net/yacy/htroot/yacy/transferURL.java +++ b/source/net/yacy/htroot/yacy/transferURL.java @@ -30,8 +30,11 @@ package net.yacy.htroot.yacy; import java.io.IOException;
import java.text.ParseException;
+import java.util.Date;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import net.yacy.cora.date.GenericFormatter;
import net.yacy.cora.document.encoding.ASCII;
@@ -76,6 +79,7 @@ public final class transferURL { // response values
String result = "";
String doublevalues = "0";
+ final StringBuilder errorURLs = new StringBuilder();
final Seed otherPeer = sb.peers.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
@@ -89,6 +93,7 @@ public final class transferURL { } else {
int received = 0;
int blocked = 0;
+ int blockedErrors = 0;
int doublecheck = 0;
// read the urls from the other properties and store
String urls;
@@ -147,12 +152,78 @@ public final class transferURL { }
doublecheck = 0;
+ final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true);
+ final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30);
+ final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410,-1");
+ final Set<Integer> permanentStatus = new HashSet<Integer>();
+ for (String s : permanentStatusStr.split(",")) {
+ try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {}
+ }
+ final long retryAfterMillis = retryAfterDays * 24L * 60L * 60L * 1000L;
+ final long now = System.currentTimeMillis();
+
for (final String id : lEm.keySet()) {
if (sb.index.exists(id)) {
doublecheck++;
- } else {
- lEntry = lEm.get(id);
-
+ // Check if entry we already have is marked as error - if so, reject incoming replacement
+ if (blockErrors) {
+ try {
+ final URIMetadataNode meta = sb.index.fulltext().getMetadata(ASCII.getBytes(id));
+ if (meta != null && meta.getFieldValue("httpstatus_i") != null) {
+ final int httpstatus = (meta.getFieldValue("httpstatus_i") instanceof Integer) ?
+ (Integer) meta.getFieldValue("httpstatus_i") :
+ Integer.parseInt(meta.getFieldValue("httpstatus_i").toString());
+ final Object failreason = meta.getFieldValue("failreason_s");
+
+ if (httpstatus != 200 && failreason != null && failreason.toString().length() > 0) {
+ boolean shouldBlock = false;
+
+ // Permanent errors (404, 410) - always block
+ if (permanentStatus.contains(httpstatus)) {
+ shouldBlock = true;
+ if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (permanent error, httpstatus=" + httpstatus + ") from peer " + otherPeerName);
+ } else {
+ // Temporary errors - check age
+ final Object loadDate = meta.getFieldValue("load_date_dt");
+ if (loadDate != null) {
+ try {
+ final Date errorDate = (loadDate instanceof Date) ? (Date) loadDate :
+ new Date(Long.parseLong(loadDate.toString()));
+ final long errorAge = now - errorDate.getTime();
+
+ if (errorAge < retryAfterMillis) {
+ shouldBlock = true;
+ if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (temporary error, httpstatus=" + httpstatus + ", age=" + (errorAge / (24*60*60*1000)) + " days) from peer " + otherPeerName);
+ } else {
+ if (Network.log.isFine()) Network.log.fine("transferURL: allowing retry of URL hash '" + id + "' (error age=" + (errorAge / (24*60*60*1000)) + " days exceeds retry threshold) from peer " + otherPeerName);
+ }
+ } catch (Exception e) {
+ // If we can't parse the date, treat as permanent error to be safe
+ shouldBlock = true;
+ }
+ } else {
+ // No load_date available - treat as permanent error
+ shouldBlock = true;
+ }
+ }
+
+ if (shouldBlock) {
+ errorURLs.append(id).append(',');
+ blocked++;
+ blockedErrors++;
+ continue;
+ }
+ }
+ }
+ } catch (final Exception e) {
+ // Ignore errors during error status check
+ }
+ }
+ }
+
+ lEntry = lEm.get(id);
+
+ if (lEntry != null) {
// write entry to database
if (Network.log.isFine()) Network.log.fine("Accepting URL from peer " + otherPeerName + ": " + lEntry.url().toNormalform(true));
try {
@@ -169,8 +240,8 @@ public final class transferURL { sb.peers.mySeed().incRU(received);
// return rewrite properties
- Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs");
- EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " URLs from peer " + otherPeerName, "", otherPeer.hash));
+ Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " (error " + blockedErrors + ") URLs, reporting " + blockedErrors + " error URLs");
+ EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " (error " + blockedErrors + ") URLs, reporting " + blockedErrors + " error URLs from peer " + otherPeerName, "", otherPeer.hash));
if (sb.getConfigBool(SwitchboardConstants.DECORATION_AUDIO, false)) Audio.Soundclip.dhtin.play(-10.0f);
if (doublecheck > 0) {
@@ -182,6 +253,10 @@ public final class transferURL { prop.put("double", doublevalues);
prop.put("result", result);
+ if (errorURLs.length() > 0) {
+ errorURLs.setLength(errorURLs.length() - 1); // remove trailing comma
+ prop.put("errorURL", errorURLs.toString());
+ }
return prop;
}
}
diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 569dfccc9..f81caaa57 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -88,6 +88,7 @@ import net.yacy.cora.document.feed.RSSFeed; import net.yacy.cora.document.feed.RSSMessage;
import net.yacy.cora.document.feed.RSSReader;
import net.yacy.cora.document.id.MultiProtocolURL;
+import net.yacy.cora.federate.solr.FailCategory;
import net.yacy.cora.federate.solr.connector.RemoteSolrConnector;
import net.yacy.cora.federate.solr.connector.SolrConnector;
import net.yacy.cora.federate.solr.instance.RemoteInstance;
@@ -789,24 +790,24 @@ public final class Protocol { }
WriteMetadataNodeToLocalIndexThread writerToLocalIndex = new WriteMetadataNodeToLocalIndexThread(event.query.getSegment(), storeDocs);
writerToLocalIndex.start();
- try { - writerToLocalIndex.join(); - } catch(InterruptedException e) { - /* - * Current thread interruption might happen while waiting - * for writeToLocalIndexThread. - */ - writerToLocalIndex.stopWriting(); - throw new InterruptedException("remoteProcess stopped!"); - } - /* Ensure freshly stored metadata is visible to queries before adding results. */ - event.query.getSegment().fulltext().commit(true); - if (storeDocs != null && !storeDocs.isEmpty()) { - event.addNodes(storeDocs, null, snip, false, target.getName() + "/" + target.hash, result.totalCount, true); - } else { - event.addRWIs(container.get(0), false, target.getName() + "/" + target.hash, result.totalCount, time); - } - } else { + try {
+ writerToLocalIndex.join();
+ } catch(InterruptedException e) {
+ /*
+ * Current thread interruption might happen while waiting
+ * for writeToLocalIndexThread.
+ */
+ writerToLocalIndex.stopWriting();
+ throw new InterruptedException("remoteProcess stopped!");
+ }
+ /* Ensure freshly stored metadata is visible to queries before adding results. */
+ event.query.getSegment().fulltext().commit(true);
+ if (storeDocs != null && !storeDocs.isEmpty()) {
+ event.addNodes(storeDocs, null, snip, false, target.getName() + "/" + target.hash, result.totalCount, true);
+ } else {
+ event.addRWIs(container.get(0), false, target.getName() + "/" + target.hash, result.totalCount, time);
+ }
+ } else {
// feed results as nodes (SolrQuery results) which carry metadata,
// to prevent a call to getMetaData for RWI results, which would fail (if no metadata in index and no display of these results)
event.addNodes(storeDocs, null, snip, false, target.getName() + "/" + target.hash, count, true);
@@ -1737,6 +1738,52 @@ public final class Protocol { return result;
}
+ // DHT error propagation: process error URLs reported by remote peer
+ String errorURLs = in.get("errorURL");
+ if ( errorURLs != null && !errorURLs.isEmpty() && !errorURLs.equals(",") ) {
+ final String[] euhs = CommonPattern.COMMA.split(errorURLs.trim());
+ if ( euhs.length > 0 ) {
+ Network.log.info("DHT: Received " + euhs.length + " error URL reports from peer " + targetSeed.getName() + "/[" + targetSeed.hash + "]");
+ for ( final String errorHash : euhs ) {
+ if ( errorHash == null || errorHash.length() != 12 ) continue;
+ try {
+ // Check if we have this URL locally without error status
+ final URIMetadataNode metadata = segment.fulltext().getMetadata(ASCII.getBytes(errorHash));
+ if ( metadata != null ) {
+ // Extract crawl depth if available; default to 0 when missing
+ int crawldepth = 0;
+ final Object cd = metadata.getFieldValue(CollectionSchema.crawldepth_i.getSolrFieldName());
+ if (cd instanceof Integer) {
+ crawldepth = ((Integer) cd).intValue();
+ } else if (cd instanceof Long) {
+ crawldepth = ((Long) cd).intValue();
+ }
+
+ // Mark as error to prevent re-distribution via DHT
+ sb.crawlQueues.errorURL.push(
+ metadata.url(),
+ crawldepth,
+ null,
+ net.yacy.cora.federate.solr.FailCategory.FINAL_LOAD_CONTEXT,
+ "DHT error propagation from peer " + targetSeed.getName(),
+ -1
+ );
+ if (Network.log.isFine()) Network.log.fine("DHT: Marked URL hash '" + errorHash + "' as error based on peer report");
+ }
+ } catch ( final Exception e ) {
+ Network.log.warn("DHT: Failed to process error URL hash '" + errorHash + "': " + e.getMessage());
+ }
+ }
+ EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(
+ new RSSMessage(
+ "Received " + euhs.length + " error URL reports from peer " + targetSeed.getName() + "/[" + targetSeed.hash + "]",
+ "",
+ targetSeed.hash
+ )
+ );
+ }
+ }
+
// in now contains a list of unknown hashes
String uhss = in.get("unknownURL");
if ( uhss == null ) {
@@ -1773,6 +1820,36 @@ public final class Protocol { sb.peers.addConnected(targetSeed); // update the peer
return result;
}
+
+ // Process error URLs reported back by receiver
+ final String rejectedURLs = in.get("errorURL");
+ if (rejectedURLs != null && !rejectedURLs.isEmpty() && !rejectedURLs.equals(",")) {
+ final String[] ruhs = CommonPattern.COMMA.split(rejectedURLs.trim());
+ if (ruhs.length > 0) {
+ Network.log.info("DHT: Received " + ruhs.length + " rejected error URL reports from peer " + targetSeed.getName());
+ for (final String errorHash : ruhs) {
+ if (errorHash == null || errorHash.length() != 12) continue;
+ try {
+ final URIMetadataNode metadata = segment.fulltext().getMetadata(ASCII.getBytes(errorHash));
+ if (metadata != null) {
+ // Extract crawl depth safely
+ int crawldepth = 0;
+ final Object cd = metadata.getFieldValue("crawldepth_i");
+ if (cd instanceof Integer) crawldepth = ((Integer) cd).intValue();
+ else if (cd instanceof Long) crawldepth = ((Long) cd).intValue();
+
+ // Mark as error locally
+ sb.crawlQueues.errorURL.push(metadata.url(), crawldepth, null,
+ FailCategory.FINAL_LOAD_CONTEXT,
+ "DHT error propagation from peer " + targetSeed.getName(), -1);
+ }
+ } catch (final Exception e) {
+ Network.log.warn("DHT: Failed to process rejected error URL hash '" + errorHash + "': " + e.getMessage());
+ }
+ }
+ }
+ }
+
EventChannel.channels(EventChannel.DHTSEND).addMessage(
new RSSMessage(
"Sent " + uhs.length + " URLs to peer " + targetSeed.getName()+ "/[" + targetSeed.hash + "]",
diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 91cf24484..0e391e8ac 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -207,6 +207,10 @@ public final class SwitchboardConstants { public static final String INDEX_RECEIVE_ALLOW = "allowReceiveIndex"; public static final String INDEX_RECEIVE_ALLOW_SEARCH = "allowReceiveIndex.search"; public static final String INDEX_RECEIVE_BLOCK_BLACKLIST = "indexReceiveBlockBlacklist"; + public static final String INDEX_RECEIVE_BLOCK_ERRORS = "indexReceiveBlockErrors"; + public static final String INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS = "indexReceiveBlockErrors.retryAfterDays"; + /** Permanent error HTTP status codes (comma-separated). Default: 404,410,-1 (-1=DNS/network errors) */ + public static final String INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT = "indexReceiveBlockErrors.permanentStatus"; /** * <p><code>public static final String <strong>INDEX_DIST_ALLOW_WHILE_CRAWLING</strong> = "allowDistributeIndexWhileCrawling"</code></p> |