From fa61beff3550850253055393b89aa18d4c5f705d Mon Sep 17 00:00:00 2001 From: pr0vieh Date: Wed, 21 Jan 2026 23:29:25 +0100 Subject: Add configurable DHT error URL blocking with retry window and web UI - Implement proactive DHT error URL rejection for both URL and RWI transfers - Add configurable opt-out via indexReceiveBlockErrors setting (default: true) - Introduce retry window for temporary errors (default: 30 days) - Permanent errors (404, 410) always blocked, configurable via permanentStatus - Add web UI controls in IndexFederated_p.html under Peer-to-Peer section - Bidirectional feedback: receivers reject and report error URLs to senders - Detailed logging shows blocked vs error-blocked counts separately - Uses load_date_dt field to calculate error age for retry decisions --- defaults/yacy.init | 79 +++++++++++++--------- htroot/IndexFederated_p.html | 9 +++ source/net/yacy/htroot/IndexFederated_p.java | 15 +++++ source/net/yacy/htroot/yacy/transferRWI.java | 86 ++++++++++++++++++++++-- source/net/yacy/htroot/yacy/transferURL.java | 82 +++++++++++++++++----- source/net/yacy/search/SwitchboardConstants.java | 3 + 6 files changed, 224 insertions(+), 50 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index 2274d4d59..0d2fc0dc3 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -295,7 +295,7 @@ proxyCacheSize = 4096 # The compression level for cached content # Supported values ranging from 0 - no compression (lower CPU, higher disk usage), to 9 - best compression (higher CPU, lower disk use) -proxyCache.compressionLevel = 9 +proxyCache.compressionLevel = 6 # Timeout value (in milliseconds) for acquiring a synchronization lock on getContent/store Cache operations # When timeout occurs, loader should fall back to regular remote resource loading @@ -577,6 +577,19 @@ allowReceiveIndex=true allowReceiveIndex.search=true indexReceiveBlockBlacklist=true +# Proactive Error-URL Rejection: block distribution of known broken URLs via DHT +# Set to false to disable this feature (opt-out) +indexReceiveBlockErrors=true + +# How many days to block error URLs before allowing retry +# Permanent errors (404, 410) are always blocked +# Temporary errors (5xx, timeouts) are blocked for this many days +indexReceiveBlockErrors.retryAfterDays=30 + +# Comma-separated list of HTTP status codes considered permanent errors (always block) +# Default: 404 (Not Found), 410 (Gone) +indexReceiveBlockErrors.permanentStatus=404,410 + # the frequency is the number of links per minute, that the peer allowes # _every_ other peer to send to this peer defaultWordReceiveFrequency=100 @@ -709,7 +722,7 @@ recrawlindex_memprereq=1048576 40_peerseedcycle_memprereq=4194304 40_peerseedcycle_loadprereq=2.0 50_localcrawl_idlesleep=2000 -50_localcrawl_busysleep=10 +50_localcrawl_busysleep=5 50_localcrawl_memprereq=25165824 50_localcrawl_loadprereq=8.0 50_localcrawl_isPaused=false @@ -717,13 +730,13 @@ recrawlindex_memprereq=1048576 55_autocrawl_busysleep=10000 55_autocrawl_memprereq=25165824 55_autocrawl_loadprereq=8.0 -60_remotecrawlloader_idlesleep=4000 -60_remotecrawlloader_busysleep=800 +60_remotecrawlloader_idlesleep=2000 +60_remotecrawlloader_busysleep=200 60_remotecrawlloader_memprereq=12582912 60_remotecrawlloader_loadprereq=8.0 60_remotecrawlloader_isPaused=false -62_remotetriggeredcrawl_idlesleep=2000 -62_remotetriggeredcrawl_busysleep=200 +62_remotetriggeredcrawl_idlesleep=1000 +62_remotetriggeredcrawl_busysleep=50 62_remotetriggeredcrawl_memprereq=12582912 62_remotetriggeredcrawl_loadprereq=8.0 62_remotetriggeredcrawl_isPaused=false @@ -759,7 +772,7 @@ reindexSolr_loadprereq=16.0 # is used to flush the RAM cache, which is the major part of the IO in YaCy performanceProfile=defaults/yacy.init performanceSpeed=100 -performanceIO=10 +performanceIO=15 # cleanup-process: # properties for tasks that are performed during cleanup @@ -800,7 +813,13 @@ javastart_priority=10 # wordCacheMaxLow/High is the number of word indexes that shall be held in the # ram cache during indexing. If you want to increase indexing speed, increase this # value i.e. up to one million, but increase also the memory limit to a minimum of 2GB -wordCacheMaxCount = 20000 +wordCacheMaxCount = 50000 + +# Maximum number of references per term in the RWI index +# If a term has more references than this value, the oldest references will be removed +# This prevents memory issues with high-frequency terms (stopwords like 'the', 'and', etc.) +# 0 = disabled (no shrinking), 10000 = recommended for standard installations +index.maxReferences = 10000 # Specifies if yacy can be used as transparent http proxy. # @@ -1063,7 +1082,7 @@ indexDistribution.maxChunkFails = 1 # limit of references per term & blob to the younges of this value # a value of <= 0 disables this feature (no limit) # a value of e.g. 100000 can improve stability and reduce load while searching very popular words -index.maxReferences = 0 +index.maxReferences = 10000 # Search sequence settings # collection: @@ -1416,24 +1435,24 @@ crawler.userAgent.clienttimeout = 10000 # experiments with timeout requests timeoutrequests = true -# interface decorations -decoration.audio = false -decoration.grafics.linkstructure = true -decoration.hostanalysis = false -decoration.simpleheadernavbar = navbar-default - -# ai settings -ai.production_models = [] -ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies. -ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable. -ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n -ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array: -ai.shield.allow-nonlocalhost = false -ai.shield.show-chat-link = false -ai.shield.rate.per-minute = 4 -ai.shield.rate.per-hour = 60 -ai.shield.rate.per-day = 120 -ai.shield.limit-all = false -ai.shield.all.per-minute = 16 -ai.shield.all.per-hour = 240 -ai.shield.all.per-day = 480 +# interface decorations +decoration.audio = false +decoration.grafics.linkstructure = true +decoration.hostanalysis = false +decoration.simpleheadernavbar = navbar-default + +# ai settings +ai.production_models = [] +ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies. +ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable. +ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n +ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array: +ai.shield.allow-nonlocalhost = false +ai.shield.show-chat-link = false +ai.shield.rate.per-minute = 4 +ai.shield.rate.per-hour = 60 +ai.shield.rate.per-day = 120 +ai.shield.limit-all = false +ai.shield.all.per-minute = 16 +ai.shield.all.per-hour = 240 +ai.shield.all.per-day = 480 diff --git a/htroot/IndexFederated_p.html b/htroot/IndexFederated_p.html index 3edd5ceb7..652899a48 100644 --- a/htroot/IndexFederated_p.html +++ b/htroot/IndexFederated_p.html @@ -117,6 +117,15 @@
support peer-to-peer index transmission (DHT RWI index)
+ +
Block known error URLs in DHT 
+
Reject URLs/RWIs with known errors from peers. Disable to opt out.
+ +
Retry after (days)
+
for temporary errors; permanent errors stay blocked.
+ +
Permanent error statuses
+
comma-separated (default: 404,410)
diff --git a/source/net/yacy/htroot/IndexFederated_p.java b/source/net/yacy/htroot/IndexFederated_p.java index bbaba2b3a..9b7e8f74c 100644 --- a/source/net/yacy/htroot/IndexFederated_p.java +++ b/source/net/yacy/htroot/IndexFederated_p.java @@ -64,6 +64,16 @@ public class IndexFederated_p { final long fileSizeMax = (OS.isWindows) ? sb.getConfigLong("filesize.max.win", Integer.MAX_VALUE) : sb.getConfigLong( "filesize.max.other", Integer.MAX_VALUE); sb.index.connectRWI(wordCacheMaxCount, fileSizeMax); } catch (final IOException e) { ConcurrentLog.logException(e); } // switch on + + // DHT error URL blocking settings + final boolean blockErrors = post.getBoolean(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS); + env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, blockErrors); + + final int retryDays = post.getInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30); + env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, retryDays); + + final String permanent = post.get(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410"); + env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, permanent); } if (post != null && post.containsKey("setcitation")) { @@ -222,6 +232,11 @@ public class IndexFederated_p { prop.put("core.service.rwi.checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false) ? 1 : 0); prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0); prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0); + + // DHT error URL blocking settings + prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS + ".checked", env.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true) ? 1 : 0); + prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, env.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30)); + prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, env.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410")); prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED_DEFAULT) ? 1 : 0); diff --git a/source/net/yacy/htroot/yacy/transferRWI.java b/source/net/yacy/htroot/yacy/transferRWI.java index 0981ff70e..d02875334 100644 --- a/source/net/yacy/htroot/yacy/transferRWI.java +++ b/source/net/yacy/htroot/yacy/transferRWI.java @@ -28,11 +28,15 @@ package net.yacy.htroot.yacy; +import java.io.IOException; import java.util.ArrayList; +import java.util.Date; import java.util.HashSet; import java.util.Iterator; import java.util.Set; +import org.apache.solr.common.SolrDocument; + import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.feed.RSSMessage; @@ -55,6 +59,7 @@ import net.yacy.peers.Seed; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; +import net.yacy.search.schema.CollectionSchema; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -75,6 +80,7 @@ public final class transferRWI { prop.put("unknownURL", ""); prop.put("pause", 60000); String result = ""; + final StringBuilder errorURLs = new StringBuilder(4000); if ((post == null) || (env == null)) { result = "post or env is null!"; logWarning(contentType, result); @@ -127,7 +133,6 @@ public final class transferRWI { int pause = 0; result = "ok"; final StringBuilder unknownURLs = new StringBuilder(6000); - final double load = Memory.getSystemLoadAverage(); final float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f); if (load > maxload) { @@ -188,6 +193,7 @@ public final class transferRWI { final ArrayList wordhashes = new ArrayList(); int received = 0; int blocked = 0; + int blockedErrors = 0; int count = 0; final Set testids = new HashSet(); while (it.hasNext()) { @@ -222,6 +228,75 @@ public final class transferRWI { continue; } + // reject RWI entries for URLs we already know are broken + final String urlHashStr = ASCII.String(urlHash); + final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true); + if (blockErrors && sb.index.fulltext().exists(urlHashStr)) { + try { + final SolrDocument errorCheck = sb.index.fulltext().getDefaultConnector().getDocumentById(urlHashStr, + CollectionSchema.httpstatus_i.getSolrFieldName(), CollectionSchema.failreason_s.getSolrFieldName(), + CollectionSchema.load_date_dt.getSolrFieldName()); + if (errorCheck != null) { + final Object httpstatus = errorCheck.getFieldValue(CollectionSchema.httpstatus_i.getSolrFieldName()); + final Object failreason = errorCheck.getFieldValue(CollectionSchema.failreason_s.getSolrFieldName()); + if (httpstatus != null && failreason != null && failreason.toString().length() > 0) { + int hs = (httpstatus instanceof Integer) ? (Integer) httpstatus : Integer.parseInt(httpstatus.toString()); + if (hs != 200) { + boolean shouldBlock = false; + + // Get configuration + final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30); + final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410"); + final Set permanentStatus = new HashSet(); + for (String s : permanentStatusStr.split(",")) { + try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {} + } + final long retryAfterMillis = retryAfterDays * 24L * 60L * 60L * 1000L; + final long now = System.currentTimeMillis(); + + // Permanent errors (404, 410) - always block + if (permanentStatus.contains(hs)) { + shouldBlock = true; + if (Network.log.isFine()) Network.log.fine("transferRWI: rejected RWI for known permanent error URL hash '" + urlHashStr + "' (httpstatus=" + hs + ") from peer " + otherPeerName); + } else { + // Temporary errors - check age + final Object loadDate = errorCheck.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName()); + if (loadDate != null) { + try { + final Date errorDate = (loadDate instanceof Date) ? (Date) loadDate : + new Date(Long.parseLong(loadDate.toString())); + final long errorAge = now - errorDate.getTime(); + + if (errorAge < retryAfterMillis) { + shouldBlock = true; + if (Network.log.isFine()) Network.log.fine("transferRWI: rejected RWI for known temporary error URL hash '" + urlHashStr + "' (httpstatus=" + hs + ", age=" + (errorAge / (24*60*60*1000)) + " days) from peer " + otherPeerName); + } else { + if (Network.log.isFine()) Network.log.fine("transferRWI: allowing retry for URL hash '" + urlHashStr + "' (error age=" + (errorAge / (24*60*60*1000)) + " days exceeds retry threshold) from peer " + otherPeerName); + } + } catch (Exception e) { + // If we can't parse the date, treat as permanent error to be safe + shouldBlock = true; + } + } else { + // No load_date available - treat as permanent error + shouldBlock = true; + } + } + + if (shouldBlock) { + errorURLs.append(urlHashStr).append(','); + blocked++; + blockedErrors++; + continue; + } + } + } + } + } catch (final IOException e) { + // ignore Solr errors during error URL check + } + } + // learn entry try { sb.index.storeRWI(ASCII.getBytes(wordHash), iEntry); @@ -251,14 +326,16 @@ public final class transferRWI { unknownURLs.append(UTF8.String(bit.next())).append(','); } if (unknownURLs.length() > 0) { unknownURLs.setLength(unknownURLs.length() - 1); } + if (errorURLs.length() > 0) { errorURLs.setLength(errorURLs.length() - 1); } + if (wordhashes.isEmpty() || received == 0) { - sb.getLog().info("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs"); + sb.getLog().info("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs, reporting " + blockedErrors + " error URLs"); } else { final String firstHash = wordhashes.get(0); final String lastHash = wordhashes.get(wordhashes.size() - 1); final long avdist = (Distribution.horizontalDHTDistance(firstHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash)) + Distribution.horizontalDHTDistance(lastHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash))) / 2; - sb.getLog().info("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + received+ " URLs from " + otherPeerName); - EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + received + " URLs from " + otherPeerName, "", otherPeer.hash)); + sb.getLog().info("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + " (error " + blockedErrors + "), requesting " + unknownURL.size() + "/" + received+ " URLs, reporting " + blockedErrors + " error URLs from " + otherPeerName); + EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + " (error " + blockedErrors + "), requesting " + unknownURL.size() + "/" + received + " URLs, reporting " + blockedErrors + " error URLs from " + otherPeerName, "", otherPeer.hash)); } result = "ok"; @@ -266,6 +343,7 @@ public final class transferRWI { } prop.put("unknownURL", unknownURLs.toString()); + prop.put("errorURL", errorURLs.toString()); prop.put("result", result); prop.put("pause", pause); diff --git a/source/net/yacy/htroot/yacy/transferURL.java b/source/net/yacy/htroot/yacy/transferURL.java index f9b95e270..2ca3f533a 100644 --- a/source/net/yacy/htroot/yacy/transferURL.java +++ b/source/net/yacy/htroot/yacy/transferURL.java @@ -30,8 +30,11 @@ package net.yacy.htroot.yacy; import java.io.IOException; import java.text.ParseException; +import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.encoding.ASCII; @@ -90,6 +93,7 @@ public final class transferURL { } else { int received = 0; int blocked = 0; + int blockedErrors = 0; int doublecheck = 0; // read the urls from the other properties and store String urls; @@ -148,26 +152,72 @@ public final class transferURL { } doublecheck = 0; + final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true); + final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30); + final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410"); + final Set permanentStatus = new HashSet(); + for (String s : permanentStatusStr.split(",")) { + try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {} + } + final long retryAfterMillis = retryAfterDays * 24L * 60L * 60L * 1000L; + final long now = System.currentTimeMillis(); + for (final String id : lEm.keySet()) { if (sb.index.exists(id)) { doublecheck++; // Check if entry we already have is marked as error - if so, reject incoming replacement - try { - final URIMetadataNode meta = sb.index.fulltext().getMetadata(ASCII.getBytes(id)); - if (meta != null && meta.getFieldValue("httpstatus_i") != null) { - final int httpstatus = (meta.getFieldValue("httpstatus_i") instanceof Integer) ? - (Integer) meta.getFieldValue("httpstatus_i") : - Integer.parseInt(meta.getFieldValue("httpstatus_i").toString()); - final Object failreason = meta.getFieldValue("failreason_s"); - if (httpstatus != 200 && failreason != null && failreason.toString().length() > 0) { - if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (known error, httpstatus=" + httpstatus + ") from peer " + otherPeerName); - errorURLs.append(id).append(','); - blocked++; - continue; + if (blockErrors) { + try { + final URIMetadataNode meta = sb.index.fulltext().getMetadata(ASCII.getBytes(id)); + if (meta != null && meta.getFieldValue("httpstatus_i") != null) { + final int httpstatus = (meta.getFieldValue("httpstatus_i") instanceof Integer) ? + (Integer) meta.getFieldValue("httpstatus_i") : + Integer.parseInt(meta.getFieldValue("httpstatus_i").toString()); + final Object failreason = meta.getFieldValue("failreason_s"); + + if (httpstatus != 200 && failreason != null && failreason.toString().length() > 0) { + boolean shouldBlock = false; + + // Permanent errors (404, 410) - always block + if (permanentStatus.contains(httpstatus)) { + shouldBlock = true; + if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (permanent error, httpstatus=" + httpstatus + ") from peer " + otherPeerName); + } else { + // Temporary errors - check age + final Object loadDate = meta.getFieldValue("load_date_dt"); + if (loadDate != null) { + try { + final Date errorDate = (loadDate instanceof Date) ? (Date) loadDate : + new Date(Long.parseLong(loadDate.toString())); + final long errorAge = now - errorDate.getTime(); + + if (errorAge < retryAfterMillis) { + shouldBlock = true; + if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (temporary error, httpstatus=" + httpstatus + ", age=" + (errorAge / (24*60*60*1000)) + " days) from peer " + otherPeerName); + } else { + if (Network.log.isFine()) Network.log.fine("transferURL: allowing retry of URL hash '" + id + "' (error age=" + (errorAge / (24*60*60*1000)) + " days exceeds retry threshold) from peer " + otherPeerName); + } + } catch (Exception e) { + // If we can't parse the date, treat as permanent error to be safe + shouldBlock = true; + } + } else { + // No load_date available - treat as permanent error + shouldBlock = true; + } + } + + if (shouldBlock) { + errorURLs.append(id).append(','); + blocked++; + blockedErrors++; + continue; + } + } } + } catch (final Exception e) { + // Ignore errors during error status check } - } catch (final Exception e) { - // Ignore errors during error status check } } @@ -190,8 +240,8 @@ public final class transferURL { sb.peers.mySeed().incRU(received); // return rewrite properties - Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs"); - EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " URLs from peer " + otherPeerName, "", otherPeer.hash)); + Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " (error " + blockedErrors + ") URLs, reporting " + blockedErrors + " error URLs"); + EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " (error " + blockedErrors + ") URLs, reporting " + blockedErrors + " error URLs from peer " + otherPeerName, "", otherPeer.hash)); if (sb.getConfigBool(SwitchboardConstants.DECORATION_AUDIO, false)) Audio.Soundclip.dhtin.play(-10.0f); if (doublecheck > 0) { diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 91cf24484..3873e15eb 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -207,6 +207,9 @@ public final class SwitchboardConstants { public static final String INDEX_RECEIVE_ALLOW = "allowReceiveIndex"; public static final String INDEX_RECEIVE_ALLOW_SEARCH = "allowReceiveIndex.search"; public static final String INDEX_RECEIVE_BLOCK_BLACKLIST = "indexReceiveBlockBlacklist"; + public static final String INDEX_RECEIVE_BLOCK_ERRORS = "indexReceiveBlockErrors"; + public static final String INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS = "indexReceiveBlockErrors.retryAfterDays"; + public static final String INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT = "indexReceiveBlockErrors.permanentStatus"; /** *

public static final String INDEX_DIST_ALLOW_WHILE_CRAWLING = "allowDistributeIndexWhileCrawling"

-- cgit v1.2.3