From 5b55319d271cca94b798ce5db5faecffc4d34eef Mon Sep 17 00:00:00 2001 From: pr0vieh Date: Tue, 20 Jan 2026 22:17:14 +0100 Subject: feat: Add two-layer DHT error propagation to prevent broken URL redistribution Problem: Failed URLs (404, DNS errors, timeouts) are continuously redistributed via DHT, causing infinite recrawl loops and network-wide index pollution. No mechanism exists in YaCy to communicate error status across peers. Solution - Layer 1 (Proactive Rejection): - Receiver checks local Solr index for httpstatus_i != 200 BEFORE accepting RWI entries - Rejects URL immediately if marked as failed previously - Adds rejected URL hash to errorURL response list - Prevents index pollution at ingestion time - Works even if sender doesn't support errorURL protocol (backward compatible) Solution - Layer 2 (Error Feedback): - Receiver reports rejected error URLs back to sender via errorURL response parameter - Sender receives errorURL list and marks those URLs locally as failed - Sender stops re-distributing these URLs to other peers - Network-wide error propagation prevents repeated distribution cycles Implementation Details: - transferURL.java: Implements proactive rejection + error reporting * Checks incoming URL against Solr error status before storing * Collects rejected URL hashes in errorURLs StringBuilder * Returns errorURL list to sender in response - Protocol.java: Processes error URL feedback from receiver * Extracts errorURL from response * Marks reported URLs locally via crawlQueues.errorURL.push() * Logs DHT error reports for monitoring Benefits: - Dramatically reduces network traffic of broken URLs - Prevents wasted crawl resources on unreachable targets - Maintains clean, usable index across distributed network - Defense-in-depth: two independent layers work together - Backward compatible: old peers ignore errorURL parameter Testing: - Log monitoring shows 'DHT: Received X rejected error URL reports from peer Y' - Proactive rejection shows 'blocked X URLs' in transfer logs - Error URLs automatically removed from circulation --- source/net/yacy/htroot/yacy/transferURL.java | 31 +++++++- source/net/yacy/peers/Protocol.java | 113 ++++++++++++++++++++++----- 2 files changed, 123 insertions(+), 21 deletions(-) diff --git a/source/net/yacy/htroot/yacy/transferURL.java b/source/net/yacy/htroot/yacy/transferURL.java index b9c9e2f82..f9b95e270 100644 --- a/source/net/yacy/htroot/yacy/transferURL.java +++ b/source/net/yacy/htroot/yacy/transferURL.java @@ -76,6 +76,7 @@ public final class transferURL { // response values String result = ""; String doublevalues = "0"; + final StringBuilder errorURLs = new StringBuilder(); final Seed otherPeer = sb.peers.get(iam); final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion())); @@ -150,9 +151,29 @@ public final class transferURL { for (final String id : lEm.keySet()) { if (sb.index.exists(id)) { doublecheck++; - } else { - lEntry = lEm.get(id); - + // Check if entry we already have is marked as error - if so, reject incoming replacement + try { + final URIMetadataNode meta = sb.index.fulltext().getMetadata(ASCII.getBytes(id)); + if (meta != null && meta.getFieldValue("httpstatus_i") != null) { + final int httpstatus = (meta.getFieldValue("httpstatus_i") instanceof Integer) ? + (Integer) meta.getFieldValue("httpstatus_i") : + Integer.parseInt(meta.getFieldValue("httpstatus_i").toString()); + final Object failreason = meta.getFieldValue("failreason_s"); + if (httpstatus != 200 && failreason != null && failreason.toString().length() > 0) { + if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (known error, httpstatus=" + httpstatus + ") from peer " + otherPeerName); + errorURLs.append(id).append(','); + blocked++; + continue; + } + } + } catch (final Exception e) { + // Ignore errors during error status check + } + } + + lEntry = lEm.get(id); + + if (lEntry != null) { // write entry to database if (Network.log.isFine()) Network.log.fine("Accepting URL from peer " + otherPeerName + ": " + lEntry.url().toNormalform(true)); try { @@ -182,6 +203,10 @@ public final class transferURL { prop.put("double", doublevalues); prop.put("result", result); + if (errorURLs.length() > 0) { + errorURLs.setLength(errorURLs.length() - 1); // remove trailing comma + prop.put("errorURL", errorURLs.toString()); + } return prop; } } diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 569dfccc9..f81caaa57 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -88,6 +88,7 @@ import net.yacy.cora.document.feed.RSSFeed; import net.yacy.cora.document.feed.RSSMessage; import net.yacy.cora.document.feed.RSSReader; import net.yacy.cora.document.id.MultiProtocolURL; +import net.yacy.cora.federate.solr.FailCategory; import net.yacy.cora.federate.solr.connector.RemoteSolrConnector; import net.yacy.cora.federate.solr.connector.SolrConnector; import net.yacy.cora.federate.solr.instance.RemoteInstance; @@ -789,24 +790,24 @@ public final class Protocol { } WriteMetadataNodeToLocalIndexThread writerToLocalIndex = new WriteMetadataNodeToLocalIndexThread(event.query.getSegment(), storeDocs); writerToLocalIndex.start(); - try { - writerToLocalIndex.join(); - } catch(InterruptedException e) { - /* - * Current thread interruption might happen while waiting - * for writeToLocalIndexThread. - */ - writerToLocalIndex.stopWriting(); - throw new InterruptedException("remoteProcess stopped!"); - } - /* Ensure freshly stored metadata is visible to queries before adding results. */ - event.query.getSegment().fulltext().commit(true); - if (storeDocs != null && !storeDocs.isEmpty()) { - event.addNodes(storeDocs, null, snip, false, target.getName() + "/" + target.hash, result.totalCount, true); - } else { - event.addRWIs(container.get(0), false, target.getName() + "/" + target.hash, result.totalCount, time); - } - } else { + try { + writerToLocalIndex.join(); + } catch(InterruptedException e) { + /* + * Current thread interruption might happen while waiting + * for writeToLocalIndexThread. + */ + writerToLocalIndex.stopWriting(); + throw new InterruptedException("remoteProcess stopped!"); + } + /* Ensure freshly stored metadata is visible to queries before adding results. */ + event.query.getSegment().fulltext().commit(true); + if (storeDocs != null && !storeDocs.isEmpty()) { + event.addNodes(storeDocs, null, snip, false, target.getName() + "/" + target.hash, result.totalCount, true); + } else { + event.addRWIs(container.get(0), false, target.getName() + "/" + target.hash, result.totalCount, time); + } + } else { // feed results as nodes (SolrQuery results) which carry metadata, // to prevent a call to getMetaData for RWI results, which would fail (if no metadata in index and no display of these results) event.addNodes(storeDocs, null, snip, false, target.getName() + "/" + target.hash, count, true); @@ -1737,6 +1738,52 @@ public final class Protocol { return result; } + // DHT error propagation: process error URLs reported by remote peer + String errorURLs = in.get("errorURL"); + if ( errorURLs != null && !errorURLs.isEmpty() && !errorURLs.equals(",") ) { + final String[] euhs = CommonPattern.COMMA.split(errorURLs.trim()); + if ( euhs.length > 0 ) { + Network.log.info("DHT: Received " + euhs.length + " error URL reports from peer " + targetSeed.getName() + "/[" + targetSeed.hash + "]"); + for ( final String errorHash : euhs ) { + if ( errorHash == null || errorHash.length() != 12 ) continue; + try { + // Check if we have this URL locally without error status + final URIMetadataNode metadata = segment.fulltext().getMetadata(ASCII.getBytes(errorHash)); + if ( metadata != null ) { + // Extract crawl depth if available; default to 0 when missing + int crawldepth = 0; + final Object cd = metadata.getFieldValue(CollectionSchema.crawldepth_i.getSolrFieldName()); + if (cd instanceof Integer) { + crawldepth = ((Integer) cd).intValue(); + } else if (cd instanceof Long) { + crawldepth = ((Long) cd).intValue(); + } + + // Mark as error to prevent re-distribution via DHT + sb.crawlQueues.errorURL.push( + metadata.url(), + crawldepth, + null, + net.yacy.cora.federate.solr.FailCategory.FINAL_LOAD_CONTEXT, + "DHT error propagation from peer " + targetSeed.getName(), + -1 + ); + if (Network.log.isFine()) Network.log.fine("DHT: Marked URL hash '" + errorHash + "' as error based on peer report"); + } + } catch ( final Exception e ) { + Network.log.warn("DHT: Failed to process error URL hash '" + errorHash + "': " + e.getMessage()); + } + } + EventChannel.channels(EventChannel.DHTRECEIVE).addMessage( + new RSSMessage( + "Received " + euhs.length + " error URL reports from peer " + targetSeed.getName() + "/[" + targetSeed.hash + "]", + "", + targetSeed.hash + ) + ); + } + } + // in now contains a list of unknown hashes String uhss = in.get("unknownURL"); if ( uhss == null ) { @@ -1773,6 +1820,36 @@ public final class Protocol { sb.peers.addConnected(targetSeed); // update the peer return result; } + + // Process error URLs reported back by receiver + final String rejectedURLs = in.get("errorURL"); + if (rejectedURLs != null && !rejectedURLs.isEmpty() && !rejectedURLs.equals(",")) { + final String[] ruhs = CommonPattern.COMMA.split(rejectedURLs.trim()); + if (ruhs.length > 0) { + Network.log.info("DHT: Received " + ruhs.length + " rejected error URL reports from peer " + targetSeed.getName()); + for (final String errorHash : ruhs) { + if (errorHash == null || errorHash.length() != 12) continue; + try { + final URIMetadataNode metadata = segment.fulltext().getMetadata(ASCII.getBytes(errorHash)); + if (metadata != null) { + // Extract crawl depth safely + int crawldepth = 0; + final Object cd = metadata.getFieldValue("crawldepth_i"); + if (cd instanceof Integer) crawldepth = ((Integer) cd).intValue(); + else if (cd instanceof Long) crawldepth = ((Long) cd).intValue(); + + // Mark as error locally + sb.crawlQueues.errorURL.push(metadata.url(), crawldepth, null, + FailCategory.FINAL_LOAD_CONTEXT, + "DHT error propagation from peer " + targetSeed.getName(), -1); + } + } catch (final Exception e) { + Network.log.warn("DHT: Failed to process rejected error URL hash '" + errorHash + "': " + e.getMessage()); + } + } + } + } + EventChannel.channels(EventChannel.DHTSEND).addMessage( new RSSMessage( "Sent " + uhs.length + " URLs to peer " + targetSeed.getName()+ "/[" + targetSeed.hash + "]", -- cgit v1.2.3 From fa61beff3550850253055393b89aa18d4c5f705d Mon Sep 17 00:00:00 2001 From: pr0vieh Date: Wed, 21 Jan 2026 23:29:25 +0100 Subject: Add configurable DHT error URL blocking with retry window and web UI - Implement proactive DHT error URL rejection for both URL and RWI transfers - Add configurable opt-out via indexReceiveBlockErrors setting (default: true) - Introduce retry window for temporary errors (default: 30 days) - Permanent errors (404, 410) always blocked, configurable via permanentStatus - Add web UI controls in IndexFederated_p.html under Peer-to-Peer section - Bidirectional feedback: receivers reject and report error URLs to senders - Detailed logging shows blocked vs error-blocked counts separately - Uses load_date_dt field to calculate error age for retry decisions --- defaults/yacy.init | 79 +++++++++++++--------- htroot/IndexFederated_p.html | 9 +++ source/net/yacy/htroot/IndexFederated_p.java | 15 +++++ source/net/yacy/htroot/yacy/transferRWI.java | 86 ++++++++++++++++++++++-- source/net/yacy/htroot/yacy/transferURL.java | 82 +++++++++++++++++----- source/net/yacy/search/SwitchboardConstants.java | 3 + 6 files changed, 224 insertions(+), 50 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index 2274d4d59..0d2fc0dc3 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -295,7 +295,7 @@ proxyCacheSize = 4096 # The compression level for cached content # Supported values ranging from 0 - no compression (lower CPU, higher disk usage), to 9 - best compression (higher CPU, lower disk use) -proxyCache.compressionLevel = 9 +proxyCache.compressionLevel = 6 # Timeout value (in milliseconds) for acquiring a synchronization lock on getContent/store Cache operations # When timeout occurs, loader should fall back to regular remote resource loading @@ -577,6 +577,19 @@ allowReceiveIndex=true allowReceiveIndex.search=true indexReceiveBlockBlacklist=true +# Proactive Error-URL Rejection: block distribution of known broken URLs via DHT +# Set to false to disable this feature (opt-out) +indexReceiveBlockErrors=true + +# How many days to block error URLs before allowing retry +# Permanent errors (404, 410) are always blocked +# Temporary errors (5xx, timeouts) are blocked for this many days +indexReceiveBlockErrors.retryAfterDays=30 + +# Comma-separated list of HTTP status codes considered permanent errors (always block) +# Default: 404 (Not Found), 410 (Gone) +indexReceiveBlockErrors.permanentStatus=404,410 + # the frequency is the number of links per minute, that the peer allowes # _every_ other peer to send to this peer defaultWordReceiveFrequency=100 @@ -709,7 +722,7 @@ recrawlindex_memprereq=1048576 40_peerseedcycle_memprereq=4194304 40_peerseedcycle_loadprereq=2.0 50_localcrawl_idlesleep=2000 -50_localcrawl_busysleep=10 +50_localcrawl_busysleep=5 50_localcrawl_memprereq=25165824 50_localcrawl_loadprereq=8.0 50_localcrawl_isPaused=false @@ -717,13 +730,13 @@ recrawlindex_memprereq=1048576 55_autocrawl_busysleep=10000 55_autocrawl_memprereq=25165824 55_autocrawl_loadprereq=8.0 -60_remotecrawlloader_idlesleep=4000 -60_remotecrawlloader_busysleep=800 +60_remotecrawlloader_idlesleep=2000 +60_remotecrawlloader_busysleep=200 60_remotecrawlloader_memprereq=12582912 60_remotecrawlloader_loadprereq=8.0 60_remotecrawlloader_isPaused=false -62_remotetriggeredcrawl_idlesleep=2000 -62_remotetriggeredcrawl_busysleep=200 +62_remotetriggeredcrawl_idlesleep=1000 +62_remotetriggeredcrawl_busysleep=50 62_remotetriggeredcrawl_memprereq=12582912 62_remotetriggeredcrawl_loadprereq=8.0 62_remotetriggeredcrawl_isPaused=false @@ -759,7 +772,7 @@ reindexSolr_loadprereq=16.0 # is used to flush the RAM cache, which is the major part of the IO in YaCy performanceProfile=defaults/yacy.init performanceSpeed=100 -performanceIO=10 +performanceIO=15 # cleanup-process: # properties for tasks that are performed during cleanup @@ -800,7 +813,13 @@ javastart_priority=10 # wordCacheMaxLow/High is the number of word indexes that shall be held in the # ram cache during indexing. If you want to increase indexing speed, increase this # value i.e. up to one million, but increase also the memory limit to a minimum of 2GB -wordCacheMaxCount = 20000 +wordCacheMaxCount = 50000 + +# Maximum number of references per term in the RWI index +# If a term has more references than this value, the oldest references will be removed +# This prevents memory issues with high-frequency terms (stopwords like 'the', 'and', etc.) +# 0 = disabled (no shrinking), 10000 = recommended for standard installations +index.maxReferences = 10000 # Specifies if yacy can be used as transparent http proxy. # @@ -1063,7 +1082,7 @@ indexDistribution.maxChunkFails = 1 # limit of references per term & blob to the younges of this value # a value of <= 0 disables this feature (no limit) # a value of e.g. 100000 can improve stability and reduce load while searching very popular words -index.maxReferences = 0 +index.maxReferences = 10000 # Search sequence settings # collection: @@ -1416,24 +1435,24 @@ crawler.userAgent.clienttimeout = 10000 # experiments with timeout requests timeoutrequests = true -# interface decorations -decoration.audio = false -decoration.grafics.linkstructure = true -decoration.hostanalysis = false -decoration.simpleheadernavbar = navbar-default - -# ai settings -ai.production_models = [] -ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies. -ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable. -ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n -ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array: -ai.shield.allow-nonlocalhost = false -ai.shield.show-chat-link = false -ai.shield.rate.per-minute = 4 -ai.shield.rate.per-hour = 60 -ai.shield.rate.per-day = 120 -ai.shield.limit-all = false -ai.shield.all.per-minute = 16 -ai.shield.all.per-hour = 240 -ai.shield.all.per-day = 480 +# interface decorations +decoration.audio = false +decoration.grafics.linkstructure = true +decoration.hostanalysis = false +decoration.simpleheadernavbar = navbar-default + +# ai settings +ai.production_models = [] +ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies. +ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable. +ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n +ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array: +ai.shield.allow-nonlocalhost = false +ai.shield.show-chat-link = false +ai.shield.rate.per-minute = 4 +ai.shield.rate.per-hour = 60 +ai.shield.rate.per-day = 120 +ai.shield.limit-all = false +ai.shield.all.per-minute = 16 +ai.shield.all.per-hour = 240 +ai.shield.all.per-day = 480 diff --git a/htroot/IndexFederated_p.html b/htroot/IndexFederated_p.html index 3edd5ceb7..652899a48 100644 --- a/htroot/IndexFederated_p.html +++ b/htroot/IndexFederated_p.html @@ -117,6 +117,15 @@
support peer-to-peer index transmission (DHT RWI index)
+ +
Block known error URLs in DHT 
+
Reject URLs/RWIs with known errors from peers. Disable to opt out.
+ +
Retry after (days)
+
for temporary errors; permanent errors stay blocked.
+ +
Permanent error statuses
+
comma-separated (default: 404,410)
diff --git a/source/net/yacy/htroot/IndexFederated_p.java b/source/net/yacy/htroot/IndexFederated_p.java index bbaba2b3a..9b7e8f74c 100644 --- a/source/net/yacy/htroot/IndexFederated_p.java +++ b/source/net/yacy/htroot/IndexFederated_p.java @@ -64,6 +64,16 @@ public class IndexFederated_p { final long fileSizeMax = (OS.isWindows) ? sb.getConfigLong("filesize.max.win", Integer.MAX_VALUE) : sb.getConfigLong( "filesize.max.other", Integer.MAX_VALUE); sb.index.connectRWI(wordCacheMaxCount, fileSizeMax); } catch (final IOException e) { ConcurrentLog.logException(e); } // switch on + + // DHT error URL blocking settings + final boolean blockErrors = post.getBoolean(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS); + env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, blockErrors); + + final int retryDays = post.getInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30); + env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, retryDays); + + final String permanent = post.get(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410"); + env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, permanent); } if (post != null && post.containsKey("setcitation")) { @@ -222,6 +232,11 @@ public class IndexFederated_p { prop.put("core.service.rwi.checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false) ? 1 : 0); prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0); prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0); + + // DHT error URL blocking settings + prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS + ".checked", env.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true) ? 1 : 0); + prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, env.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30)); + prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, env.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410")); prop.put("solr.indexing.solrremote.checked", env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED, SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED_DEFAULT) ? 1 : 0); diff --git a/source/net/yacy/htroot/yacy/transferRWI.java b/source/net/yacy/htroot/yacy/transferRWI.java index 0981ff70e..d02875334 100644 --- a/source/net/yacy/htroot/yacy/transferRWI.java +++ b/source/net/yacy/htroot/yacy/transferRWI.java @@ -28,11 +28,15 @@ package net.yacy.htroot.yacy; +import java.io.IOException; import java.util.ArrayList; +import java.util.Date; import java.util.HashSet; import java.util.Iterator; import java.util.Set; +import org.apache.solr.common.SolrDocument; + import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.feed.RSSMessage; @@ -55,6 +59,7 @@ import net.yacy.peers.Seed; import net.yacy.repository.Blacklist.BlacklistType; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; +import net.yacy.search.schema.CollectionSchema; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -75,6 +80,7 @@ public final class transferRWI { prop.put("unknownURL", ""); prop.put("pause", 60000); String result = ""; + final StringBuilder errorURLs = new StringBuilder(4000); if ((post == null) || (env == null)) { result = "post or env is null!"; logWarning(contentType, result); @@ -127,7 +133,6 @@ public final class transferRWI { int pause = 0; result = "ok"; final StringBuilder unknownURLs = new StringBuilder(6000); - final double load = Memory.getSystemLoadAverage(); final float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f); if (load > maxload) { @@ -188,6 +193,7 @@ public final class transferRWI { final ArrayList wordhashes = new ArrayList(); int received = 0; int blocked = 0; + int blockedErrors = 0; int count = 0; final Set testids = new HashSet(); while (it.hasNext()) { @@ -222,6 +228,75 @@ public final class transferRWI { continue; } + // reject RWI entries for URLs we already know are broken + final String urlHashStr = ASCII.String(urlHash); + final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true); + if (blockErrors && sb.index.fulltext().exists(urlHashStr)) { + try { + final SolrDocument errorCheck = sb.index.fulltext().getDefaultConnector().getDocumentById(urlHashStr, + CollectionSchema.httpstatus_i.getSolrFieldName(), CollectionSchema.failreason_s.getSolrFieldName(), + CollectionSchema.load_date_dt.getSolrFieldName()); + if (errorCheck != null) { + final Object httpstatus = errorCheck.getFieldValue(CollectionSchema.httpstatus_i.getSolrFieldName()); + final Object failreason = errorCheck.getFieldValue(CollectionSchema.failreason_s.getSolrFieldName()); + if (httpstatus != null && failreason != null && failreason.toString().length() > 0) { + int hs = (httpstatus instanceof Integer) ? (Integer) httpstatus : Integer.parseInt(httpstatus.toString()); + if (hs != 200) { + boolean shouldBlock = false; + + // Get configuration + final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30); + final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410"); + final Set permanentStatus = new HashSet(); + for (String s : permanentStatusStr.split(",")) { + try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {} + } + final long retryAfterMillis = retryAfterDays * 24L * 60L * 60L * 1000L; + final long now = System.currentTimeMillis(); + + // Permanent errors (404, 410) - always block + if (permanentStatus.contains(hs)) { + shouldBlock = true; + if (Network.log.isFine()) Network.log.fine("transferRWI: rejected RWI for known permanent error URL hash '" + urlHashStr + "' (httpstatus=" + hs + ") from peer " + otherPeerName); + } else { + // Temporary errors - check age + final Object loadDate = errorCheck.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName()); + if (loadDate != null) { + try { + final Date errorDate = (loadDate instanceof Date) ? (Date) loadDate : + new Date(Long.parseLong(loadDate.toString())); + final long errorAge = now - errorDate.getTime(); + + if (errorAge < retryAfterMillis) { + shouldBlock = true; + if (Network.log.isFine()) Network.log.fine("transferRWI: rejected RWI for known temporary error URL hash '" + urlHashStr + "' (httpstatus=" + hs + ", age=" + (errorAge / (24*60*60*1000)) + " days) from peer " + otherPeerName); + } else { + if (Network.log.isFine()) Network.log.fine("transferRWI: allowing retry for URL hash '" + urlHashStr + "' (error age=" + (errorAge / (24*60*60*1000)) + " days exceeds retry threshold) from peer " + otherPeerName); + } + } catch (Exception e) { + // If we can't parse the date, treat as permanent error to be safe + shouldBlock = true; + } + } else { + // No load_date available - treat as permanent error + shouldBlock = true; + } + } + + if (shouldBlock) { + errorURLs.append(urlHashStr).append(','); + blocked++; + blockedErrors++; + continue; + } + } + } + } + } catch (final IOException e) { + // ignore Solr errors during error URL check + } + } + // learn entry try { sb.index.storeRWI(ASCII.getBytes(wordHash), iEntry); @@ -251,14 +326,16 @@ public final class transferRWI { unknownURLs.append(UTF8.String(bit.next())).append(','); } if (unknownURLs.length() > 0) { unknownURLs.setLength(unknownURLs.length() - 1); } + if (errorURLs.length() > 0) { errorURLs.setLength(errorURLs.length() - 1); } + if (wordhashes.isEmpty() || received == 0) { - sb.getLog().info("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs"); + sb.getLog().info("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs, reporting " + blockedErrors + " error URLs"); } else { final String firstHash = wordhashes.get(0); final String lastHash = wordhashes.get(wordhashes.size() - 1); final long avdist = (Distribution.horizontalDHTDistance(firstHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash)) + Distribution.horizontalDHTDistance(lastHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash))) / 2; - sb.getLog().info("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + received+ " URLs from " + otherPeerName); - EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + received + " URLs from " + otherPeerName, "", otherPeer.hash)); + sb.getLog().info("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + " (error " + blockedErrors + "), requesting " + unknownURL.size() + "/" + received+ " URLs, reporting " + blockedErrors + " error URLs from " + otherPeerName); + EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + " (error " + blockedErrors + "), requesting " + unknownURL.size() + "/" + received + " URLs, reporting " + blockedErrors + " error URLs from " + otherPeerName, "", otherPeer.hash)); } result = "ok"; @@ -266,6 +343,7 @@ public final class transferRWI { } prop.put("unknownURL", unknownURLs.toString()); + prop.put("errorURL", errorURLs.toString()); prop.put("result", result); prop.put("pause", pause); diff --git a/source/net/yacy/htroot/yacy/transferURL.java b/source/net/yacy/htroot/yacy/transferURL.java index f9b95e270..2ca3f533a 100644 --- a/source/net/yacy/htroot/yacy/transferURL.java +++ b/source/net/yacy/htroot/yacy/transferURL.java @@ -30,8 +30,11 @@ package net.yacy.htroot.yacy; import java.io.IOException; import java.text.ParseException; +import java.util.Date; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import net.yacy.cora.date.GenericFormatter; import net.yacy.cora.document.encoding.ASCII; @@ -90,6 +93,7 @@ public final class transferURL { } else { int received = 0; int blocked = 0; + int blockedErrors = 0; int doublecheck = 0; // read the urls from the other properties and store String urls; @@ -148,26 +152,72 @@ public final class transferURL { } doublecheck = 0; + final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true); + final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30); + final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410"); + final Set permanentStatus = new HashSet(); + for (String s : permanentStatusStr.split(",")) { + try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {} + } + final long retryAfterMillis = retryAfterDays * 24L * 60L * 60L * 1000L; + final long now = System.currentTimeMillis(); + for (final String id : lEm.keySet()) { if (sb.index.exists(id)) { doublecheck++; // Check if entry we already have is marked as error - if so, reject incoming replacement - try { - final URIMetadataNode meta = sb.index.fulltext().getMetadata(ASCII.getBytes(id)); - if (meta != null && meta.getFieldValue("httpstatus_i") != null) { - final int httpstatus = (meta.getFieldValue("httpstatus_i") instanceof Integer) ? - (Integer) meta.getFieldValue("httpstatus_i") : - Integer.parseInt(meta.getFieldValue("httpstatus_i").toString()); - final Object failreason = meta.getFieldValue("failreason_s"); - if (httpstatus != 200 && failreason != null && failreason.toString().length() > 0) { - if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (known error, httpstatus=" + httpstatus + ") from peer " + otherPeerName); - errorURLs.append(id).append(','); - blocked++; - continue; + if (blockErrors) { + try { + final URIMetadataNode meta = sb.index.fulltext().getMetadata(ASCII.getBytes(id)); + if (meta != null && meta.getFieldValue("httpstatus_i") != null) { + final int httpstatus = (meta.getFieldValue("httpstatus_i") instanceof Integer) ? + (Integer) meta.getFieldValue("httpstatus_i") : + Integer.parseInt(meta.getFieldValue("httpstatus_i").toString()); + final Object failreason = meta.getFieldValue("failreason_s"); + + if (httpstatus != 200 && failreason != null && failreason.toString().length() > 0) { + boolean shouldBlock = false; + + // Permanent errors (404, 410) - always block + if (permanentStatus.contains(httpstatus)) { + shouldBlock = true; + if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (permanent error, httpstatus=" + httpstatus + ") from peer " + otherPeerName); + } else { + // Temporary errors - check age + final Object loadDate = meta.getFieldValue("load_date_dt"); + if (loadDate != null) { + try { + final Date errorDate = (loadDate instanceof Date) ? (Date) loadDate : + new Date(Long.parseLong(loadDate.toString())); + final long errorAge = now - errorDate.getTime(); + + if (errorAge < retryAfterMillis) { + shouldBlock = true; + if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (temporary error, httpstatus=" + httpstatus + ", age=" + (errorAge / (24*60*60*1000)) + " days) from peer " + otherPeerName); + } else { + if (Network.log.isFine()) Network.log.fine("transferURL: allowing retry of URL hash '" + id + "' (error age=" + (errorAge / (24*60*60*1000)) + " days exceeds retry threshold) from peer " + otherPeerName); + } + } catch (Exception e) { + // If we can't parse the date, treat as permanent error to be safe + shouldBlock = true; + } + } else { + // No load_date available - treat as permanent error + shouldBlock = true; + } + } + + if (shouldBlock) { + errorURLs.append(id).append(','); + blocked++; + blockedErrors++; + continue; + } + } } + } catch (final Exception e) { + // Ignore errors during error status check } - } catch (final Exception e) { - // Ignore errors during error status check } } @@ -190,8 +240,8 @@ public final class transferURL { sb.peers.mySeed().incRU(received); // return rewrite properties - Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs"); - EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " URLs from peer " + otherPeerName, "", otherPeer.hash)); + Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " (error " + blockedErrors + ") URLs, reporting " + blockedErrors + " error URLs"); + EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " (error " + blockedErrors + ") URLs, reporting " + blockedErrors + " error URLs from peer " + otherPeerName, "", otherPeer.hash)); if (sb.getConfigBool(SwitchboardConstants.DECORATION_AUDIO, false)) Audio.Soundclip.dhtin.play(-10.0f); if (doublecheck > 0) { diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 91cf24484..3873e15eb 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -207,6 +207,9 @@ public final class SwitchboardConstants { public static final String INDEX_RECEIVE_ALLOW = "allowReceiveIndex"; public static final String INDEX_RECEIVE_ALLOW_SEARCH = "allowReceiveIndex.search"; public static final String INDEX_RECEIVE_BLOCK_BLACKLIST = "indexReceiveBlockBlacklist"; + public static final String INDEX_RECEIVE_BLOCK_ERRORS = "indexReceiveBlockErrors"; + public static final String INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS = "indexReceiveBlockErrors.retryAfterDays"; + public static final String INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT = "indexReceiveBlockErrors.permanentStatus"; /** *

public static final String INDEX_DIST_ALLOW_WHILE_CRAWLING = "allowDistributeIndexWhileCrawling"

-- cgit v1.2.3 From 2744918f9fa10885d01fe801f5d4c88da372ef94 Mon Sep 17 00:00:00 2001 From: pr0vieh Date: Wed, 21 Jan 2026 23:46:43 +0100 Subject: Add DNS/network errors (-1) to permanent error status codes - DNS errors (NXDOMAIN, SERVFAIL, UnknownHostException) now treated as permanent failures - Updated default permanentStatus from '404,410' to '404,410,-1' - Added documentation explaining -1 status code represents DNS/network failures - Updated UI description in IndexFederated_p.html to reflect DNS error handling - Affects both transferURL and transferRWI DHT operations --- defaults/yacy.init | 5 +++-- htroot/IndexFederated_p.html | 2 +- source/net/yacy/htroot/IndexFederated_p.java | 2 +- source/net/yacy/htroot/yacy/transferRWI.java | 2 +- source/net/yacy/htroot/yacy/transferURL.java | 2 +- source/net/yacy/search/SwitchboardConstants.java | 1 + 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index 0d2fc0dc3..810f68614 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -587,8 +587,9 @@ indexReceiveBlockErrors=true indexReceiveBlockErrors.retryAfterDays=30 # Comma-separated list of HTTP status codes considered permanent errors (always block) -# Default: 404 (Not Found), 410 (Gone) -indexReceiveBlockErrors.permanentStatus=404,410 +# -1 = DNS/Network errors (UnknownHost, connection failures) +# 404 = Not Found, 410 = Gone +indexReceiveBlockErrors.permanentStatus=404,410,-1 # the frequency is the number of links per minute, that the peer allowes # _every_ other peer to send to this peer diff --git a/htroot/IndexFederated_p.html b/htroot/IndexFederated_p.html index 652899a48..c50c95137 100644 --- a/htroot/IndexFederated_p.html +++ b/htroot/IndexFederated_p.html @@ -125,7 +125,7 @@
for temporary errors; permanent errors stay blocked.
Permanent error statuses
-
comma-separated (default: 404,410)
+
comma-separated (default: 404,410,-1; -1=DNS/network errors)
diff --git a/source/net/yacy/htroot/IndexFederated_p.java b/source/net/yacy/htroot/IndexFederated_p.java index 9b7e8f74c..4acf5d996 100644 --- a/source/net/yacy/htroot/IndexFederated_p.java +++ b/source/net/yacy/htroot/IndexFederated_p.java @@ -72,7 +72,7 @@ public class IndexFederated_p { final int retryDays = post.getInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30); env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, retryDays); - final String permanent = post.get(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410"); + final String permanent = post.get(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410,-1"); env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, permanent); } diff --git a/source/net/yacy/htroot/yacy/transferRWI.java b/source/net/yacy/htroot/yacy/transferRWI.java index d02875334..38c0bbbc4 100644 --- a/source/net/yacy/htroot/yacy/transferRWI.java +++ b/source/net/yacy/htroot/yacy/transferRWI.java @@ -246,7 +246,7 @@ public final class transferRWI { // Get configuration final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30); - final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410"); + final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410,-1"); final Set permanentStatus = new HashSet(); for (String s : permanentStatusStr.split(",")) { try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {} diff --git a/source/net/yacy/htroot/yacy/transferURL.java b/source/net/yacy/htroot/yacy/transferURL.java index 2ca3f533a..07af06601 100644 --- a/source/net/yacy/htroot/yacy/transferURL.java +++ b/source/net/yacy/htroot/yacy/transferURL.java @@ -154,7 +154,7 @@ public final class transferURL { doublecheck = 0; final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true); final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30); - final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410"); + final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410,-1"); final Set permanentStatus = new HashSet(); for (String s : permanentStatusStr.split(",")) { try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {} diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 3873e15eb..0e391e8ac 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -209,6 +209,7 @@ public final class SwitchboardConstants { public static final String INDEX_RECEIVE_BLOCK_BLACKLIST = "indexReceiveBlockBlacklist"; public static final String INDEX_RECEIVE_BLOCK_ERRORS = "indexReceiveBlockErrors"; public static final String INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS = "indexReceiveBlockErrors.retryAfterDays"; + /** Permanent error HTTP status codes (comma-separated). Default: 404,410,-1 (-1=DNS/network errors) */ public static final String INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT = "indexReceiveBlockErrors.permanentStatus"; /** -- cgit v1.2.3 From be15c1a90f58a586e75ab0a17ea5e8877b2cbe87 Mon Sep 17 00:00:00 2001 From: pr0vieh Date: Wed, 21 Jan 2026 23:57:51 +0100 Subject: Revert performance defaults in yacy.init --- defaults/yacy.init | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/defaults/yacy.init b/defaults/yacy.init index 810f68614..dd0db4496 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -295,7 +295,7 @@ proxyCacheSize = 4096 # The compression level for cached content # Supported values ranging from 0 - no compression (lower CPU, higher disk usage), to 9 - best compression (higher CPU, lower disk use) -proxyCache.compressionLevel = 6 +proxyCache.compressionLevel = 9 # Timeout value (in milliseconds) for acquiring a synchronization lock on getContent/store Cache operations # When timeout occurs, loader should fall back to regular remote resource loading @@ -723,7 +723,7 @@ recrawlindex_memprereq=1048576 40_peerseedcycle_memprereq=4194304 40_peerseedcycle_loadprereq=2.0 50_localcrawl_idlesleep=2000 -50_localcrawl_busysleep=5 +50_localcrawl_busysleep=10 50_localcrawl_memprereq=25165824 50_localcrawl_loadprereq=8.0 50_localcrawl_isPaused=false @@ -731,13 +731,13 @@ recrawlindex_memprereq=1048576 55_autocrawl_busysleep=10000 55_autocrawl_memprereq=25165824 55_autocrawl_loadprereq=8.0 -60_remotecrawlloader_idlesleep=2000 -60_remotecrawlloader_busysleep=200 +60_remotecrawlloader_idlesleep=4000 +60_remotecrawlloader_busysleep=800 60_remotecrawlloader_memprereq=12582912 60_remotecrawlloader_loadprereq=8.0 60_remotecrawlloader_isPaused=false -62_remotetriggeredcrawl_idlesleep=1000 -62_remotetriggeredcrawl_busysleep=50 +62_remotetriggeredcrawl_idlesleep=2000 +62_remotetriggeredcrawl_busysleep=200 62_remotetriggeredcrawl_memprereq=12582912 62_remotetriggeredcrawl_loadprereq=8.0 62_remotetriggeredcrawl_isPaused=false @@ -773,7 +773,7 @@ reindexSolr_loadprereq=16.0 # is used to flush the RAM cache, which is the major part of the IO in YaCy performanceProfile=defaults/yacy.init performanceSpeed=100 -performanceIO=15 +performanceIO=10 # cleanup-process: # properties for tasks that are performed during cleanup @@ -814,13 +814,7 @@ javastart_priority=10 # wordCacheMaxLow/High is the number of word indexes that shall be held in the # ram cache during indexing. If you want to increase indexing speed, increase this # value i.e. up to one million, but increase also the memory limit to a minimum of 2GB -wordCacheMaxCount = 50000 - -# Maximum number of references per term in the RWI index -# If a term has more references than this value, the oldest references will be removed -# This prevents memory issues with high-frequency terms (stopwords like 'the', 'and', etc.) -# 0 = disabled (no shrinking), 10000 = recommended for standard installations -index.maxReferences = 10000 +wordCacheMaxCount = 20000 # Specifies if yacy can be used as transparent http proxy. # @@ -1083,7 +1077,7 @@ indexDistribution.maxChunkFails = 1 # limit of references per term & blob to the younges of this value # a value of <= 0 disables this feature (no limit) # a value of e.g. 100000 can improve stability and reduce load while searching very popular words -index.maxReferences = 10000 +index.maxReferences = 0 # Search sequence settings # collection: -- cgit v1.2.3