Add configurable DHT error URL blocking with retry window and web UI

- Implement proactive DHT error URL rejection for both URL and RWI transfers
- Add configurable opt-out via indexReceiveBlockErrors setting (default: true)
- Introduce retry window for temporary errors (default: 30 days)
- Permanent errors (404, 410) always blocked, configurable via permanentStatus
- Add web UI controls in IndexFederated_p.html under Peer-to-Peer section
- Bidirectional feedback: receivers reject and report error URLs to senders
- Detailed logging shows blocked vs error-blocked counts separately
- Uses load_date_dt field to calculate error age for retry decisions

4 files changed, 166 insertions, 20 deletions

diff --git a/source/net/yacy/htroot/IndexFederated_p.java b/source/net/yacy/htroot/IndexFederated_p.java
index bbaba2b3a..9b7e8f74c 100644
--- a/source/net/yacy/htroot/IndexFederated_p.java
+++ b/source/net/yacy/htroot/IndexFederated_p.java
@@ -64,6 +64,16 @@ public class IndexFederated_p {
                 final long fileSizeMax = (OS.isWindows) ? sb.getConfigLong("filesize.max.win", Integer.MAX_VALUE) : sb.getConfigLong( "filesize.max.other", Integer.MAX_VALUE);
                 sb.index.connectRWI(wordCacheMaxCount, fileSizeMax);
             } catch (final IOException e) { ConcurrentLog.logException(e); } // switch on
+
+            // DHT error URL blocking settings
+            final boolean blockErrors = post.getBoolean(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS);
+            env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, blockErrors);
+
+            final int retryDays = post.getInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30);
+            env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, retryDays);
+
+            final String permanent = post.get(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410");
+            env.setConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, permanent);
         }
 
         if (post != null && post.containsKey("setcitation")) {
@@ -222,6 +232,11 @@ public class IndexFederated_p {
         prop.put("core.service.rwi.checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_RWI, false) ? 1 : 0);
         prop.put(SwitchboardConstants.CORE_SERVICE_CITATION + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_CITATION, false) ? 1 : 0);
         prop.put(SwitchboardConstants.CORE_SERVICE_WEBGRAPH + ".checked", env.getConfigBool(SwitchboardConstants.CORE_SERVICE_WEBGRAPH, false) ? 1 : 0);
+
+                // DHT error URL blocking settings
+                prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS + ".checked", env.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true) ? 1 : 0);
+                prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, env.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30));
+                prop.put(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, env.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410"));
 		prop.put("solr.indexing.solrremote.checked",
 				env.getConfigBool(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED,
 						SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_ENABLED_DEFAULT) ? 1 : 0);
diff --git a/source/net/yacy/htroot/yacy/transferRWI.java b/source/net/yacy/htroot/yacy/transferRWI.java
index 0981ff70e..d02875334 100644
--- a/source/net/yacy/htroot/yacy/transferRWI.java
+++ b/source/net/yacy/htroot/yacy/transferRWI.java
@@ -28,11 +28,15 @@
 

 package net.yacy.htroot.yacy;

 

+import java.io.IOException;

 import java.util.ArrayList;

+import java.util.Date;

 import java.util.HashSet;

 import java.util.Iterator;

 import java.util.Set;

 

+import org.apache.solr.common.SolrDocument;

+

 import net.yacy.cora.document.encoding.ASCII;

 import net.yacy.cora.document.encoding.UTF8;

 import net.yacy.cora.document.feed.RSSMessage;

@@ -55,6 +59,7 @@ import net.yacy.peers.Seed;
 import net.yacy.repository.Blacklist.BlacklistType;

 import net.yacy.search.Switchboard;

 import net.yacy.search.SwitchboardConstants;

+import net.yacy.search.schema.CollectionSchema;

 import net.yacy.server.serverObjects;

 import net.yacy.server.serverSwitch;

 

@@ -75,6 +80,7 @@ public final class transferRWI {
         prop.put("unknownURL", "");

         prop.put("pause", 60000);

         String result = "";

+        final StringBuilder errorURLs = new StringBuilder(4000);

         if ((post == null) || (env == null)) {

             result = "post or env is null!";

             logWarning(contentType, result);

@@ -127,7 +133,6 @@ public final class transferRWI {
         int pause = 0;

         result = "ok";

         final StringBuilder unknownURLs = new StringBuilder(6000);

-

         final double load = Memory.getSystemLoadAverage();

         final float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f);

         if (load > maxload) {

@@ -188,6 +193,7 @@ public final class transferRWI {
             final ArrayList<String> wordhashes = new ArrayList<String>();

             int received = 0;

             int blocked = 0;

+            int blockedErrors = 0;

             int count = 0;

             final Set<String> testids = new HashSet<String>();

             while (it.hasNext()) {

@@ -222,6 +228,75 @@ public final class transferRWI {
                     continue;

                 }

 

+                // reject RWI entries for URLs we already know are broken

+                final String urlHashStr = ASCII.String(urlHash);

+                final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true);

+                if (blockErrors && sb.index.fulltext().exists(urlHashStr)) {

+                    try {

+                        final SolrDocument errorCheck = sb.index.fulltext().getDefaultConnector().getDocumentById(urlHashStr,

+                                CollectionSchema.httpstatus_i.getSolrFieldName(), CollectionSchema.failreason_s.getSolrFieldName(),

+                                CollectionSchema.load_date_dt.getSolrFieldName());

+                        if (errorCheck != null) {

+                            final Object httpstatus = errorCheck.getFieldValue(CollectionSchema.httpstatus_i.getSolrFieldName());

+                            final Object failreason = errorCheck.getFieldValue(CollectionSchema.failreason_s.getSolrFieldName());

+                            if (httpstatus != null && failreason != null && failreason.toString().length() > 0) {

+                                int hs = (httpstatus instanceof Integer) ? (Integer) httpstatus : Integer.parseInt(httpstatus.toString());

+                                if (hs != 200) {

+                                    boolean shouldBlock = false;

+                                    

+                                    // Get configuration

+                                    final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30);

+                                    final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410");

+                                    final Set<Integer> permanentStatus = new HashSet<Integer>();

+                                    for (String s : permanentStatusStr.split(",")) {

+                                        try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {}

+                                    }

+                                    final long retryAfterMillis = retryAfterDays * 24L * 60L * 60L * 1000L;

+                                    final long now = System.currentTimeMillis();

+                                    

+                                    // Permanent errors (404, 410) - always block

+                                    if (permanentStatus.contains(hs)) {

+                                        shouldBlock = true;

+                                        if (Network.log.isFine()) Network.log.fine("transferRWI: rejected RWI for known permanent error URL hash '" + urlHashStr + "' (httpstatus=" + hs + ") from peer " + otherPeerName);

+                                    } else {

+                                        // Temporary errors - check age

+                                        final Object loadDate = errorCheck.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName());

+                                        if (loadDate != null) {

+                                            try {

+                                                final Date errorDate = (loadDate instanceof Date) ? (Date) loadDate : 

+                                                    new Date(Long.parseLong(loadDate.toString()));

+                                                final long errorAge = now - errorDate.getTime();

+                                                

+                                                if (errorAge < retryAfterMillis) {

+                                                    shouldBlock = true;

+                                                    if (Network.log.isFine()) Network.log.fine("transferRWI: rejected RWI for known temporary error URL hash '" + urlHashStr + "' (httpstatus=" + hs + ", age=" + (errorAge / (24*60*60*1000)) + " days) from peer " + otherPeerName);

+                                                } else {

+                                                    if (Network.log.isFine()) Network.log.fine("transferRWI: allowing retry for URL hash '" + urlHashStr + "' (error age=" + (errorAge / (24*60*60*1000)) + " days exceeds retry threshold) from peer " + otherPeerName);

+                                                }

+                                            } catch (Exception e) {

+                                                // If we can't parse the date, treat as permanent error to be safe

+                                                shouldBlock = true;

+                                            }

+                                        } else {

+                                            // No load_date available - treat as permanent error

+                                            shouldBlock = true;

+                                        }

+                                    }

+                                    

+                                    if (shouldBlock) {

+                                        errorURLs.append(urlHashStr).append(',');

+                                        blocked++;

+                                        blockedErrors++;

+                                        continue;

+                                    }

+                                }

+                            }

+                        }

+                    } catch (final IOException e) {

+                        // ignore Solr errors during error URL check

+                    }

+                }

+

                 // learn entry

                 try {

                     sb.index.storeRWI(ASCII.getBytes(wordHash), iEntry);

@@ -251,14 +326,16 @@ public final class transferRWI {
                 unknownURLs.append(UTF8.String(bit.next())).append(',');

             }

             if (unknownURLs.length() > 0) { unknownURLs.setLength(unknownURLs.length() - 1); }

+            if (errorURLs.length() > 0) { errorURLs.setLength(errorURLs.length() - 1); }

+            

             if (wordhashes.isEmpty() || received == 0) {

-                sb.getLog().info("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");

+                sb.getLog().info("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs, reporting " + blockedErrors + " error URLs");

             } else {

                 final String firstHash = wordhashes.get(0);

                 final String lastHash = wordhashes.get(wordhashes.size() - 1);

                 final long avdist = (Distribution.horizontalDHTDistance(firstHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash)) + Distribution.horizontalDHTDistance(lastHash.getBytes(), ASCII.getBytes(sb.peers.mySeed().hash))) / 2;

-                sb.getLog().info("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + received+ " URLs from " + otherPeerName);

-                EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + ", requesting " + unknownURL.size() + "/" + received + " URLs from " + otherPeerName, "", otherPeer.hash));

+                sb.getLog().info("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + " (error " + blockedErrors + "), requesting " + unknownURL.size() + "/" + received+ " URLs, reporting " + blockedErrors + " error URLs from " + otherPeerName);

+                EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs, " + wordc + " Words [" + firstHash + " .. " + lastHash + "], processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, " + avdist + ", blocked " + blocked + " (error " + blockedErrors + "), requesting " + unknownURL.size() + "/" + received + " URLs, reporting " + blockedErrors + " error URLs from " + otherPeerName, "", otherPeer.hash));

             }

             result = "ok";

 

@@ -266,6 +343,7 @@ public final class transferRWI {
         }

 

         prop.put("unknownURL", unknownURLs.toString());

+        prop.put("errorURL", errorURLs.toString());

         prop.put("result", result);

         prop.put("pause", pause);

 

diff --git a/source/net/yacy/htroot/yacy/transferURL.java b/source/net/yacy/htroot/yacy/transferURL.java
index f9b95e270..2ca3f533a 100644
--- a/source/net/yacy/htroot/yacy/transferURL.java
+++ b/source/net/yacy/htroot/yacy/transferURL.java
@@ -30,8 +30,11 @@ package net.yacy.htroot.yacy;
 

 import java.io.IOException;

 import java.text.ParseException;

+import java.util.Date;

 import java.util.HashMap;

+import java.util.HashSet;

 import java.util.Map;

+import java.util.Set;

 

 import net.yacy.cora.date.GenericFormatter;

 import net.yacy.cora.document.encoding.ASCII;

@@ -90,6 +93,7 @@ public final class transferURL {
         } else {

             int received = 0;

             int blocked = 0;

+            int blockedErrors = 0;

             int doublecheck = 0;

             // read the urls from the other properties and store

             String urls;

@@ -148,26 +152,72 @@ public final class transferURL {
             }

 

             doublecheck = 0;

+            final boolean blockErrors = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS, true);

+            final int retryAfterDays = sb.getConfigInt(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS, 30);

+            final String permanentStatusStr = sb.getConfig(SwitchboardConstants.INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT, "404,410");

+            final Set<Integer> permanentStatus = new HashSet<Integer>();

+            for (String s : permanentStatusStr.split(",")) {

+                try { permanentStatus.add(Integer.parseInt(s.trim())); } catch (NumberFormatException e) {}

+            }

+            final long retryAfterMillis = retryAfterDays * 24L * 60L * 60L * 1000L;

+            final long now = System.currentTimeMillis();

+            

             for (final String id : lEm.keySet()) {

                 if (sb.index.exists(id)) {

                     doublecheck++;

                     // Check if entry we already have is marked as error - if so, reject incoming replacement

-                    try {

-                        final URIMetadataNode meta = sb.index.fulltext().getMetadata(ASCII.getBytes(id));

-                        if (meta != null && meta.getFieldValue("httpstatus_i") != null) {

-                            final int httpstatus = (meta.getFieldValue("httpstatus_i") instanceof Integer) ? 

-                                (Integer) meta.getFieldValue("httpstatus_i") : 

-                                Integer.parseInt(meta.getFieldValue("httpstatus_i").toString());

-                            final Object failreason = meta.getFieldValue("failreason_s");

-                            if (httpstatus != 200 && failreason != null && failreason.toString().length() > 0) {

-                                if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (known error, httpstatus=" + httpstatus + ") from peer " + otherPeerName);

-                                errorURLs.append(id).append(',');

-                                blocked++;

-                                continue;

+                    if (blockErrors) {

+                        try {

+                            final URIMetadataNode meta = sb.index.fulltext().getMetadata(ASCII.getBytes(id));

+                            if (meta != null && meta.getFieldValue("httpstatus_i") != null) {

+                                final int httpstatus = (meta.getFieldValue("httpstatus_i") instanceof Integer) ? 

+                                    (Integer) meta.getFieldValue("httpstatus_i") : 

+                                    Integer.parseInt(meta.getFieldValue("httpstatus_i").toString());

+                                final Object failreason = meta.getFieldValue("failreason_s");

+                                

+                                if (httpstatus != 200 && failreason != null && failreason.toString().length() > 0) {

+                                    boolean shouldBlock = false;

+                                    

+                                    // Permanent errors (404, 410) - always block

+                                    if (permanentStatus.contains(httpstatus)) {

+                                        shouldBlock = true;

+                                        if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (permanent error, httpstatus=" + httpstatus + ") from peer " + otherPeerName);

+                                    } else {

+                                        // Temporary errors - check age

+                                        final Object loadDate = meta.getFieldValue("load_date_dt");

+                                        if (loadDate != null) {

+                                            try {

+                                                final Date errorDate = (loadDate instanceof Date) ? (Date) loadDate : 

+                                                    new Date(Long.parseLong(loadDate.toString()));

+                                                final long errorAge = now - errorDate.getTime();

+                                                

+                                                if (errorAge < retryAfterMillis) {

+                                                    shouldBlock = true;

+                                                    if (Network.log.isFine()) Network.log.fine("transferURL: rejected URL hash '" + id + "' (temporary error, httpstatus=" + httpstatus + ", age=" + (errorAge / (24*60*60*1000)) + " days) from peer " + otherPeerName);

+                                                } else {

+                                                    if (Network.log.isFine()) Network.log.fine("transferURL: allowing retry of URL hash '" + id + "' (error age=" + (errorAge / (24*60*60*1000)) + " days exceeds retry threshold) from peer " + otherPeerName);

+                                                }

+                                            } catch (Exception e) {

+                                                // If we can't parse the date, treat as permanent error to be safe

+                                                shouldBlock = true;

+                                            }

+                                        } else {

+                                            // No load_date available - treat as permanent error

+                                            shouldBlock = true;

+                                        }

+                                    }

+                                    

+                                    if (shouldBlock) {

+                                        errorURLs.append(id).append(',');

+                                        blocked++;

+                                        blockedErrors++;

+                                        continue;

+                                    }

+                                }

                             }

+                        } catch (final Exception e) {

+                            // Ignore errors during error status check

                         }

-                    } catch (final Exception e) {

-                        // Ignore errors during error status check

                     }

                 }

                 

@@ -190,8 +240,8 @@ public final class transferURL {
             sb.peers.mySeed().incRU(received);

 

             // return rewrite properties

-            Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs");

-            EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " URLs from peer " + otherPeerName, "", otherPeer.hash));

+            Network.log.info("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " (error " + blockedErrors + ") URLs, reporting " + blockedErrors + " error URLs");

+            EventChannel.channels(EventChannel.DHTRECEIVE).addMessage(new RSSMessage("Received " + received + ", blocked " + blocked + " (error " + blockedErrors + ") URLs, reporting " + blockedErrors + " error URLs from peer " + otherPeerName, "", otherPeer.hash));

             if (sb.getConfigBool(SwitchboardConstants.DECORATION_AUDIO, false)) Audio.Soundclip.dhtin.play(-10.0f);

 

             if (doublecheck > 0) {

diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java
index 91cf24484..3873e15eb 100644
--- a/source/net/yacy/search/SwitchboardConstants.java
+++ b/source/net/yacy/search/SwitchboardConstants.java
@@ -207,6 +207,9 @@ public final class SwitchboardConstants {
     public static final String INDEX_RECEIVE_ALLOW              = "allowReceiveIndex";
     public static final String INDEX_RECEIVE_ALLOW_SEARCH       = "allowReceiveIndex.search";
     public static final String INDEX_RECEIVE_BLOCK_BLACKLIST    = "indexReceiveBlockBlacklist";
+    public static final String INDEX_RECEIVE_BLOCK_ERRORS       = "indexReceiveBlockErrors";
+    public static final String INDEX_RECEIVE_BLOCK_ERRORS_RETRY_DAYS = "indexReceiveBlockErrors.retryAfterDays";
+    public static final String INDEX_RECEIVE_BLOCK_ERRORS_PERMANENT  = "indexReceiveBlockErrors.permanentStatus";
 
     /**
      * <p><code>public static final String <strong>INDEX_DIST_ALLOW_WHILE_CRAWLING</strong> = "allowDistributeIndexWhileCrawling"</code></p>