1 files changed, 49 insertions, 30 deletions
diff --git a/defaults/yacy.init b/defaults/yacy.init
index 2274d4d59..0d2fc0dc3 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -295,7 +295,7 @@ proxyCacheSize = 4096
 
 # The compression level for cached content
 # Supported values ranging from 0 - no compression (lower CPU, higher disk usage), to 9 - best compression (higher CPU, lower disk use)
-proxyCache.compressionLevel = 9
+proxyCache.compressionLevel = 6
 
 # Timeout value (in milliseconds) for acquiring a synchronization lock on getContent/store Cache operations
 # When timeout occurs, loader should fall back to regular remote resource loading
@@ -577,6 +577,19 @@ allowReceiveIndex=true
 allowReceiveIndex.search=true
 indexReceiveBlockBlacklist=true
 
+# Proactive Error-URL Rejection: block distribution of known broken URLs via DHT
+# Set to false to disable this feature (opt-out)
+indexReceiveBlockErrors=true
+
+# How many days to block error URLs before allowing retry
+# Permanent errors (404, 410) are always blocked
+# Temporary errors (5xx, timeouts) are blocked for this many days
+indexReceiveBlockErrors.retryAfterDays=30
+
+# Comma-separated list of HTTP status codes considered permanent errors (always block)
+# Default: 404 (Not Found), 410 (Gone)
+indexReceiveBlockErrors.permanentStatus=404,410
+
 # the frequency is the number of links per minute, that the peer allowes
 # _every_ other peer to send to this peer
 defaultWordReceiveFrequency=100
@@ -709,7 +722,7 @@ recrawlindex_memprereq=1048576
 40_peerseedcycle_memprereq=4194304
 40_peerseedcycle_loadprereq=2.0
 50_localcrawl_idlesleep=2000
-50_localcrawl_busysleep=10
+50_localcrawl_busysleep=5
 50_localcrawl_memprereq=25165824
 50_localcrawl_loadprereq=8.0
 50_localcrawl_isPaused=false
@@ -717,13 +730,13 @@ recrawlindex_memprereq=1048576
 55_autocrawl_busysleep=10000
 55_autocrawl_memprereq=25165824
 55_autocrawl_loadprereq=8.0
-60_remotecrawlloader_idlesleep=4000
-60_remotecrawlloader_busysleep=800
+60_remotecrawlloader_idlesleep=2000
+60_remotecrawlloader_busysleep=200
 60_remotecrawlloader_memprereq=12582912
 60_remotecrawlloader_loadprereq=8.0
 60_remotecrawlloader_isPaused=false
-62_remotetriggeredcrawl_idlesleep=2000
-62_remotetriggeredcrawl_busysleep=200
+62_remotetriggeredcrawl_idlesleep=1000
+62_remotetriggeredcrawl_busysleep=50
 62_remotetriggeredcrawl_memprereq=12582912
 62_remotetriggeredcrawl_loadprereq=8.0
 62_remotetriggeredcrawl_isPaused=false
@@ -759,7 +772,7 @@ reindexSolr_loadprereq=16.0
 # is used to flush the RAM cache, which is the major part of the IO in YaCy
 performanceProfile=defaults/yacy.init
 performanceSpeed=100
-performanceIO=10
+performanceIO=15
 
 # cleanup-process:
 # properties for tasks that are performed during cleanup
@@ -800,7 +813,13 @@ javastart_priority=10
 # wordCacheMaxLow/High is the number of word indexes that shall be held in the
 # ram cache during indexing. If you want to increase indexing speed, increase this
 # value i.e. up to one million, but increase also the memory limit to a minimum of 2GB
-wordCacheMaxCount = 20000
+wordCacheMaxCount = 50000
+
+# Maximum number of references per term in the RWI index
+# If a term has more references than this value, the oldest references will be removed
+# This prevents memory issues with high-frequency terms (stopwords like 'the', 'and', etc.)
+# 0 = disabled (no shrinking), 10000 = recommended for standard installations
+index.maxReferences = 10000
 
 # Specifies if yacy can be used as transparent http proxy.
 # 
@@ -1063,7 +1082,7 @@ indexDistribution.maxChunkFails = 1
 # limit of references per term & blob to the younges of this value
 # a value of <= 0 disables this feature (no limit)
 # a value of e.g. 100000 can improve stability and reduce load while searching very popular words
-index.maxReferences = 0
+index.maxReferences = 10000
 
 # Search sequence settings
 # collection:
@@ -1416,24 +1435,24 @@ crawler.userAgent.clienttimeout = 10000
 # experiments with timeout requests
 timeoutrequests = true
 
-# interface decorations
-decoration.audio = false
-decoration.grafics.linkstructure = true
-decoration.hostanalysis = false
-decoration.simpleheadernavbar = navbar-default
-
-# ai settings
-ai.production_models = []
-ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies.
-ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.
-ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n
-ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array: 
-ai.shield.allow-nonlocalhost = false
-ai.shield.show-chat-link = false
-ai.shield.rate.per-minute = 4
-ai.shield.rate.per-hour = 60
-ai.shield.rate.per-day = 120
-ai.shield.limit-all = false
-ai.shield.all.per-minute = 16
-ai.shield.all.per-hour = 240
-ai.shield.all.per-day = 480
+# interface decorations
+decoration.audio = false
+decoration.grafics.linkstructure = true
+decoration.hostanalysis = false
+decoration.simpleheadernavbar = navbar-default
+
+# ai settings
+ai.production_models = []
+ai.system-prompt = You are a smart and helpful chatbot. If possible, use friendly emojies.
+ai.llm-system-prefix = \n\nYou may receive additional expert knowledge in the user prompt after a 'Additional Information' headline to enhance your knowledge. Use it only if applicable.
+ai.llm-user-prefix = \n\nAdditional Information:\n\nbelow you find a collection of texts that might be useful to generate a response. Do not discuss these documents, just use them to answer the question above.\n\n
+ai.llm-query-generator-prefix = Make a list of search words with low document frequency for the following prompt; use a JSON Array: 
+ai.shield.allow-nonlocalhost = false
+ai.shield.show-chat-link = false
+ai.shield.rate.per-minute = 4
+ai.shield.rate.per-hour = 60
+ai.shield.rate.per-day = 120
+ai.shield.limit-all = false
+ai.shield.all.per-minute = 16
+ai.shield.all.per-hour = 240
+ai.shield.all.per-day = 480