Made /yacysearch access rate limitations user configurable

With a new admin page at /SearchAccessRate_p.html in menu Network Access
> Local Search > Access Rate Limitations

6 files changed, 308 insertions, 89 deletions

diff --git a/htroot/ConfigPortal_p.java b/htroot/ConfigPortal_p.java
index 7b6b96c6f..8b1af4c2a 100644
--- a/htroot/ConfigPortal_p.java
+++ b/htroot/ConfigPortal_p.java
@@ -25,10 +25,6 @@
 // along with this program; if not, write to the Free Software
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
 import java.net.MalformedURLException;
 import java.time.Duration;
 import java.util.Properties;
@@ -131,27 +127,7 @@ public class ConfigPortal_p {
             }
             if (post.containsKey("searchpage_default")) {
                 // load defaults from defaults/yacy.init file
-                final Properties config = new Properties();
-                final String mes = "ConfigPortal";
-                FileInputStream fis = null;
-                try {
-                    fis = new FileInputStream(new File(sb.appPath, "defaults/yacy.init"));
-                    config.load(fis);
-                } catch (final FileNotFoundException e) {
-                    ConcurrentLog.severe(mes, "could not find configuration file.");
-                    return prop;
-                } catch (final IOException e) {
-                    ConcurrentLog.severe(mes, "could not read configuration file.");
-                    return prop;
-                } finally {
-                    if (fis != null) {
-                        try {
-                            fis.close();
-                        } catch (final IOException e) {
-                            ConcurrentLog.logException(e);
-                        }
-                    }
-                }
+                final Properties config = sb.loadDefaultConfig();
                 sb.setConfig(SwitchboardConstants.GREETING, config.getProperty(SwitchboardConstants.GREETING,"P2P Web Search"));
                 sb.setConfig(SwitchboardConstants.GREETING_HOMEPAGE, config.getProperty(SwitchboardConstants.GREETING_HOMEPAGE,"https://yacy.net"));
                 sb.setConfig(SwitchboardConstants.GREETING_LARGE_IMAGE, config.getProperty(SwitchboardConstants.GREETING_LARGE_IMAGE,"env/grafics/YaCyLogo_120ppi.png"));
diff --git a/htroot/ConfigSearchPage_p.java b/htroot/ConfigSearchPage_p.java
index d3d0716b8..be7e58660 100644
--- a/htroot/ConfigSearchPage_p.java
+++ b/htroot/ConfigSearchPage_p.java
@@ -25,10 +25,6 @@
 // along with this program; if not, write to the Free Software
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
 import java.sql.Date;
 import java.util.HashSet;
 import java.util.Map;
@@ -37,7 +33,6 @@ import java.util.Set;
 
 import net.yacy.cora.date.GenericFormatter;
 import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.cora.util.ConcurrentLog;
 import net.yacy.data.TransactionManager;
 import net.yacy.data.WorkTables;
 import net.yacy.search.Switchboard;
@@ -163,27 +158,7 @@ public class ConfigSearchPage_p {
 
             if (post.containsKey("searchpage_default")) {
                 // load defaults from defaults/yacy.init file
-                final Properties config = new Properties();
-                final String mes = "ConfigSearchPage_p";
-                FileInputStream fis = null;
-                try {
-                    fis = new FileInputStream(new File(sb.appPath, "defaults/yacy.init"));
-                    config.load(fis);
-                } catch (final FileNotFoundException e) {
-                    ConcurrentLog.severe(mes, "could not find configuration file.");
-                    return prop;
-                } catch (final IOException e) {
-                    ConcurrentLog.severe(mes, "could not read configuration file.");
-                    return prop;
-                } finally {
-                    if (fis != null) {
-                        try {
-                            fis.close();
-                        } catch (final IOException e) {
-                            ConcurrentLog.logException(e);
-                        }
-                    }
-                }
+                final Properties config = sb.loadDefaultConfig();
                 sb.setConfig("publicTopmenu", config.getProperty("publicTopmenu","true"));
 				sb.setConfig(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN,
 						config.getProperty(SwitchboardConstants.SEARCH_PUBLIC_TOP_NAV_BAR_LOGIN,
diff --git a/htroot/SearchAccessRate_p.html b/htroot/SearchAccessRate_p.html
new file mode 100644
index 000000000..f2f43f590
--- /dev/null
+++ b/htroot/SearchAccessRate_p.html
@@ -0,0 +1,158 @@
+<!DOCTYPE html>

+<html lang="en">

+<head>

+<title>YaCy '#[clientname]#': Local Search access rate

+	limitations</title> #%env/templates/metas.template%#

+</head>

+<body id="Settings">

+	#%env/templates/header.template%#

+	#%env/templates/submenuAccessTracker.template%#

+	<h2>Local Search access rate limitations</h2>

+	<p>

+		You can configure here limitations on access rate to this peer search

+		interface by unauthenticated users and users without extended search

+		right (see the <a href="ConfigAccounts_p.html">Accounts</a>

+		configuration page for details on users rights).

+	</p>

+

+	<form action="SearchAccessRate_p.html" method="post"

+		class="form-horizontal">

+		<input type="hidden" name="transactionToken"

+			value="#[transactionToken]#" />

+		<fieldset>

+			<legend>YaCy search</legend>

+

+			<p>Access rate limitations to this peer search interface. When a

+				user with limited rights (unauthenticated or without extended search

+				right) exceeds a limit, the search is blocked.</p>

+

+			<div class="form-group">

+				<label class="control-label col-sm-6 col-md-4 col-lg-3"

+					for="search.public.max.access.3s">Max searches in 3s</label>

+				<div class="col-sm-3 col-md-2">

+					<input class="form-control" id="search.public.max.access.3s"

+						name="search.public.max.access.3s" type="number"

+						value="#[search.public.max.access.3s]#" min="0" max="2147483647" />

+				</div>

+			</div>

+			<div class="form-group">

+				<label class="control-label col-sm-6 col-md-4 col-lg-3"

+					for="search.public.max.access.1mn">Max searches in 1mn</label>

+				<div class="col-sm-3 col-md-2">

+					<input class="form-control" id="search.public.max.access.1mn"

+						name="search.public.max.access.1mn" type="number"

+						value="#[search.public.max.access.1mn]#" min="0" max="2147483647" />

+				</div>

+			</div>

+			<div class="form-group">

+				<label class="control-label col-sm-6 col-md-4 col-lg-3"

+					for="search.public.max.access.10mn">Max searches in 10mn</label>

+				<div class="col-sm-3 col-md-2">

+					<input class="form-control" id="search.public.max.access.10mn"

+						name="search.public.max.access.10mn" type="number"

+						value="#[search.public.max.access.10mn]#" min="0" max="2147483647" />

+				</div>

+			</div>

+		</fieldset>

+

+		<fieldset>

+			<legend>Peer-to-peer search</legend>

+

+			<p>Access rate limitations to the peer-to-peer search mode. When

+				a user with limited rights (unauthenticated or without extended

+				search right) exceeds a limit, the search scope falls back to only

+				this local peer index.</p>

+

+			<div class="form-group">

+				<label class="control-label col-sm-6 col-md-4 col-lg-3"

+					for="search.public.max.p2p.access.3s">Max searches in 3s</label>

+				<div class="col-sm-3 col-md-2">

+					<input class="form-control" id="search.public.max.p2p.access.3s"

+						name="search.public.max.p2p.access.3s" type="number"

+						value="#[search.public.max.p2p.access.3s]#" min="0"

+						max="2147483647" />

+				</div>

+			</div>

+			<div class="form-group">

+				<label class="control-label col-sm-6 col-md-4 col-lg-3"

+					for="search.public.max.p2p.access.1mn">Max searches in 1mn</label>

+				<div class="col-sm-3 col-md-2">

+					<input class="form-control" id="search.public.max.p2p.access.1mn"

+						name="search.public.max.p2p.access.1mn" type="number"

+						value="#[search.public.max.p2p.access.1mn]#" min="0"

+						max="2147483647" />

+				</div>

+			</div>

+			<div class="form-group">

+				<label class="control-label col-sm-6 col-md-4 col-lg-3"

+					for="search.public.max.p2p.access.10mn">Max searches in 10mn</label>

+				<div class="col-sm-3 col-md-2">

+					<input class="form-control" id="search.public.max.p2p.access.10mn"

+						name="search.public.max.p2p.access.10mn" type="number"

+						value="#[search.public.max.p2p.access.10mn]#" min="0"

+						max="2147483647" />

+				</div>

+			</div>

+		</fieldset>

+

+		<fieldset>

+			<legend>Remote snippet load</legend>

+

+			<p>Limitations on snippet loading from remote websites. When a

+				user with limited rights (unauthenticated or without extended search

+				right) exceeds a limit, the snippets fetch strategy falls back to

+				'CACHEONLY' (check the default Snippet Fetch Strategy on the <a href="ConfigPortal_p.html">Search Portal</a> configuration page).</p>

+

+			<div class="form-group">

+				<label class="control-label col-sm-6 col-md-4 col-lg-3"

+					for="search.public.max.remoteSnippet.access.3s">Max

+					searches in 3s</label>

+				<div class="col-sm-3 col-md-2">

+					<input class="form-control"

+						id="search.public.max.remoteSnippet.access.3s"

+						name="search.public.max.remoteSnippet.access.3s" type="number"

+						value="#[search.public.max.remoteSnippet.access.3s]#" min="0"

+						max="2147483647" />

+				</div>

+			</div>

+			<div class="form-group">

+				<label class="control-label col-sm-6 col-md-4 col-lg-3"

+					for="search.public.max.remoteSnippet.access.1mn">Max

+					searches in 1mn</label>

+				<div class="col-sm-3 col-md-2">

+					<input class="form-control"

+						id="search.public.max.remoteSnippet.access.1mn"

+						name="search.public.max.remoteSnippet.access.1mn" type="number"

+						value="#[search.public.max.remoteSnippet.access.1mn]#" min="0"

+						max="2147483647" />

+				</div>

+			</div>

+			<div class="form-group">

+				<label class="control-label col-sm-6 col-md-4 col-lg-3"

+					for="search.public.max.remoteSnippet.access.10mn">Max

+					searches in 10mn</label>

+				<div class="col-sm-3 col-md-2">

+					<input class="form-control"

+						id="search.public.max.remoteSnippet.access.10mn"

+						name="search.public.max.remoteSnippet.access.10mn" type="number"

+						value="#[search.public.max.remoteSnippet.access.10mn]#" min="0"

+						max="2147483647" />

+				</div>

+			</div>

+		</fieldset>

+

+		<div class="form-group">

+			<div class="col-xs-offset-1 col-sm-offset-3 col-md-offset-3 col-lg-offset-2">

+				<input type="submit" class="btn btn-primary" name="set"

+					value="Submit" aria-describedby="changeInfo" /> <input

+					type="submit" class="btn btn-default" name="setDefaults"

+					value="Set defaults" title="Reset to defaults settings"

+					aria-describedby="changeInfo" /> <em id="changeInfo">Changes

+					will take effect immediately.</em>

+			</div>

+		</div>

+	</form>

+

+	#%env/templates/footer.template%#

+</body>

+</html>
\ No newline at end of file
diff --git a/htroot/SearchAccessRate_p.java b/htroot/SearchAccessRate_p.java
new file mode 100644
index 000000000..62ad0853b
--- /dev/null
+++ b/htroot/SearchAccessRate_p.java
@@ -0,0 +1,85 @@
+
+// SearchAccessRate_p.java
+// Copyright 2019 by luccioman; https://github.com/luccioman
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import java.util.Properties;
+
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.data.TransactionManager;
+import net.yacy.search.SearchAccessRateConstants;
+import net.yacy.search.Switchboard;
+import net.yacy.server.serverObjects;
+import net.yacy.server.serverSwitch;
+
+/**
+ * Handle configuration of access rate limitations to the peer search interface
+ * through the SearchAccessRate_p.html page.
+ */
+public class SearchAccessRate_p {
+
+	/**
+	 * @param header the current request headers
+	 * @param post   the request parameters
+	 * @param env    holds the server environment
+	 * @return a serverObjects instance filled with the properties required by the
+	 *         related template
+	 */
+	public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
+		final Switchboard sb = (Switchboard) env;
+		final serverObjects prop = new serverObjects();
+
+		/* Acquire a transaction token for the next POST form submission */
+		prop.put(TransactionManager.TRANSACTION_TOKEN_PARAM, TransactionManager.getTransactionToken(header));
+
+		if (post != null) {
+			/*
+			 * Check the transaction is valid : validation apply then for every uses of the
+			 * post parameters
+			 */
+			TransactionManager.checkPostTransaction(header, post);
+
+			if (post.containsKey("set")) {
+				/* Setting configuration values */
+				for (final SearchAccessRateConstants config : SearchAccessRateConstants.values()) {
+					sb.setConfig(config.getKey(), Math.max(0, post.getInt(config.getKey(), config.getDefaultValue())));
+				}
+			} else if (post.containsKey("setDefaults")) {
+				/* Resetting to defaults */
+				final Properties defaultConfig = sb.loadDefaultConfig();
+
+				for (final SearchAccessRateConstants config : SearchAccessRateConstants.values()) {
+					sb.setConfig(config.getKey(),
+							defaultConfig.getProperty(config.getKey(), String.valueOf(config.getDefaultValue())));
+				}
+
+			}
+
+		}
+
+		for (final SearchAccessRateConstants config : SearchAccessRateConstants.values()) {
+			/* Fill prop for template rendering */
+			prop.put(config.getKey(), sb.getConfigInt(config.getKey(), config.getDefaultValue()));
+		}
+
+		return prop;
+	}
+
+}
diff --git a/htroot/env/templates/submenuAccessTracker.template b/htroot/env/templates/submenuAccessTracker.template
index a08d32d44..d37cc12d7 100644
--- a/htroot/env/templates/submenuAccessTracker.template
+++ b/htroot/env/templates/submenuAccessTracker.template
@@ -15,6 +15,7 @@
       <ul class="SubMenu">
         <li><a href="AccessTracker_p.html?page=2" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Log</a></li>
         <li><a href="AccessTracker_p.html?page=3" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Host Tracker</a></li>
+        <li><a href="SearchAccessRate_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Access Rate Limitations</a></li>
       </ul>
     </div>
     
diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java
index 64248e3f5..181eeed3f 100644
--- a/htroot/yacysearch.java
+++ b/htroot/yacysearch.java
@@ -78,6 +78,7 @@ import net.yacy.peers.EventChannel;
 import net.yacy.peers.NewsPool;
 import net.yacy.peers.graphics.ProfilingGraph;
 import net.yacy.search.EventTracker;
+import net.yacy.search.SearchAccessRateConstants;
 import net.yacy.search.Switchboard;
 import net.yacy.search.SwitchboardConstants;
 import net.yacy.search.index.Segment;
@@ -321,48 +322,52 @@ public class yacysearch {
                 + " gets no permission to search");
         } else if ( !extendedSearchRights && !localhostAccess && !intranetMode ) {
             // in case that we do a global search or we want to fetch snippets, we check for DoS cases
+        	final int accInThreeSeconds;
+        	final int accInOneMinute;
+        	final int accInTenMinutes;
             synchronized ( trackerHandles ) {
-                final int accInThreeSeconds =
+                accInThreeSeconds =
                     trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size();
-                final int accInOneMinute =
+                accInOneMinute =
                     trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 60000)).size();
-                final int accInTenMinutes =
+                accInTenMinutes =
                     trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 600000)).size();
-                // protections against too strong YaCy network load, reduces remote search
-                if ( global ) {
-                    if ( accInTenMinutes >= 60 || accInOneMinute >= 6 || accInThreeSeconds >= 1 ) {
-                        global = false;
-                        ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM "
-                            + client
-                            + ": "
-                            + accInThreeSeconds
-                            + "/3s, "
-                            + accInOneMinute
-                            + "/60s, "
-                            + accInTenMinutes
-                            + "/600s, "
-                            + " requests, disallowed global search");
-                    }
-                }
-                // protection against too many remote server snippet loads (protects traffic on server)
-                if ( snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ) {
-                    if ( accInTenMinutes >= 20 || accInOneMinute >= 4 || accInThreeSeconds >= 1 ) {
-                        snippetFetchStrategy = CacheStrategy.CACHEONLY;
-                        ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM "
-                            + client
-                            + ": "
-                            + accInThreeSeconds
-                            + "/3s, "
-                            + accInOneMinute
-                            + "/60s, "
-                            + accInTenMinutes
-                            + "/600s, "
-                            + " requests, disallowed remote snippet loading");
-                    }
+            }
+            // protections against too strong YaCy network load, reduces remote search
+            if ( global ) {
+				if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_10MN.getKey(),
+						SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_10MN.getDefaultValue())
+						|| accInOneMinute >= sb.getConfigInt(
+								SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_1MN.getKey(),
+								SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_1MN.getDefaultValue())
+						|| accInThreeSeconds >= sb.getConfigInt(
+								SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_3S.getKey(),
+								SearchAccessRateConstants.PUBLIC_MAX_P2P_ACCESS_3S.getDefaultValue())) {
+                    global = false;
+                    ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM "
+                        + client
+                        + ": "
+                        + accInThreeSeconds
+                        + "/3s, "
+                        + accInOneMinute
+                        + "/60s, "
+                        + accInTenMinutes
+                        + "/600s, "
+                        + " requests, disallowed global search");
                 }
-                // general load protection
-                if ( accInTenMinutes >= 3000 || accInOneMinute >= 600 || accInThreeSeconds >= 60 ) {
-                    block = true;
+            }
+            // protection against too many remote server snippet loads (protects traffic on server)
+            if ( snippetFetchStrategy != null && snippetFetchStrategy.isAllowedToFetchOnline() ) {
+				if (accInTenMinutes >= sb.getConfigInt(
+						SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_10MN.getKey(),
+						SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_10MN.getDefaultValue())
+						|| accInOneMinute >= sb.getConfigInt(
+								SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_1MN.getKey(),
+								SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_1MN.getDefaultValue())
+						|| accInThreeSeconds >= sb.getConfigInt(
+								SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_3S.getKey(),
+								SearchAccessRateConstants.PUBLIC_MAX_REMOTE_SNIPPET_ACCESS_3S.getDefaultValue())) {
+                    snippetFetchStrategy = CacheStrategy.CACHEONLY;
                     ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM "
                         + client
                         + ": "
@@ -372,9 +377,28 @@ public class yacysearch {
                         + "/60s, "
                         + accInTenMinutes
                         + "/600s, "
-                        + " requests, disallowed search");
+                        + " requests, disallowed remote snippet loading");
                 }
             }
+            // general load protection
+			if (accInTenMinutes >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getKey(),
+					SearchAccessRateConstants.PUBLIC_MAX_ACCESS_10MN.getDefaultValue())
+					|| accInOneMinute >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getKey(),
+							SearchAccessRateConstants.PUBLIC_MAX_ACCESS_1MN.getDefaultValue())
+					|| accInThreeSeconds >= sb.getConfigInt(SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getKey(),
+							SearchAccessRateConstants.PUBLIC_MAX_ACCESS_3S.getDefaultValue())) {
+                block = true;
+                ConcurrentLog.warn("LOCAL_SEARCH", "ACCESS CONTROL: CLIENT FROM "
+                    + client
+                    + ": "
+                    + accInThreeSeconds
+                    + "/3s, "
+                    + accInOneMinute
+                    + "/60s, "
+                    + accInTenMinutes
+                    + "/600s, "
+                    + " requests, disallowed search");
+            }
         }
 
         if ( !block ) {