added hazelcast and some modifications to align legacy YaCy with

YaCyGrid

19 files changed, 421 insertions, 98 deletions

diff --git a/.classpath b/.classpath
index 0df6df910..34756f84a 100644
--- a/.classpath
+++ b/.classpath
@@ -112,6 +112,7 @@
 	<classpathentry kind="lib" path="libt/hamcrest-library-2.2.jar"/>
 	<classpathentry kind="lib" path="lib/commons-collections4-4.4.jar"/>
 	<classpathentry kind="lib" path="lib/guava-25.1-jre.jar"/>
+	<classpathentry kind="lib" path="lib/hazelcast-4.2.jar"/>
 	<classpathentry kind="src" path="htroot/api/blacklists"/>
 	<classpathentry kind="src" path="htroot/api/bookmarks/posts"/>
 	<classpathentry kind="src" path="htroot/api/bookmarks/tags"/>
diff --git a/build.xml b/build.xml
index 5cf33c3af..f4309acaf 100644
--- a/build.xml
+++ b/build.xml
@@ -194,6 +194,7 @@
         <pathelement location="${lib}/commons-math3-3.4.1.jar" />
         <pathelement location="${lib}/fontbox-2.0.15.jar" />
         <pathelement location="${lib}/guava-25.1-jre.jar" />
+        <pathelement location="${lib}/hazelcast-4.2.jar" />
         <pathelement location="${lib}/http2-client-9.4.34.v20201102.jar" />
         <pathelement location="${lib}/http2-common-9.4.34.v20201102.jar" />
         <pathelement location="${lib}/http2-http-client-transport-9.4.34.v20201102.jar" />
diff --git a/htroot/Status.java b/htroot/Status.java
index aa409d75a..0899bb341 100644
--- a/htroot/Status.java
+++ b/htroot/Status.java
@@ -320,7 +320,7 @@ public class Status
         prop.put("usedDisk", Formatter.bytesToString(sb.observer.getSizeOfDataPath(true)));

         prop.put("freeDisk", Formatter.bytesToString(sb.observer.getUsableSpace()));

         prop.put("processors", WorkflowProcessor.availableCPU);

-        prop.put("load", Memory.load());

+        prop.put("load", Memory.getSystemLoadAverage());

         ThreadMXBean threadbean = ManagementFactory.getThreadMXBean();

         prop.put("processesCurrentInclDaemon", threadbean.getThreadCount());

         prop.put("processesCurrentOnlyDaemon", threadbean.getDaemonThreadCount());

diff --git a/htroot/api/localpeers.java b/htroot/api/localpeers.java
index ba933d125..920008bc5 100644
--- a/htroot/api/localpeers.java
+++ b/htroot/api/localpeers.java
@@ -17,7 +17,19 @@
 // along with this program; if not, write to the Free Software
 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 
+import java.net.UnknownHostException;
+import java.util.Collection;
+import java.util.Map;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import com.hazelcast.cluster.Member;
+import com.hazelcast.core.HazelcastInstance;
+
 import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.util.Memory;
 import net.yacy.search.Switchboard;
 import net.yacy.server.serverObjects;
 import net.yacy.server.serverSwitch;
@@ -28,14 +40,50 @@ public class localpeers {
     public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
         final servletProperties prop = new servletProperties();
         int c = 0;
-        for (String urlstub: Switchboard.getSwitchboard().localpeers) {
-            prop.putJSON("peers_" + c + "_urlstub", urlstub);
+        for (String urlstub: Switchboard.getSwitchboard().localcluster_scan) {
+            prop.putJSON("peers_" + c + "_urlstub", urlstub); // a usrlstub is a full url with protocol, host and port up the the path start including first "/"
             c++;
         }
         prop.put("peers", c);
-
+        try {
+            prop.put("status", systemStatus().toString(2));
+        } catch (JSONException e) {
+            prop.put("status", "");
+        }
         // return rewrite properties
         return prop;
     }
 
-}
+    public static JSONObject systemStatus() throws JSONException {
+
+        // generate json
+        JSONObject systemStatus = new JSONObject(true);
+        Memory.status().forEach((k, v) -> {try {systemStatus.put(k, v);} catch (JSONException e) {}});
+        JSONArray members = new JSONArray();
+        HazelcastInstance hi = Switchboard.getSwitchboard().localcluster_hazelcast;
+        String uuid = hi.getCluster().getLocalMember().getUuid().toString();
+        hi.getMap("status").put(uuid, Memory.status());
+        for (Member member: hi.getCluster().getMembers()) {
+            JSONObject m = new JSONObject(true);
+            uuid = member.getUuid().toString();
+            m.put("uuid", uuid);
+            m.put("host", member.getAddress().getHost());
+            try {m.put("ip", member.getAddress().getInetAddress().getHostAddress());} catch (JSONException | UnknownHostException e) {}
+            m.put("port", member.getAddress().getPort());
+            m.put("isLite", member.isLiteMember());
+            m.put("isLocal", member.localMember());
+            @SuppressWarnings("unchecked")
+            Map<String, Object> status = (Map<String, Object>) hi.getMap("status").get(uuid);
+            m.put("status", status);
+            members.put(m);
+        }
+        systemStatus.put("hazelcast_cluster_name", hi.getConfig().getClusterName());
+        systemStatus.put("hazelcast_instance_name", hi.getConfig().getInstanceName());
+        Collection<String> interfaces = hi.getConfig().getNetworkConfig().getInterfaces().getInterfaces();
+        systemStatus.put("hazelcast_interfaces", interfaces);
+        systemStatus.put("hazelcast_members", members);
+        systemStatus.put("hazelcast_members_count", members.length());
+
+        return systemStatus;
+    }
+}
\ No newline at end of file
diff --git a/htroot/api/localpeers.json b/htroot/api/localpeers.json
index e2be9012d..4a6f747e2 100644
--- a/htroot/api/localpeers.json
+++ b/htroot/api/localpeers.json
@@ -1,5 +1,6 @@
 {
 "peers" : [#{peers|,}#
 {"urlstub":"#[urlstub]#"}
-#{/peers}#]
+#{/peers}#],
+"status": #[status]#
 }
\ No newline at end of file
diff --git a/htroot/api/status_p.java b/htroot/api/status_p.java
index 24b5ca1b6..1b877aa54 100644
--- a/htroot/api/status_p.java
+++ b/htroot/api/status_p.java
@@ -76,7 +76,7 @@ public class status_p {
         prop.putNum("usedDisk", sb.observer.getSizeOfDataPath(true));

         prop.putNum("freeDisk", sb.observer.getUsableSpace());

         prop.putNum("processors", WorkflowProcessor.availableCPU);

-        prop.putNum("load", Memory.load());

+        prop.putNum("load", Memory.getSystemLoadAverage());

 

         // proxy traffic

         prop.put("trafficIn", ByteCount.getGlobalCount());

diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java
index ce497d60c..a5182305e 100644
--- a/htroot/yacy/transferRWI.java
+++ b/htroot/yacy/transferRWI.java
@@ -101,7 +101,7 @@ public final class transferRWI {
             return prop;

         }

         // load tests

-        if (Memory.load() > 2.0 || MemoryControl.shortStatus()) {

+        if (Memory.getSystemLoadAverage() > 2.0 || MemoryControl.shortStatus()) {

             // check also Protocol.metadataRetrievalRunning.get() > 0 ?

             result = "too high load"; // don't tell too much details

             prop.put("result", result);

@@ -126,7 +126,7 @@ public final class transferRWI {
         result = "ok";

         final StringBuilder unknownURLs = new StringBuilder(6000);

 

-        double load = Memory.load();

+        double load = Memory.getSystemLoadAverage();

         float maxload = sb.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f);

         if (load > maxload) {

             // too high local load. this is bad but we must reject this to protect ourself!

diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java
index 386e8aa9b..9e1765680 100644
--- a/htroot/yacysearchitem.java
+++ b/htroot/yacysearchitem.java
@@ -437,7 +437,7 @@ public class yacysearchitem {
             boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted;
             boolean stealthmode = p2pmode && theSearch.query.isLocal();
             if ((sb.getConfigBool(SwitchboardConstants.HEURISTIC_SEARCHRESULTS, false) ||
-                (sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ACTIVE, false) && sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ENABLED, false) && Memory.load() < 1.0)) &&
+                (sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ACTIVE, false) && sb.getConfigBool(SwitchboardConstants.GREEDYLEARNING_ENABLED, false) && Memory.getSystemLoadAverage() < 1.0)) &&
                 !stealthmode) sb.heuristicSearchResults(result);
             theSearch.query.transmitcount = item + 1;
             return prop;
diff --git a/lib/hazelcast-4.2.License b/lib/hazelcast-4.2.License
new file mode 100644
index 000000000..d64569567
--- /dev/null
+++ b/lib/hazelcast-4.2.License
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/lib/hazelcast-4.2.jar b/lib/hazelcast-4.2.jar
new file mode 100644
index 000000000..1abd2f768
--- /dev/null
+++ b/lib/hazelcast-4.2.jar
diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java
index d6262e260..f4b142e96 100644
--- a/source/net/yacy/cora/protocol/Domains.java
+++ b/source/net/yacy/cora/protocol/Domains.java
@@ -109,6 +109,12 @@ public class Domains {
     private static Thread domaininit = null;
     static {
         localHostNames.add(LOCALHOST);
+        // look up the host name
+        try {
+            LOCALHOST_NAME = InetAddress.getLocalHost().getCanonicalHostName();
+        } catch (final UnknownHostException e) {}
+        localHostNames.add(LOCALHOST_NAME);
+
         try {
             InetAddress localHostAddress = InetAddress.getLocalHost();
             if (localHostAddress != null) myHostAddresses.add(localHostAddress);
@@ -118,6 +124,12 @@ public class Domains {
             if (moreAddresses != null) myHostAddresses.addAll(Arrays.asList(moreAddresses));
         } catch (final UnknownHostException e) {}
 
+        for (InetAddress a: myHostAddresses) {
+            if (a.isAnyLocalAddress() || a.isLinkLocalAddress() || a.isLoopbackAddress() || a.isSiteLocalAddress()) {
+                localHostAddresses.add(a);
+            }
+        }
+
         // to get the local host name, a dns lookup is necessary.
         // if such a lookup blocks, it can cause that the static initiatializer does not finish fast
         // therefore we start the host name lookup as concurrent thread
@@ -139,11 +151,6 @@ public class Domains {
                 } catch (final SocketException e) {
                 }
 
-                // now look up the host name
-                try {
-                    LOCALHOST_NAME = getHostName(InetAddress.getLocalHost());
-                } catch (final UnknownHostException e) {}
-
                 // after the host name was resolved, we try to look up more local addresses
                 // using the host name:
                 try {
diff --git a/source/net/yacy/cora/util/Memory.java b/source/net/yacy/cora/util/Memory.java
index 569e72c55..d0a171cf8 100644
--- a/source/net/yacy/cora/util/Memory.java
+++ b/source/net/yacy/cora/util/Memory.java
@@ -22,6 +22,12 @@ package net.yacy.cora.util;
 
 import java.lang.management.ManagementFactory;
 import java.lang.management.ThreadInfo;
+import java.lang.reflect.Method;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import net.yacy.http.YaCyHttpServer;
+import net.yacy.search.Switchboard;
 
 public class Memory {
 
@@ -74,15 +80,50 @@ public class Memory {
     public static final long cores() {
         return runtime.availableProcessors();
     }
-    
+
     /**
      * get the system load within the last minute
      * @return the system load or a negative number if the load is not available
      */
-    public static double load() {
+    public static double getSystemLoadAverage() {
         return ManagementFactory.getOperatingSystemMXBean().getSystemLoadAverage();
     }
-    
+
+    public static double getSystemCpuLoad() {
+        return getOSBean("getSystemCpuLoad");
+    }
+
+    public static double getProcessCpuLoad() {
+        return getOSBean("getProcessCpuLoad");
+    }
+
+    private static double getOSBean(String name) {
+        try {
+            Method m = ManagementFactory.getOperatingSystemMXBean().getClass().getMethod(name);
+            m.setAccessible(true);
+            Object o = m.invoke(ManagementFactory.getOperatingSystemMXBean());
+            if (o instanceof Double) return ((Double) o).doubleValue();
+        } catch (Throwable e) {}
+        return 0.0d;
+    }
+
+    public static Map<String, Object> status() {
+        Runtime runtime = Runtime.getRuntime();
+        Map<String, Object> status = new LinkedHashMap<>();
+        status.put("service", "Peer");
+        status.put("assigned_memory", runtime.maxMemory());
+        status.put("used_memory", runtime.totalMemory() - runtime.freeMemory());
+        status.put("available_memory", runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory());
+        status.put("cores", runtime.availableProcessors());
+        status.put("threads", Thread.activeCount());
+        status.put("deadlocks", deadlocks());
+        status.put("load_system_average", Memory.getSystemLoadAverage());
+        status.put("load_process_cpu", Memory.getProcessCpuLoad());
+        YaCyHttpServer server = Switchboard.getSwitchboard().getHttpServer();
+        status.put("server_threads", server == null ? 0 : server.getServerThreads());
+        return status;
+    }
+
     /**
      * find out the number of thread deadlocks. WARNING: this is a time-consuming task
      * @return the number of deadlocked threads
@@ -92,7 +133,7 @@ public class Memory {
         if (deadlockIDs == null) return 0;
         return deadlockIDs.length;
     }
-    
+
     /**
      * write deadlocked threads as to the log as warning
      */
diff --git a/source/net/yacy/http/Jetty9HttpServerImpl.java b/source/net/yacy/http/Jetty9HttpServerImpl.java
index 0780453d6..5163dd8d3 100644
--- a/source/net/yacy/http/Jetty9HttpServerImpl.java
+++ b/source/net/yacy/http/Jetty9HttpServerImpl.java
@@ -74,7 +74,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
      */
     public Jetty9HttpServerImpl(int port) {
         Switchboard sb = Switchboard.getSwitchboard();
-        
+
         server = new Server();
 
         int cores = ProcessorUtils.availableProcessors();
@@ -86,10 +86,10 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
         connector.setIdleTimeout(9000); // timout in ms when no bytes send / received
         connector.setAcceptQueueSize(128);
         server.addConnector(connector);
-        
+
         // add ssl/https connector
         boolean useSSL = sb.getConfigBool("server.https", false);
-      
+
         if (useSSL) {
             final SslContextFactory sslContextFactory = new SslContextFactory.Server();
             final SSLContext sslContext = initSslContext(sb);
@@ -150,21 +150,21 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
         sholder.setAsyncSupported(true); // needed for YaCyQoSFilter
         //sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html
         htrootContext.addServlet(sholder, "/*");
-        
-		final GzipHandler gzipHandler = new GzipHandler();
-		/*
-		 * Decompression of incoming requests body is required for index distribution
-		 * APIs /yacy/transferRWI.html and /yacy/transferURL.html This was previously
-		 * handled by a GZIPRequestWrapper in the YaCyDefaultServlet.
-		 */
-		gzipHandler.setInflateBufferSize(4096);
-		
-		if (!sb.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
-				SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT)) {
-			/* Gzip compression of responses can be disabled by user configuration */
-			gzipHandler.setExcludedMethods(HttpMethod.GET.asString(), HttpMethod.POST.asString());
-		}
-		htrootContext.setGzipHandler(gzipHandler);
+
+        final GzipHandler gzipHandler = new GzipHandler();
+        /*
+         * Decompression of incoming requests body is required for index distribution
+         * APIs /yacy/transferRWI.html and /yacy/transferURL.html This was previously
+         * handled by a GZIPRequestWrapper in the YaCyDefaultServlet.
+         */
+        gzipHandler.setInflateBufferSize(4096);
+
+        if (!sb.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP,
+                SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT)) {
+            /* Gzip compression of responses can be disabled by user configuration */
+            gzipHandler.setExcludedMethods(HttpMethod.GET.asString(), HttpMethod.POST.asString());
+        }
+        htrootContext.setGzipHandler(gzipHandler);
 
         // -----------------------------------------------------------------------------
         // here we set and map the mandatory servlets, needed for typical YaCy operation
@@ -177,13 +177,13 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
         //htrootContext.addServlet(SolrSelectServlet.class, "/solr/select"); // uses the default core, collection1
         //htrootContext.addServlet(SolrSelectServlet.class, "/solr/collection1/select"); // the same servlet, identifies the collection1 core using the path
         //htrootContext.addServlet(SolrSelectServlet.class, "/solr/webgraph/select"); // the same servlet, identifies the webgraph core using the path
-        
+
         //htrootContext.addServlet(SolrServlet.class, "/solr/collection1/admin/luke");
         //htrootContext.addServlet(SolrServlet.class, "/solr/webgraph/admin/luke");
 
         // add proxy?url= servlet
         //htrootContext.addServlet(YaCyProxyServlet.class,"/proxy.html");
-        
+
         // add GSA servlet
         //htrootContext.addServlet(GSAsearchServlet.class,"/gsa/search");
         // --- eof default servlet mappings --------------------------------------------
@@ -210,7 +210,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
         allrequesthandlers.addHandler(context);
         allrequesthandlers.addHandler(htrootContext);    
         allrequesthandlers.addHandler(new DefaultHandler()); // if not handled by other handler 
-        
+
         YaCyLoginService loginService = new YaCyLoginService();
         // this is very important (as it is part of the user password hash)
         // changes will ivalidate all current existing user-password-hashes (from userDB)
@@ -229,39 +229,39 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
         if (!white.equals("*")) { // full ip (allowed ranges 0-255 or prefix  10.0-255,0,0-100  or CIDR notation 192.168.1.0/24)
             final StringTokenizer st = new StringTokenizer(white, ",");
             final InetAccessHandler whiteListHandler;
-			if (white.contains("|")) {
-				/*
-				 * At least one pattern includes a path definition : we must use the
-				 * InetPathAccessHandler as InetAccessHandler doesn't support path patterns
-				 */
-				whiteListHandler = new InetPathAccessHandler();
-			} else {
-				whiteListHandler = new InetAccessHandler();
-			}
+            if (white.contains("|")) {
+                /*
+                 * At least one pattern includes a path definition : we must use the
+                 * InetPathAccessHandler as InetAccessHandler doesn't support path patterns
+                 */
+                whiteListHandler = new InetPathAccessHandler();
+            } else {
+                whiteListHandler = new InetAccessHandler();
+            }
             int i = 0;
             while (st.hasMoreTokens()) {
                 final String pattern = st.nextToken();
                 try {
-                	whiteListHandler.include(pattern);
+                    whiteListHandler.include(pattern);
                 } catch (final IllegalArgumentException nex) { // catch format exception on wrong ip address pattern
                     ConcurrentLog.severe("SERVER", "Server Access Settings - IP filter: " + nex.getMessage());
                     continue;
                 }
                 i++;
-            }          
+            }
             if (i > 0) {
-            	final String loopbackAddress = InetAddress.getLoopbackAddress().getHostAddress();
-            	whiteListHandler.include(loopbackAddress);
+                final String loopbackAddress = InetAddress.getLoopbackAddress().getHostAddress();
+                whiteListHandler.include(loopbackAddress);
                 whiteListHandler.setHandler(crashHandler);
                 this.server.setHandler(whiteListHandler);
-                
+
                 ConcurrentLog.info("SERVER","activated IP access restriction to: [" + loopbackAddress + "," + white +"]");
             } else {
                 server.setHandler(crashHandler); // InetAccessHandler not needed
             }
         } else {
             server.setHandler(crashHandler); // InetAccessHandler not needed
-        }        
+        }
     }
 
     /**
@@ -311,7 +311,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
         }
         return -1;
     }
-    
+
     /**
      * reconnect with new port settings (after waiting milsec) - routine returns
      * immediately
@@ -418,92 +418,90 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
 
         // getting the keystore file name
         String keyStoreFileName = sb.getConfig("keyStore", "").trim();
- 
+
         // getting the keystore pwd
         String keyStorePwd = sb.getConfig("keyStorePassword", "").trim();
- 
+
         // take a look if we have something to import
         final String pkcs12ImportFile = sb.getConfig("pkcs12ImportFile", "").trim();
-        
+
         // if no keyStore and no import is defined, then set the default key
         if (keyStoreFileName.isEmpty() && keyStorePwd.isEmpty() && pkcs12ImportFile.isEmpty()) {
             keyStoreFileName = "defaults/freeworldKeystore";
             keyStorePwd = "freeworld";
             sb.setConfig("keyStore", keyStoreFileName);
             sb.setConfig("keyStorePassword", keyStorePwd);
-        } 
-        
+        }
+
         if (pkcs12ImportFile.length() > 0) {
             ConcurrentLog.info("SERVER", "Import certificates from import file '" + pkcs12ImportFile + "'.");
- 
+
             try {
                 // getting the password
                 final String pkcs12ImportPwd = sb.getConfig("pkcs12ImportPwd", "").trim();
- 
+
                 // creating tool to import cert
                 final PKCS12Tool pkcsTool = new PKCS12Tool(pkcs12ImportFile,pkcs12ImportPwd);
- 
+
                 // creating a new keystore file
                 if (keyStoreFileName.isEmpty()) {
                     // using the default keystore name
                     keyStoreFileName = "DATA/SETTINGS/myPeerKeystore";
- 
+
                     // creating an empty java keystore
                     final KeyStore ks = KeyStore.getInstance("JKS");
                     ks.load(null,keyStorePwd.toCharArray());
                     try (
-                    	/* Automatically closed by this try-with-resources statement */	
-                    	final FileOutputStream ksOut = new FileOutputStream(keyStoreFileName);
+                        /* Automatically closed by this try-with-resources statement */    
+                        final FileOutputStream ksOut = new FileOutputStream(keyStoreFileName);
                     ) {
-                    	ks.store(ksOut, keyStorePwd.toCharArray());
+                        ks.store(ksOut, keyStorePwd.toCharArray());
                     }
- 
+
                     // storing path to keystore into config file
                     sb.setConfig("keyStore", keyStoreFileName);
                 }
- 
+
                 // importing certificate
                 pkcsTool.importToJKS(keyStoreFileName, keyStorePwd);
- 
+
                 // removing entries from config file
                 sb.setConfig("pkcs12ImportFile", "");
                 sb.setConfig("pkcs12ImportPwd", "");
- 
+
                 // deleting original import file
                 // TODO: should we do this
- 
             } catch (final Exception e) {
                 ConcurrentLog.severe("SERVER", "Unable to import certificate from import file '" + pkcs12ImportFile + "'.",e);
             }
         } else if (keyStoreFileName.isEmpty()) return null;
- 
- 
+
         // get the ssl context
         try {
             ConcurrentLog.info("SERVER","Initializing SSL support ...");
- 
+
             // creating a new keystore instance of type (java key store)
             if (ConcurrentLog.isFine("SERVER")) ConcurrentLog.fine("SERVER", "Initializing keystore ...");
             final KeyStore ks = KeyStore.getInstance("JKS");
- 
+
             // loading keystore data from file
             if (ConcurrentLog.isFine("SERVER")) ConcurrentLog.fine("SERVER","Loading keystore file " + keyStoreFileName);
             final FileInputStream stream = new FileInputStream(keyStoreFileName);
             try {
-            	ks.load(stream, keyStorePwd.toCharArray());
+                ks.load(stream, keyStorePwd.toCharArray());
             } finally {
-            	try {
-            		stream.close();
-            	} catch(IOException ioe) {
-            		ConcurrentLog.warn("SERVER", "Could not close input stream on file " + keyStoreFileName);
-            	}
+                try {
+                    stream.close();
+                } catch(IOException ioe) {
+                    ConcurrentLog.warn("SERVER", "Could not close input stream on file " + keyStoreFileName);
+                }
             }
- 
+
             // creating a keystore factory
             if (ConcurrentLog.isFine("SERVER")) ConcurrentLog.fine("SERVER","Initializing key manager factory ...");
             final KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
             kmf.init(ks,keyStorePwd.toCharArray());
- 
+
             // initializing the ssl context
             if (ConcurrentLog.isFine("SERVER")) ConcurrentLog.fine("SERVER","Initializing SSL context ...");
             final SSLContext sslcontext = SSLContext.getInstance("TLS");
@@ -517,7 +515,12 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
             return null;
         }
     }
-    
+
+    @Override
+    public int getServerThreads() {
+        return server == null ? 0 : server.getThreadPool().getThreads() - server.getThreadPool().getIdleThreads();
+    }
+
     @Override
     public String toString() {
         return this.server.dump() + "\n\n" + this.server.getState();
diff --git a/source/net/yacy/http/YaCyHttpServer.java b/source/net/yacy/http/YaCyHttpServer.java
index 61cf27c61..c25bd2caa 100644
--- a/source/net/yacy/http/YaCyHttpServer.java
+++ b/source/net/yacy/http/YaCyHttpServer.java
@@ -18,4 +18,5 @@ public interface YaCyHttpServer {
     abstract boolean withSSL();

     abstract void reconnect(int milsec);

     abstract String getVersion();

+    abstract int getServerThreads();

 }

diff --git a/source/net/yacy/kelondro/workflow/AbstractBusyThread.java b/source/net/yacy/kelondro/workflow/AbstractBusyThread.java
index 898bcd435..82b97065c 100644
--- a/source/net/yacy/kelondro/workflow/AbstractBusyThread.java
+++ b/source/net/yacy/kelondro/workflow/AbstractBusyThread.java
@@ -202,8 +202,8 @@ public abstract class AbstractBusyThread extends AbstractThread implements BusyT
                 ratz(this.idlePause);
                 idletime += System.currentTimeMillis() - timestamp;
             //} else if ((memnow = serverMemory.available()) > memprereq) try {
-            } else if (Memory.load() > loadprereq) {
-            	logSystem("Thread '" + this.getName() + "' runs high load cycle. current: " + Memory.load() + " max.: " + loadprereq);
+            } else if (Memory.getSystemLoadAverage() > loadprereq) {
+            	logSystem("Thread '" + this.getName() + "' runs high load cycle. current: " + Memory.getSystemLoadAverage() + " max.: " + loadprereq);
                 timestamp = System.currentTimeMillis();
                 ratz(this.idlePause);
                 highCPUCycles++;
diff --git a/source/net/yacy/peers/Dispatcher.java b/source/net/yacy/peers/Dispatcher.java
index d38faf903..d3af84dbf 100644
--- a/source/net/yacy/peers/Dispatcher.java
+++ b/source/net/yacy/peers/Dispatcher.java
@@ -367,7 +367,7 @@ public class Dispatcher implements WorkflowTask<Transmission.Chunk> {
         while (Protocol.metadataRetrievalRunning.get() > 0) try {Thread.sleep(1000);} catch (InterruptedException e) {break;}
         
         // we must test this here again
-        while (Memory.load() > this.env.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f)) try {Thread.sleep(10000);} catch (InterruptedException e) {break;}
+        while (Memory.getSystemLoadAverage() > this.env.getConfigFloat(SwitchboardConstants.INDEX_DIST_LOADPREREQ, 2.0f)) try {Thread.sleep(10000);} catch (InterruptedException e) {break;}
         
         // do the transmission
         final boolean success = chunk.transmit();
diff --git a/source/net/yacy/peers/RemoteSearch.java b/source/net/yacy/peers/RemoteSearch.java
index 52a4fdae6..70c0f7a1a 100644
--- a/source/net/yacy/peers/RemoteSearch.java
+++ b/source/net/yacy/peers/RemoteSearch.java
@@ -187,7 +187,7 @@ public class RemoteSearch extends Thread {
         if (indexingQueueSize > 0) {redundancy = Math.max(1, redundancy - 1); healthMessage.append(", indexingQueueSize > 0");}

         if (indexingQueueSize > 10) {redundancy = Math.max(1, redundancy - 1); healthMessage.append(", indexingQueueSize > 10");}

         if (indexingQueueSize > 50) {redundancy = Math.max(1, redundancy - 1); healthMessage.append(", indexingQueueSize > 50");}

-        if (Memory.load() > 2.0) {redundancy = Math.max(1, redundancy - 1); healthMessage.append(", load() > 2.0");}

+        if (Memory.getSystemLoadAverage() > 2.0) {redundancy = Math.max(1, redundancy - 1); healthMessage.append(", load() > 2.0");}

         if (Memory.cores() < 4) {redundancy = Math.max(1, redundancy - 1); healthMessage.append(", cores() < 4");}

         if (Memory.cores() == 1) {redundancy = 1; healthMessage.append(", cores() == 1");}

         final int minage = 3;

@@ -275,7 +275,7 @@ public class RemoteSearch extends Thread {
 					event.query.isStrictContentDom(), useFacets, event.excludeintext_image);

             for (Seed s: robinsonPeers) {

 				if (MemoryControl.shortStatus()

-						|| Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR,

+						|| Memory.getSystemLoadAverage() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR,

 								SwitchboardConstants.REMOTESEARCH_MAXLOAD_SOLR_DEFAULT)) {

 					continue;

 				}

@@ -289,7 +289,7 @@ public class RemoteSearch extends Thread {
             for (Seed dhtPeer: dhtPeers) {

                 if (dhtPeer == null || dhtPeer.hash == null) continue;

 				if (MemoryControl.shortStatus()

-						|| Memory.load() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI,

+						|| Memory.getSystemLoadAverage() > sb.getConfigFloat(SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI,

 								SwitchboardConstants.REMOTESEARCH_MAXLOAD_RWI_DEFAULT)) {

 					continue;

 				}

diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java
index 85a78900f..abd827622 100644
--- a/source/net/yacy/search/Switchboard.java
+++ b/source/net/yacy/search/Switchboard.java
@@ -107,6 +107,12 @@ import org.json.JSONTokener;
 import com.cybozu.labs.langdetect.DetectorFactory;
 import com.cybozu.labs.langdetect.LangDetectException;
 import com.google.common.io.Files;
+import com.hazelcast.config.Config;
+import com.hazelcast.config.InterfacesConfig;
+import com.hazelcast.config.JoinConfig;
+import com.hazelcast.config.NetworkConfig;
+import com.hazelcast.core.Hazelcast;
+import com.hazelcast.core.HazelcastInstance;
 
 import net.yacy.contentcontrol.ContentControlFilterUpdateThread;
 import net.yacy.contentcontrol.SMWListSyncThread;
@@ -309,7 +315,8 @@ public final class Switchboard extends serverSwitch {
     private Dispatcher dhtDispatcher;
     public LinkedBlockingQueue<String> trail; // connect infos from cytag servlet
     public SeedDB peers;
-    public Set<String> localpeers;
+    public Set<String> localcluster_scan;
+    public HazelcastInstance localcluster_hazelcast;
     public WorkTables tables;
     public Tray tray;
     private long lastStats = 0; // time when the last row was written to the stats table
@@ -640,14 +647,25 @@ public final class Switchboard extends serverSwitch {
                 this.exceed134217727);
         String agent = getConfig(SwitchboardConstants.NETWORK_UNIT_AGENT, "");
         if (!agent.isEmpty()) this.peers.setMyName(agent); // this can thus be set using the environment variable yacy.network.unit.agent
-        this.localpeers = Collections.newSetFromMap(new ConcurrentHashMap<>());
+        this.localcluster_scan = Collections.newSetFromMap(new ConcurrentHashMap<>());
         new OneTimeBusyThread("Switchboard.scanForOtherYaCyInIntranet") {
             @Override
             public boolean jobImpl() throws Exception {
-                Switchboard.this.localpeers.addAll(Scanner.scanForOtherYaCyInIntranet());
+                Switchboard.this.localcluster_scan.addAll(Scanner.scanForOtherYaCyInIntranet());
                 return true;
             }
         }.start();
+        // initialize hazelcast
+        InterfacesConfig interfacesConfig = new InterfacesConfig();
+        Domains.myIntranetIPs().forEach(ip -> interfacesConfig.addInterface(ip.getHostAddress()));
+        NetworkConfig networkConfig = new NetworkConfig().setInterfaces(interfacesConfig);
+        JoinConfig join = networkConfig.getJoin();
+        join.getMulticastConfig().setEnabled(true);
+        Config config = new Config().setClusterName("YaCyP2P").setInstanceName("Peer").setNetworkConfig(networkConfig);
+        config.getCPSubsystemConfig().setCPMemberCount(3);
+        localcluster_hazelcast = Hazelcast.newHazelcastInstance(config);
+        String uuid = localcluster_hazelcast.getCluster().getLocalMember().getUuid().toString();
+        localcluster_hazelcast.getMap("status").put(uuid, Memory.status());
 
         // load domainList
         try {
@@ -2951,8 +2969,8 @@ public final class Switchboard extends serverSwitch {
                 if (!reference_index_exist) log.info("postprocessing deactivated: no reference index avilable; activate citation index or webgraph");
                 boolean minimum_ram_fullfilled = MemoryControl.available() > getConfigLong("postprocessing.minimum_ram", 0);
                 if (!minimum_ram_fullfilled) log.info("postprocessing deactivated: no enough ram (" + MemoryControl.available() + "), needed " + getConfigLong("postprocessing.minimum_ram", 0) + ", to force change field postprocessing.minimum_ram");
-                boolean minimum_load_fullfilled = Memory.load() < getConfigFloat("postprocessing.maximum_load", 0);
-                if (!minimum_load_fullfilled) log.info("postprocessing deactivated: too high load (" + Memory.load() + ") > " + getConfigFloat("postprocessing.maximum_load", 0) + ", to force change field postprocessing.maximum_load");
+                boolean minimum_load_fullfilled = Memory.getSystemLoadAverage() < getConfigFloat("postprocessing.maximum_load", 0);
+                if (!minimum_load_fullfilled) log.info("postprocessing deactivated: too high load (" + Memory.getSystemLoadAverage() + ") > " + getConfigFloat("postprocessing.maximum_load", 0) + ", to force change field postprocessing.maximum_load");
                 boolean postprocessing = process_key_exist && reference_index_exist && minimum_ram_fullfilled && minimum_load_fullfilled;
                 if (!postprocessing) log.info("postprocessing deactivated: constraints violated");
 
diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java
index 5c5763956..746a5900d 100644
--- a/source/net/yacy/server/serverSwitch.java
+++ b/source/net/yacy/server/serverSwitch.java
@@ -742,7 +742,7 @@ public class serverSwitch {
     }

 

     public YaCyHttpServer getHttpServer() {

-        return httpserver;

+        return this.httpserver;

     }

 

 }