Server-validated X-Real-IP proxy handling, publicPort lifecycle fix, peer-statistic iam parameter, transfer-IP fallback, and matching tests/docs

8 files changed, 97 insertions, 51 deletions

diff --git a/source/net/yacy/cora/protocol/RequestHeader.java b/source/net/yacy/cora/protocol/RequestHeader.java
index 2a4140af8..35f43cc3b 100644
--- a/source/net/yacy/cora/protocol/RequestHeader.java
+++ b/source/net/yacy/cora/protocol/RequestHeader.java
@@ -77,6 +77,9 @@ public class RequestHeader extends HeaderFramework implements HttpServletRequest
     public static final String X_CACHE = "X-Cache";
     public static final String X_CACHE_LOOKUP = "X-Cache-Lookup";
     public static final String X_Real_IP = "X-Real-IP";
+    /** Trusted effective client IP, populated by the HTTP server after proxy validation. */
+    public static final String EFFECTIVE_CLIENT_IP_ATTRIBUTE =
+            RequestHeader.class.getName() + ".effectiveClientIp";
 
     public static final String COOKIE = "Cookie";
 
@@ -705,10 +708,8 @@ public class RequestHeader extends HeaderFramework implements HttpServletRequest
      * The IP address of the host that opened the TCP connection (the real socket peer).
      * <p>
      * In contrast to {@link #getRemoteAddr()} and {@link #client(ServletRequest)} this
-     * <b>never</b> honors the client-controlled X-Real-IP request header. It must therefore
-     * be used for all authentication and access-control decisions: X-Real-IP is trivially
-     * spoofable by a direct client and must only be trusted for peer routing behind a
-     * trusted reverse proxy, not for authentication.
+     * always returns the socket peer. It must therefore be used for authentication and
+     * access-control decisions.
      *
      * @return the socket peer IP address
      */
@@ -720,23 +721,20 @@ public class RequestHeader extends HeaderFramework implements HttpServletRequest
     }
 
     /**
-     * Resolve the client IP for peer routing and logging. This honors the X-Real-IP
-     * request header (set e.g. by an nginx reverse proxy via
-     * "proxy_set_header X-Real-IP $remote_addr;").
-     * <p>
-     * <b>Do not use this for authentication or access control</b> - the header is
-     * client-controlled and spoofable. Use {@link #getRemoteSocketAddr()} for that.
+     * Resolve the client IP for peer routing and logging. A reverse-proxy address is
+     * honored only when the HTTP server has validated the socket peer and populated
+     * {@link #EFFECTIVE_CLIENT_IP_ATTRIBUTE}. Client-supplied forwarding headers are
+     * never read here directly.
      *
      * @param request the servlet request
      * @return the routing client IP address
      */
     public static String client(final ServletRequest request) {
-        String clientHost = request.getRemoteAddr();
-        if (request instanceof HttpServletRequest) {
-        	String XRealIP = ((HttpServletRequest) request).getHeader(X_Real_IP);
-        	if (XRealIP != null && XRealIP.length() > 0) clientHost = XRealIP; // get IP through nginx config "proxy_set_header X-Real-IP $remote_addr;"
+        final Object effectiveClientIp = request.getAttribute(EFFECTIVE_CLIENT_IP_ATTRIBUTE);
+        if (effectiveClientIp instanceof String && !((String) effectiveClientIp).isEmpty()) {
+            return (String) effectiveClientIp;
         }
-		return clientHost;
+        return request.getRemoteAddr();
     }
     
     @Override
@@ -748,12 +746,9 @@ public class RequestHeader extends HeaderFramework implements HttpServletRequest
     }
     
     public static String host(final ServletRequest request) {
-        String clientHost = request.getRemoteHost();
-        if (request instanceof HttpServletRequest) {
-        	String XRealIP = ((HttpServletRequest) request).getHeader(X_Real_IP);
-        	if (XRealIP != null && XRealIP.length() > 0) clientHost = XRealIP; // get IP through nginx config "proxy_set_header X-Real-IP $remote_addr;"
-        }
-		return clientHost;
+        final Object effectiveClientIp = request.getAttribute(EFFECTIVE_CLIENT_IP_ATTRIBUTE);
+        return effectiveClientIp instanceof String && !((String) effectiveClientIp).isEmpty()
+                ? (String) effectiveClientIp : request.getRemoteHost();
     }
 
     @Override
diff --git a/source/net/yacy/htroot/SettingsAck_p.java b/source/net/yacy/htroot/SettingsAck_p.java
index 9836c7435..665bad611 100644
--- a/source/net/yacy/htroot/SettingsAck_p.java
+++ b/source/net/yacy/htroot/SettingsAck_p.java
@@ -203,15 +203,20 @@ public class SettingsAck_p {
 
             // publicPort
             final String publicPort =  (post.get("publicPort")).trim();
-            try {
-                final Integer pport = Integer.parseInt(publicPort);
-                if(pport < 65535 && pport >= 0) {
-                    serverCore.usePublicPort = true;
-                    sb.peers.mySeed().setPort(pport);
-                    env.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, publicPort);
+            if (publicPort.isEmpty()) {
+                serverCore.usePublicPort = false;
+                env.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, "");
+            } else {
+                try {
+                    final Integer pport = Integer.parseInt(publicPort);
+                    if (Seed.isProperPort(pport)) {
+                        serverCore.usePublicPort = true;
+                        sb.peers.mySeed().setPort(pport);
+                        env.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, publicPort);
+                    }
+                } catch (final NumberFormatException e) {
+                    // Keep the previously configured public port on invalid input.
                 }
-            } catch (final NumberFormatException e) {
-                // noop
             }
 
             // server access data
diff --git a/source/net/yacy/htroot/yacy/search.java b/source/net/yacy/htroot/yacy/search.java
index 499ab6776..40965d324 100644
--- a/source/net/yacy/htroot/yacy/search.java
+++ b/source/net/yacy/htroot/yacy/search.java
@@ -45,6 +45,7 @@ import net.yacy.cora.document.encoding.ASCII;
 import net.yacy.cora.document.feed.RSSMessage;
 import net.yacy.cora.document.id.MultiProtocolURL;
 import net.yacy.cora.lod.vocabulary.Tagging;
+import net.yacy.cora.order.Base64Order;
 import net.yacy.cora.protocol.Domains;
 import net.yacy.cora.protocol.HeaderFramework;
 import net.yacy.cora.protocol.RequestHeader;
@@ -54,6 +55,7 @@ import net.yacy.cora.storage.HandleSet;
 import net.yacy.cora.util.SpaceExceededException;
 import net.yacy.gui.Audio;
 import net.yacy.kelondro.data.meta.URIMetadataNode;
+import net.yacy.kelondro.data.word.Word;
 import net.yacy.kelondro.data.word.WordReference;
 import net.yacy.kelondro.data.word.WordReferenceFactory;
 import net.yacy.kelondro.data.word.WordReferenceRow;
@@ -66,6 +68,7 @@ import net.yacy.peers.EventChannel;
 import net.yacy.peers.Network;
 import net.yacy.peers.Protocol;
 import net.yacy.peers.Seed;
+import net.yacy.peers.SeedDB;
 import net.yacy.peers.graphics.ProfilingGraph;
 import net.yacy.search.EventTracker;
 import net.yacy.search.Switchboard;
@@ -108,6 +111,7 @@ public final class search {
         //System.out.println("yacy: search received request = " + post.toString());
 
         final String  oseed  = post.get("myseed", ""); // complete seed of the requesting peer
+        final String  iam    = post.get("iam", "");    // seed hash of the requesting peer
 //      final String  youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability
         final String  query  = post.get("query", "");  // a string of word hashes that shall be searched and combined
         final String  exclude= post.get("exclude", "");// a string of word hashes that shall not be within the search result
@@ -167,8 +171,7 @@ public final class search {
         }
 
         // check the search tracker
-        TreeSet<Long> trackerHandles = sb.remoteSearchTracker.get(client);
-        if (trackerHandles == null) trackerHandles = new TreeSet<Long>();
+        final TreeSet<Long> trackerHandles = sb.remoteSearchTracker.computeIfAbsent(client, key -> new TreeSet<Long>());
         boolean block = false;
         synchronized (trackerHandles) {
             if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() -   3000)).size() >  1) {
@@ -430,7 +433,7 @@ public final class search {
         }
 
         // prepare search statistics
-        theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), -1, true, false, false);
+        theQuery.remotepeer = resolveRemotePeer(sb.peers, iam, client);
         theQuery.searchtime = System.currentTimeMillis() - timestamp;
         theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime();
         theQuery.snippetcomputationtime = (theSearch == null) ? 0 : theSearch.getSnippetComputationTime();
@@ -442,7 +445,6 @@ public final class search {
             // we don't need too much entries in the list; remove superfluous
             while (trackerHandles.size() > 36) if (!trackerHandles.remove(trackerHandles.first())) break;
         }
-        sb.remoteSearchTracker.put(client, trackerHandles);
         if (MemoryControl.shortStatus()) sb.remoteSearchTracker.clear();
 
         // log
@@ -460,4 +462,21 @@ public final class search {
         return prop;
     }
 
+    /**
+     * Resolve the peer used for search statistics. Prefer the requester's known
+     * peer hash because multiple peers can share one public IP behind NAT. The
+     * IP lookup is retained for older requests and unknown peer hashes.
+     */
+    static Seed resolveRemotePeer(final SeedDB peers, final String iam, final String client) {
+        Seed remotePeer = null;
+        if (iam != null && iam.length() == Word.commonHashLength
+                && Base64Order.enhancedCoder.wellformed(ASCII.getBytes(iam))) {
+            remotePeer = peers.get(iam);
+        }
+        if (remotePeer == null && client != null) {
+            remotePeer = peers.lookupByIP(Domains.dnsResolve(client), -1, true, false, false);
+        }
+        return remotePeer;
+    }
+
 }
diff --git a/source/net/yacy/http/Jetty12HttpServer.java b/source/net/yacy/http/Jetty12HttpServer.java
index cc72eab49..56fc2b6b5 100644
--- a/source/net/yacy/http/Jetty12HttpServer.java
+++ b/source/net/yacy/http/Jetty12HttpServer.java
@@ -482,6 +482,12 @@ public class Jetty12HttpServer implements YaCyHttpServer {
                 throws IOException, ServletException {
             AdminSecurity.AuthenticationContext.setSocketPeerIp(baseRequest.getRemoteAddr());
             try {
+                final Switchboard switchboard = Switchboard.getSwitchboard();
+                request.setAttribute(RequestHeader.EFFECTIVE_CLIENT_IP_ATTRIBUTE,
+                        resolveTrustedClientIp(request,
+                                switchboard.getConfig(
+                                        SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED,
+                                        SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED_DEFAULT)));
                 super.handle(pathInContext, baseRequest, request, response);
             } finally {
                 AdminSecurity.AuthenticationContext.clear();
@@ -493,9 +499,7 @@ public class Jetty12HttpServer implements YaCyHttpServer {
                 final org.eclipse.jetty.ee8.nested.Request request) {
             final Switchboard switchboard = Switchboard.getSwitchboard();
             final String socketRemoteIp = request.getRemoteAddr();
-            final String trackingRemoteIp = resolveTrackingClientIp(request,
-                    switchboard.getConfig(SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED,
-                            SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED_DEFAULT));
+            final String trackingRemoteIp = RequestHeader.client(request);
             serverAccessTracker.track(trackingRemoteIp, pathInContext);
             final AdminSecurity.AccessPolicy policy = new AdminSecurity.AccessPolicy(
                     switchboard.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_All_PAGES, false),
@@ -519,8 +523,8 @@ public class Jetty12HttpServer implements YaCyHttpServer {
             return roleInfo;
         }
 
-        /** Resolve the client address for display and tracking, never for access control. */
-        static String resolveTrackingClientIp(final HttpServletRequest request,
+        /** Resolve the trusted client address for routing and tracking, never for access control. */
+        static String resolveTrustedClientIp(final HttpServletRequest request,
                 final String trustedProxyPatterns) {
             final String socketRemoteIp = request.getRemoteAddr();
             if (!ProxyAccessPolicy.isClientAllowed(trustedProxyPatterns, socketRemoteIp)) {
diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java
index b874a7676..026679cae 100644
--- a/source/net/yacy/peers/Protocol.java
+++ b/source/net/yacy/peers/Protocol.java
@@ -1734,7 +1734,7 @@ public final class Protocol {
         String result = in.get("result");

         if ( result == null ) {

             String errorCause = "no result from transferRWI";

-            String usedIP = in.get(Seed.IP);

+            final String usedIP = transferTargetIP(in, targetSeed);
             sb.peers.peerActions.interfaceDeparture(targetSeed, usedIP); // disconnect unavailable peer

             return errorCause;

         }

@@ -1817,7 +1817,7 @@ public final class Protocol {
         result = in.get("result");

         if ( result == null ) {

             String errorCause = "no result from transferURL";

-            String usedIP = in.get(Seed.IP);

+            final String usedIP = transferTargetIP(in, targetSeed);
             sb.peers.peerActions.interfaceDeparture(targetSeed, usedIP); // disconnect unavailable peer ip

             return errorCause;

         }

@@ -1866,6 +1866,19 @@ public final class Protocol {
         return null;

     }

 

+    /**
+     * Return the address recorded by a transfer attempt. Malformed responses may
+     * omit that local metadata; in that case use the first address from the
+     * canonical, ordered {@link Seed#getIPs()} view as a last-resort fallback.
+     */
+    static String transferTargetIP(final Map<String, String> response, final Seed targetSeed) {
+        final String usedIP = response == null ? null : response.get(Seed.IP);
+        if (usedIP != null && !usedIP.isEmpty()) return usedIP;
+
+        final Set<String> targetIPs = targetSeed.getIPs();
+        return targetIPs.isEmpty() ? null : targetIPs.iterator().next();
+    }
+
     /**

      * Transfer Reverse Word Index entries to remote peer. If the used IP is not

      * responding, this IP (interface) is removed from targtSeed IP list. Remote

diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java
index b2fbc8edb..fa6a1230d 100644
--- a/source/net/yacy/peers/Seed.java
+++ b/source/net/yacy/peers/Seed.java
@@ -376,7 +376,11 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
      * If no feedback from other peers exist, then all locally determined IPs are returned.

      * If a feedback from other peers exist, then return at most two IPs:

      * the latest IPv4 and the latest IPv6 which was returned during a hello process from a remote peer

-     * @return a set of IPs which are supposed to be my own public IPs

+     * This is the canonical view of a peer's addresses. Callers performing a
+     * network operation should iterate the returned set to allow address-family
+     * fallback. When only a last-resort single address is needed, use the first
+     * iterator entry after checking that the set is not empty.
+     * @return an ordered set of IPs which are supposed to be my own public IPs
      */

     public final Set<String> getIPs() {

         Set<String> h = new LinkedHashSet<>();

diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java
index 405fbcab8..994d2a978 100644
--- a/source/net/yacy/search/SwitchboardConstants.java
+++ b/source/net/yacy/search/SwitchboardConstants.java
@@ -66,7 +66,7 @@ public final class SwitchboardConstants {
     public static final String SERVER_SHUTDOWNPORT          = "port.shutdown"; // local port to listen for a shutdown signal (0 <= disabled)
     public static final String SERVER_STATICIP              = "staticIP"; // static IP of http server
     public static final String SERVER_PUBLICPORT            = "publicPort";
-    /** Socket peers whose X-Real-IP header may be used for request tracking. */
+    /** Socket peers whose validated X-Real-IP header may be used for routing and tracking. */
     public static final String SERVER_REVERSE_PROXY_TRUSTED = "server.reverseProxy.trusted";
     public static final String SERVER_REVERSE_PROXY_TRUSTED_DEFAULT =
             "127[.]0[.]0[.]1,0:0:0:0:0:0:0:1,::1";
diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java
index 17c0c8904..21fa8dd16 100644
--- a/source/net/yacy/server/serverSwitch.java
+++ b/source/net/yacy/server/serverSwitch.java
@@ -227,15 +227,21 @@ public class serverSwitch {
      *

      * @see #getLocalPort()

      */

-    public int getPublicPort(final String key, final int dflt) {

-

-        if (this.isConnectedViaUpnp && this.upnpPortMap.containsKey(key)) {

-            return this.upnpPortMap.get(key).intValue();

-        }

-

-        // TODO: add way of setting and retrieving port for manual NAT

-

-        return this.getConfigInt(key, dflt);

+    public int getPublicPort(final String key, final int dflt) {
+
+        if (SwitchboardConstants.SERVER_PORT.equals(key)) {
+            final int configuredPublicPort =
+                    this.getConfigInt(SwitchboardConstants.SERVER_PUBLICPORT, -1);
+            if (Seed.isProperPort(configuredPublicPort)) {
+                return configuredPublicPort;
+            }
+        }
+
+        if (this.isConnectedViaUpnp && this.upnpPortMap.containsKey(key)) {
+            return this.upnpPortMap.get(key).intValue();
+        }
+
+        return this.getConfigInt(key, dflt);
     }

 

     /**