diff options
23 files changed, 435 insertions, 82 deletions
diff --git a/defaults/yacy.init b/defaults/yacy.init index 471a39ec2..d9d2c2007 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -32,9 +32,12 @@ upnp.remoteHost = bindPort =
# Comma-separated regular expressions matching trusted reverse-proxy socket IPs. -# X-Real-IP is used for access tracking only when the direct connection comes -# from one of these addresses. Authentication and access control always use the -# direct socket IP. Keep this list restricted to proxies controlled by you. +# A single valid IPv4 or IPv6 address from X-Real-IP is used as the effective +# client address for routing, logging, and other request attribution only when +# the direct connection comes from one of these addresses. Authentication and +# access control always use the direct socket IP. +# Keep this list restricted to proxies controlled by you, and configure them to +# overwrite rather than pass through a client-supplied X-Real-IP value. server.reverseProxy.trusted=127[.]0[.]0[.]1,0:0:0:0:0:0:0:1,::1 # TLS/SSL support:
@@ -549,8 +552,9 @@ debug.snippets.statistics.enabled=false #staticIP if you have a static IP, you can use this setting
staticIP=
-#publicPort if you use a different port to access YaCy than the one it listens on, you can use this setting
-publicPort=
+# Optional public HTTP/P2P port (1-65535) advertised instead of the UPnP or +# local listening port. Leave empty to disable the manual override. +publicPort= # each time YaCy starts up, it can trigger the local browser to show the
# status page. This is active by default, to make it easier for first-time
diff --git a/help/AccessTracker_p.md b/help/AccessTracker_p.md index 09684e4bc..63a56634b 100644 --- a/help/AccessTracker_p.md +++ b/help/AccessTracker_p.md @@ -63,12 +63,12 @@ GET or POST /AccessTracker_p.html?host=...&page=... Expect observations: counts, logs, queues, timing, network rows, thread states, or resource values. Monitoring does not fix the issue by itself; it points to the next page or setting to change. When YaCy is reached through a reverse proxy, Access Tracker uses `X-Real-IP` -only when the proxy's direct socket IP matches one of the comma-separated regular -expressions in `server.reverseProxy.trusted`. Loopback proxies are trusted by -default. Authentication and access-control decisions continue to use the direct -socket IP, not the forwarded address. The reverse proxy must overwrite -`X-Real-IP` with the client address; it must not pass a client-supplied value -unchanged. +only when the proxy's direct socket IP matches one of the comma-separated +regular expressions in `server.reverseProxy.trusted` and the header contains +one valid IPv4 or IPv6 address. Loopback proxies are trusted by default. +Authentication and access-control decisions continue to use the direct socket +IP, not the forwarded address. The reverse proxy must overwrite `X-Real-IP` +with the client address; it must not pass a client-supplied value unchanged. ## Related Pages diff --git a/help/SettingsAck_p.md b/help/SettingsAck_p.md index 20d9224fa..96844bfa9 100644 --- a/help/SettingsAck_p.md +++ b/help/SettingsAck_p.md @@ -58,11 +58,12 @@ The table explains values that an agent or script must set deliberately. Paramet | `fileHost` | Host or domain scope. | Set only when this option is part of the intended request; otherwise omit it and let YaCy use the page default. | | `msgForwardingEnabled` | Enables the named feature. | Changes stored data, configuration, or a running job. Use the authenticated action flow where required and verify the result. | | `port` | HTTP port where YaCy listens. Values below 1024 are ignored by this form; changing the port triggers reconnect/redirect behavior. | Changes stored data, configuration, or a running job. Use the authenticated action flow where required and verify the result. | +| `publicPort` | Optional public non-TLS HTTP port, from 1 through 65535, advertised in YaCy's seed/P2P metadata. It takes precedence over UPnP and the local listening port. An empty value disables the manual override. | Processed with the `serveraccount` settings action. Invalid, non-numeric, zero, or out-of-range values leave the previous setting unchanged. Set it when NAT or a reverse proxy exposes YaCy on a different external port. | | `proxyaccess` | Saves the proxy client IP-number filter. | Changes proxy access scope; verify the filter before submitting. | | `proxyfilter` | Filter expression. It decides which records are included, excluded, displayed, exported, or processed on this page. | Controls the scope or format of the result. Prefer the narrowest value that answers the request. | | `remoteProxyHost` | Host or domain scope. | Set only when this option is part of the intended request; otherwise omit it and let YaCy use the page default. | | `remoteProxyUser` | User or account value. | Set only when this option is part of the intended request; otherwise omit it and let YaCy use the page default. | -| `serveraccount` | User or account value. | Controls the scope or format of the result. Prefer the narrowest value that answers the request. | +| `serveraccount` | Action marker whose presence applies the Server Access form, including `fileHost`, `staticIP`, `publicPort`, and `serverfilter`. Its submitted value is not interpreted. | Requires administrator access and a valid transaction token. Include the complete Server Access form state because the action processes its related fields together. | | `serverfilter` | Filter expression. It decides which records are included, excluded, displayed, exported, or processed on this page. | Controls the scope or format of the result. Prefer the narrowest value that answers the request. | | `serveruser` | User or account value. | Set only when this option is part of the intended request; otherwise omit it and let YaCy use the page default. | | `urlproxydomains` | Host or domain scope. | Set only when this option is part of the intended request; otherwise omit it and let YaCy use the page default. | @@ -75,6 +76,12 @@ Example request shape: GET or POST /SettingsAck_p.html?port=...&adminaccount=...&adminpw1=...&adminpw2=...&adminuser=... ``` +For a server-access form submission, `serveraccount` selects the action and +`publicPort` carries the optional override. A valid value is stored and applied +to the current peer seed. Clearing the value removes the override; subsequent +seed refreshes then use the UPnP mapping when available, otherwise the local +HTTP listening port. The setting does not override the public TLS port. + ## What To Expect A successful change is visible as a saved value, a confirmation, or changed behavior on a related page. Some settings take effect immediately; others require reconnecting, reloading translations, restarting services, or watching the status page. diff --git a/help/yacy/hello.md b/help/yacy/hello.md index f73593ce5..bda5bad2c 100644 --- a/help/yacy/hello.md +++ b/help/yacy/hello.md @@ -34,6 +34,13 @@ Call the endpoint as a protocol surface. Use exact parameter names and encoded v This is a peer-service endpoint for YaCy peer communication, not a normal editing page. +Behind a reverse proxy, YaCy accepts `X-Real-IP` as the effective client and +routing address only when the proxy socket IP matches +`server.reverseProxy.trusted` and the header contains one valid IPv4 or IPv6 +address. Otherwise YaCy uses the socket IP. Authentication and access control +always use the socket IP. The proxy must overwrite, not pass through, any +client-supplied `X-Real-IP` value. + ## Automation And API Page backend: `source/net/yacy/htroot/yacy/hello.java`. diff --git a/help/yacy/list.md b/help/yacy/list.md index 546f4aa24..cc6f59848 100644 --- a/help/yacy/list.md +++ b/help/yacy/list.md @@ -34,6 +34,13 @@ Call the endpoint as a protocol surface. Use exact parameter names and encoded v This is a peer-service endpoint for YaCy peer communication, not a normal editing page. +Behind a reverse proxy, YaCy accepts `X-Real-IP` as the effective client and +routing address only when the proxy socket IP matches +`server.reverseProxy.trusted` and the header contains one valid IPv4 or IPv6 +address. Otherwise YaCy uses the socket IP. Authentication and access control +always use the socket IP. The proxy must overwrite, not pass through, any +client-supplied `X-Real-IP` value. + ## Automation And API Page backend: `source/net/yacy/htroot/yacy/list.java`. diff --git a/help/yacy/message.md b/help/yacy/message.md index aeeab8bf7..d8a4fec12 100644 --- a/help/yacy/message.md +++ b/help/yacy/message.md @@ -34,6 +34,13 @@ Call the endpoint as a protocol surface. Use exact parameter names and encoded v This is a peer-service endpoint for YaCy peer communication, not a normal editing page. +Behind a reverse proxy, YaCy accepts `X-Real-IP` as the effective client and +routing address only when the proxy socket IP matches +`server.reverseProxy.trusted` and the header contains one valid IPv4 or IPv6 +address. Otherwise YaCy uses the socket IP. Authentication and access control +always use the socket IP. The proxy must overwrite, not pass through, any +client-supplied `X-Real-IP` value. + ## Automation And API Page backend: `source/net/yacy/htroot/yacy/message.java`. diff --git a/help/yacy/profile.md b/help/yacy/profile.md index 53d642f76..303829a6f 100644 --- a/help/yacy/profile.md +++ b/help/yacy/profile.md @@ -34,6 +34,13 @@ Call the endpoint as a protocol surface. Use exact parameter names and encoded v This is a peer-service endpoint for YaCy peer communication, not a normal editing page. +Behind a reverse proxy, YaCy accepts `X-Real-IP` as the effective client and +routing address only when the proxy socket IP matches +`server.reverseProxy.trusted` and the header contains one valid IPv4 or IPv6 +address. Otherwise YaCy uses the socket IP. Authentication and access control +always use the socket IP. The proxy must overwrite, not pass through, any +client-supplied `X-Real-IP` value. + ## Automation And API Page backend: `source/net/yacy/htroot/yacy/profile.java`. diff --git a/help/yacy/query.md b/help/yacy/query.md index aa785593b..c385ae264 100644 --- a/help/yacy/query.md +++ b/help/yacy/query.md @@ -34,6 +34,13 @@ Call the endpoint as a protocol surface. Use exact parameter names and encoded v This is a peer-service endpoint for YaCy peer communication, not a normal editing page. +Behind a reverse proxy, YaCy accepts `X-Real-IP` as the effective client and +routing address only when the proxy socket IP matches +`server.reverseProxy.trusted` and the header contains one valid IPv4 or IPv6 +address. Otherwise YaCy uses the socket IP. Authentication and access control +always use the socket IP. The proxy must overwrite, not pass through, any +client-supplied `X-Real-IP` value. + ## Automation And API Page backend: `source/net/yacy/htroot/yacy/query.java`. diff --git a/help/yacy/search.md b/help/yacy/search.md index e8cba7e54..c7b3387e8 100644 --- a/help/yacy/search.md +++ b/help/yacy/search.md @@ -34,6 +34,13 @@ Call the endpoint as a protocol surface. Use exact parameter names and encoded v This is a peer-service endpoint for YaCy peer communication, not a normal editing page. +Behind a reverse proxy, YaCy accepts `X-Real-IP` as the effective client and +routing address only when the proxy socket IP matches +`server.reverseProxy.trusted` and the header contains one valid IPv4 or IPv6 +address. Otherwise YaCy uses the socket IP. Authentication and access control +always use the socket IP. The proxy must overwrite, not pass through, any +client-supplied `X-Real-IP` value. + ## Automation And API Page backend: `source/net/yacy/htroot/yacy/search.java`. @@ -53,6 +60,7 @@ The table explains values that an agent or script must set deliberately. Paramet | `contentdom` | Content domain filter. Common values are `all`, `text`, `image`, `audio`, `video`, and `app`; use it to ask for web pages, media, or application documents deliberately. | Set only when this option is part of the intended request; otherwise omit it and let YaCy use the page default. | | `count` | SRU-style result count. It is an alternative to `maximumRecords` on search endpoints. | Controls the scope or format of the result. Prefer the narrowest value that answers the request. | | `filter` | Filter text or expression used to narrow the displayed records. | Controls the scope or format of the result. Prefer the narrowest value that answers the request. | +| `iam` | Twelve-character enhanced-Base64 seed hash of the requesting YaCy peer. When it identifies a known peer, YaCy uses it for search-statistics attribution so peers sharing an address behind NAT remain distinguishable. | This is attribution metadata, not an independent authentication credential. Missing, malformed, or unknown hashes fall back to the effective request client IP. | | `language` | Interface language. Values are `browser` for the browser-preferred language, `default` for English, or a language code such as `de`, `fr`, `es`, `zh`, `ja`, or `ko`. | Changes stored data, configuration, or a running job. Use the authenticated action flow where required and verify the result. | | `query` | Search text. Use ordinary search terms, quoted phrases where supported by YaCy query parsing, and optional YaCy modifiers such as collection filters when you intentionally need them. | Controls the scope or format of the result. Prefer the narrowest value that answers the request. | | `sitehost` | Host or domain scope. | Set only when this option is part of the intended request; otherwise omit it and let YaCy use the page default. | @@ -63,7 +71,7 @@ The table explains values that an agent or script must set deliberately. Paramet Example request shape: ```http -GET or POST /yacy/search.html?language=...&query=...&contentdom=...&abstracts=...&author=... +GET or POST /yacy/search.html?iam=...&language=...&query=...&contentdom=...&abstracts=...&author=... ``` ## What To Expect diff --git a/help/yacy/transferRWI.md b/help/yacy/transferRWI.md index 3f7fe06cf..5cea79956 100644 --- a/help/yacy/transferRWI.md +++ b/help/yacy/transferRWI.md @@ -34,6 +34,13 @@ Call the endpoint as a protocol surface. Use exact parameter names and encoded v This is a peer-service endpoint for YaCy peer communication, not a normal editing page. +Behind a reverse proxy, YaCy accepts `X-Real-IP` as the effective client and +routing address only when the proxy socket IP matches +`server.reverseProxy.trusted` and the header contains one valid IPv4 or IPv6 +address. Otherwise YaCy uses the socket IP. Authentication and access control +always use the socket IP. The proxy must overwrite, not pass through, any +client-supplied `X-Real-IP` value. + ## Automation And API Page backend: `source/net/yacy/htroot/yacy/transferRWI.java`. diff --git a/source/net/yacy/cora/protocol/RequestHeader.java b/source/net/yacy/cora/protocol/RequestHeader.java index 2a4140af8..35f43cc3b 100644 --- a/source/net/yacy/cora/protocol/RequestHeader.java +++ b/source/net/yacy/cora/protocol/RequestHeader.java @@ -77,6 +77,9 @@ public class RequestHeader extends HeaderFramework implements HttpServletRequest public static final String X_CACHE = "X-Cache"; public static final String X_CACHE_LOOKUP = "X-Cache-Lookup"; public static final String X_Real_IP = "X-Real-IP"; + /** Trusted effective client IP, populated by the HTTP server after proxy validation. */ + public static final String EFFECTIVE_CLIENT_IP_ATTRIBUTE = + RequestHeader.class.getName() + ".effectiveClientIp"; public static final String COOKIE = "Cookie"; @@ -705,10 +708,8 @@ public class RequestHeader extends HeaderFramework implements HttpServletRequest * The IP address of the host that opened the TCP connection (the real socket peer). * <p> * In contrast to {@link #getRemoteAddr()} and {@link #client(ServletRequest)} this - * <b>never</b> honors the client-controlled X-Real-IP request header. It must therefore - * be used for all authentication and access-control decisions: X-Real-IP is trivially - * spoofable by a direct client and must only be trusted for peer routing behind a - * trusted reverse proxy, not for authentication. + * always returns the socket peer. It must therefore be used for authentication and + * access-control decisions. * * @return the socket peer IP address */ @@ -720,23 +721,20 @@ public class RequestHeader extends HeaderFramework implements HttpServletRequest } /** - * Resolve the client IP for peer routing and logging. This honors the X-Real-IP - * request header (set e.g. by an nginx reverse proxy via - * "proxy_set_header X-Real-IP $remote_addr;"). - * <p> - * <b>Do not use this for authentication or access control</b> - the header is - * client-controlled and spoofable. Use {@link #getRemoteSocketAddr()} for that. + * Resolve the client IP for peer routing and logging. A reverse-proxy address is + * honored only when the HTTP server has validated the socket peer and populated + * {@link #EFFECTIVE_CLIENT_IP_ATTRIBUTE}. Client-supplied forwarding headers are + * never read here directly. * * @param request the servlet request * @return the routing client IP address */ public static String client(final ServletRequest request) { - String clientHost = request.getRemoteAddr(); - if (request instanceof HttpServletRequest) { - String XRealIP = ((HttpServletRequest) request).getHeader(X_Real_IP); - if (XRealIP != null && XRealIP.length() > 0) clientHost = XRealIP; // get IP through nginx config "proxy_set_header X-Real-IP $remote_addr;" + final Object effectiveClientIp = request.getAttribute(EFFECTIVE_CLIENT_IP_ATTRIBUTE); + if (effectiveClientIp instanceof String && !((String) effectiveClientIp).isEmpty()) { + return (String) effectiveClientIp; } - return clientHost; + return request.getRemoteAddr(); } @Override @@ -748,12 +746,9 @@ public class RequestHeader extends HeaderFramework implements HttpServletRequest } public static String host(final ServletRequest request) { - String clientHost = request.getRemoteHost(); - if (request instanceof HttpServletRequest) { - String XRealIP = ((HttpServletRequest) request).getHeader(X_Real_IP); - if (XRealIP != null && XRealIP.length() > 0) clientHost = XRealIP; // get IP through nginx config "proxy_set_header X-Real-IP $remote_addr;" - } - return clientHost; + final Object effectiveClientIp = request.getAttribute(EFFECTIVE_CLIENT_IP_ATTRIBUTE); + return effectiveClientIp instanceof String && !((String) effectiveClientIp).isEmpty() + ? (String) effectiveClientIp : request.getRemoteHost(); } @Override diff --git a/source/net/yacy/htroot/SettingsAck_p.java b/source/net/yacy/htroot/SettingsAck_p.java index 9836c7435..665bad611 100644 --- a/source/net/yacy/htroot/SettingsAck_p.java +++ b/source/net/yacy/htroot/SettingsAck_p.java @@ -203,15 +203,20 @@ public class SettingsAck_p { // publicPort final String publicPort = (post.get("publicPort")).trim(); - try { - final Integer pport = Integer.parseInt(publicPort); - if(pport < 65535 && pport >= 0) { - serverCore.usePublicPort = true; - sb.peers.mySeed().setPort(pport); - env.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, publicPort); + if (publicPort.isEmpty()) { + serverCore.usePublicPort = false; + env.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, ""); + } else { + try { + final Integer pport = Integer.parseInt(publicPort); + if (Seed.isProperPort(pport)) { + serverCore.usePublicPort = true; + sb.peers.mySeed().setPort(pport); + env.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, publicPort); + } + } catch (final NumberFormatException e) { + // Keep the previously configured public port on invalid input. } - } catch (final NumberFormatException e) { - // noop } // server access data diff --git a/source/net/yacy/htroot/yacy/search.java b/source/net/yacy/htroot/yacy/search.java index 499ab6776..40965d324 100644 --- a/source/net/yacy/htroot/yacy/search.java +++ b/source/net/yacy/htroot/yacy/search.java @@ -45,6 +45,7 @@ import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.feed.RSSMessage; import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.lod.vocabulary.Tagging; +import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; @@ -54,6 +55,7 @@ import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.SpaceExceededException; import net.yacy.gui.Audio; import net.yacy.kelondro.data.meta.URIMetadataNode; +import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.data.word.WordReferenceFactory; import net.yacy.kelondro.data.word.WordReferenceRow; @@ -66,6 +68,7 @@ import net.yacy.peers.EventChannel; import net.yacy.peers.Network; import net.yacy.peers.Protocol; import net.yacy.peers.Seed; +import net.yacy.peers.SeedDB; import net.yacy.peers.graphics.ProfilingGraph; import net.yacy.search.EventTracker; import net.yacy.search.Switchboard; @@ -108,6 +111,7 @@ public final class search { //System.out.println("yacy: search received request = " + post.toString()); final String oseed = post.get("myseed", ""); // complete seed of the requesting peer + final String iam = post.get("iam", ""); // seed hash of the requesting peer // final String youare = post.get("youare", ""); // seed hash of the target peer, used for testing network stability final String query = post.get("query", ""); // a string of word hashes that shall be searched and combined final String exclude= post.get("exclude", "");// a string of word hashes that shall not be within the search result @@ -167,8 +171,7 @@ public final class search { } // check the search tracker - TreeSet<Long> trackerHandles = sb.remoteSearchTracker.get(client); - if (trackerHandles == null) trackerHandles = new TreeSet<Long>(); + final TreeSet<Long> trackerHandles = sb.remoteSearchTracker.computeIfAbsent(client, key -> new TreeSet<Long>()); boolean block = false; synchronized (trackerHandles) { if (trackerHandles.tailSet(Long.valueOf(System.currentTimeMillis() - 3000)).size() > 1) { @@ -430,7 +433,7 @@ public final class search { } // prepare search statistics - theQuery.remotepeer = client == null ? null : sb.peers.lookupByIP(Domains.dnsResolve(client), -1, true, false, false); + theQuery.remotepeer = resolveRemotePeer(sb.peers, iam, client); theQuery.searchtime = System.currentTimeMillis() - timestamp; theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime(); theQuery.snippetcomputationtime = (theSearch == null) ? 0 : theSearch.getSnippetComputationTime(); @@ -442,7 +445,6 @@ public final class search { // we don't need too much entries in the list; remove superfluous while (trackerHandles.size() > 36) if (!trackerHandles.remove(trackerHandles.first())) break; } - sb.remoteSearchTracker.put(client, trackerHandles); if (MemoryControl.shortStatus()) sb.remoteSearchTracker.clear(); // log @@ -460,4 +462,21 @@ public final class search { return prop; } + /** + * Resolve the peer used for search statistics. Prefer the requester's known + * peer hash because multiple peers can share one public IP behind NAT. The + * IP lookup is retained for older requests and unknown peer hashes. + */ + static Seed resolveRemotePeer(final SeedDB peers, final String iam, final String client) { + Seed remotePeer = null; + if (iam != null && iam.length() == Word.commonHashLength + && Base64Order.enhancedCoder.wellformed(ASCII.getBytes(iam))) { + remotePeer = peers.get(iam); + } + if (remotePeer == null && client != null) { + remotePeer = peers.lookupByIP(Domains.dnsResolve(client), -1, true, false, false); + } + return remotePeer; + } + } diff --git a/source/net/yacy/http/Jetty12HttpServer.java b/source/net/yacy/http/Jetty12HttpServer.java index cc72eab49..56fc2b6b5 100644 --- a/source/net/yacy/http/Jetty12HttpServer.java +++ b/source/net/yacy/http/Jetty12HttpServer.java @@ -482,6 +482,12 @@ public class Jetty12HttpServer implements YaCyHttpServer { throws IOException, ServletException { AdminSecurity.AuthenticationContext.setSocketPeerIp(baseRequest.getRemoteAddr()); try { + final Switchboard switchboard = Switchboard.getSwitchboard(); + request.setAttribute(RequestHeader.EFFECTIVE_CLIENT_IP_ATTRIBUTE, + resolveTrustedClientIp(request, + switchboard.getConfig( + SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED, + SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED_DEFAULT))); super.handle(pathInContext, baseRequest, request, response); } finally { AdminSecurity.AuthenticationContext.clear(); @@ -493,9 +499,7 @@ public class Jetty12HttpServer implements YaCyHttpServer { final org.eclipse.jetty.ee8.nested.Request request) { final Switchboard switchboard = Switchboard.getSwitchboard(); final String socketRemoteIp = request.getRemoteAddr(); - final String trackingRemoteIp = resolveTrackingClientIp(request, - switchboard.getConfig(SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED, - SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED_DEFAULT)); + final String trackingRemoteIp = RequestHeader.client(request); serverAccessTracker.track(trackingRemoteIp, pathInContext); final AdminSecurity.AccessPolicy policy = new AdminSecurity.AccessPolicy( switchboard.getConfigBool(SwitchboardConstants.ADMIN_ACCOUNT_All_PAGES, false), @@ -519,8 +523,8 @@ public class Jetty12HttpServer implements YaCyHttpServer { return roleInfo; } - /** Resolve the client address for display and tracking, never for access control. */ - static String resolveTrackingClientIp(final HttpServletRequest request, + /** Resolve the trusted client address for routing and tracking, never for access control. */ + static String resolveTrustedClientIp(final HttpServletRequest request, final String trustedProxyPatterns) { final String socketRemoteIp = request.getRemoteAddr(); if (!ProxyAccessPolicy.isClientAllowed(trustedProxyPatterns, socketRemoteIp)) { diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index b874a7676..026679cae 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -1734,7 +1734,7 @@ public final class Protocol { String result = in.get("result");
if ( result == null ) {
String errorCause = "no result from transferRWI";
- String usedIP = in.get(Seed.IP);
+ final String usedIP = transferTargetIP(in, targetSeed); sb.peers.peerActions.interfaceDeparture(targetSeed, usedIP); // disconnect unavailable peer
return errorCause;
}
@@ -1817,7 +1817,7 @@ public final class Protocol { result = in.get("result");
if ( result == null ) {
String errorCause = "no result from transferURL";
- String usedIP = in.get(Seed.IP);
+ final String usedIP = transferTargetIP(in, targetSeed); sb.peers.peerActions.interfaceDeparture(targetSeed, usedIP); // disconnect unavailable peer ip
return errorCause;
}
@@ -1866,6 +1866,19 @@ public final class Protocol { return null;
}
+ /** + * Return the address recorded by a transfer attempt. Malformed responses may + * omit that local metadata; in that case use the first address from the + * canonical, ordered {@link Seed#getIPs()} view as a last-resort fallback. + */ + static String transferTargetIP(final Map<String, String> response, final Seed targetSeed) { + final String usedIP = response == null ? null : response.get(Seed.IP); + if (usedIP != null && !usedIP.isEmpty()) return usedIP; + + final Set<String> targetIPs = targetSeed.getIPs(); + return targetIPs.isEmpty() ? null : targetIPs.iterator().next(); + } + /**
* Transfer Reverse Word Index entries to remote peer. If the used IP is not
* responding, this IP (interface) is removed from targtSeed IP list. Remote
diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index b2fbc8edb..fa6a1230d 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -376,7 +376,11 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> * If no feedback from other peers exist, then all locally determined IPs are returned.
* If a feedback from other peers exist, then return at most two IPs:
* the latest IPv4 and the latest IPv6 which was returned during a hello process from a remote peer
- * @return a set of IPs which are supposed to be my own public IPs
+ * This is the canonical view of a peer's addresses. Callers performing a + * network operation should iterate the returned set to allow address-family + * fallback. When only a last-resort single address is needed, use the first + * iterator entry after checking that the set is not empty. + * @return an ordered set of IPs which are supposed to be my own public IPs */
public final Set<String> getIPs() {
Set<String> h = new LinkedHashSet<>();
diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 405fbcab8..994d2a978 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -66,7 +66,7 @@ public final class SwitchboardConstants { public static final String SERVER_SHUTDOWNPORT = "port.shutdown"; // local port to listen for a shutdown signal (0 <= disabled) public static final String SERVER_STATICIP = "staticIP"; // static IP of http server public static final String SERVER_PUBLICPORT = "publicPort"; - /** Socket peers whose X-Real-IP header may be used for request tracking. */ + /** Socket peers whose validated X-Real-IP header may be used for routing and tracking. */ public static final String SERVER_REVERSE_PROXY_TRUSTED = "server.reverseProxy.trusted"; public static final String SERVER_REVERSE_PROXY_TRUSTED_DEFAULT = "127[.]0[.]0[.]1,0:0:0:0:0:0:0:1,::1"; diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java index 17c0c8904..21fa8dd16 100644 --- a/source/net/yacy/server/serverSwitch.java +++ b/source/net/yacy/server/serverSwitch.java @@ -227,15 +227,21 @@ public class serverSwitch { *
* @see #getLocalPort()
*/
- public int getPublicPort(final String key, final int dflt) {
-
- if (this.isConnectedViaUpnp && this.upnpPortMap.containsKey(key)) {
- return this.upnpPortMap.get(key).intValue();
- }
-
- // TODO: add way of setting and retrieving port for manual NAT
-
- return this.getConfigInt(key, dflt);
+ public int getPublicPort(final String key, final int dflt) { + + if (SwitchboardConstants.SERVER_PORT.equals(key)) { + final int configuredPublicPort = + this.getConfigInt(SwitchboardConstants.SERVER_PUBLICPORT, -1); + if (Seed.isProperPort(configuredPublicPort)) { + return configuredPublicPort; + } + } + + if (this.isConnectedViaUpnp && this.upnpPortMap.containsKey(key)) { + return this.upnpPortMap.get(key).intValue(); + } + + return this.getConfigInt(key, dflt); }
/**
diff --git a/test/java/net/yacy/cora/protocol/RequestHeaderTest.java b/test/java/net/yacy/cora/protocol/RequestHeaderTest.java index 0931d8e0c..6b953971b 100644 --- a/test/java/net/yacy/cora/protocol/RequestHeaderTest.java +++ b/test/java/net/yacy/cora/protocol/RequestHeaderTest.java @@ -37,9 +37,10 @@ public class RequestHeaderTest { /**
* Build a minimal HttpServletRequest stub answering getRemoteAddr() with the
- * given socket peer address and returning the given X-Real-IP header value.
- */
- private static HttpServletRequest stubRequest(final String socketPeer, final String xRealIP) {
+ * given socket peer address, X-Real-IP header and server-validated effective IP. + */ + private static HttpServletRequest stubRequest(final String socketPeer, final String xRealIP, + final String effectiveClientIp) { final InvocationHandler h = new InvocationHandler() {
@Override
public Object invoke(Object proxy, Method method, Object[] args) {
@@ -48,8 +49,11 @@ public class RequestHeaderTest { return socketPeer;
case "getRemoteHost":
return socketPeer;
- case "getHeader":
- return RequestHeader.X_Real_IP.equals(args[0]) ? xRealIP : null;
+ case "getHeader": + return RequestHeader.X_Real_IP.equals(args[0]) ? xRealIP : null; + case "getAttribute": + return RequestHeader.EFFECTIVE_CLIENT_IP_ATTRIBUTE.equals(args[0]) + ? effectiveClientIp : null; default:
return null;
}
@@ -70,20 +74,27 @@ public class RequestHeaderTest { final String remoteClient = "203.0.113.7"; // a non-local address (TEST-NET-3)
// spoofing attempt: remote socket peer, but X-Real-IP claims localhost
- final RequestHeader spoofed = new RequestHeader(stubRequest(remoteClient, "127.0.0.1"));
- // routing accessor honors the header (kept for peer routing behind a trusted proxy) ...
- assertEquals("127.0.0.1", spoofed.getRemoteAddr());
- // ... but the authentication accessor must return the true socket peer
- assertEquals(remoteClient, spoofed.getRemoteSocketAddr());
- assertFalse("spoofed X-Real-IP must not grant localhost access", spoofed.accessFromLocalhost());
-
- // genuine localhost access still works
- final RequestHeader local = new RequestHeader(stubRequest("127.0.0.1", null));
+ final RequestHeader spoofed = new RequestHeader( + stubRequest(remoteClient, "127.0.0.1", null)); + // Neither routing nor authentication may honor an unvalidated forwarding header. + assertEquals(remoteClient, spoofed.getRemoteAddr()); + assertEquals(remoteClient, spoofed.getRemoteSocketAddr()); + assertFalse("spoofed X-Real-IP must not grant localhost access", spoofed.accessFromLocalhost()); + + // A trusted HTTP-server decision is visible to P2P routing, but not socket security. + final RequestHeader proxied = new RequestHeader( + stubRequest("127.0.0.1", "198.51.100.23", "198.51.100.23")); + assertEquals("198.51.100.23", proxied.getRemoteAddr()); + assertEquals("198.51.100.23", proxied.getRemoteHost()); + assertEquals("127.0.0.1", proxied.getRemoteSocketAddr()); + + // genuine localhost access still works + final RequestHeader local = new RequestHeader(stubRequest("127.0.0.1", null, null)); assertEquals("127.0.0.1", local.getRemoteSocketAddr());
assertTrue(local.accessFromLocalhost());
// remote client without spoofing stays remote
- final RequestHeader remote = new RequestHeader(stubRequest(remoteClient, null));
+ final RequestHeader remote = new RequestHeader(stubRequest(remoteClient, null, null)); assertFalse(remote.accessFromLocalhost());
}
diff --git a/test/java/net/yacy/htroot/yacy/SearchPeerResolutionTest.java b/test/java/net/yacy/htroot/yacy/SearchPeerResolutionTest.java new file mode 100644 index 000000000..4f1359f26 --- /dev/null +++ b/test/java/net/yacy/htroot/yacy/SearchPeerResolutionTest.java @@ -0,0 +1,79 @@ +package net.yacy.htroot.yacy; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import net.yacy.peers.Seed; +import net.yacy.peers.SeedDB; + +public class SearchPeerResolutionTest { + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private SeedDB seedDB; + + @Before + public void setUp() throws Exception { + final File networkRoot = this.temporaryFolder.newFolder("network"); + this.seedDB = new SeedDB( + networkRoot, + "seedActive.db", + "seedPassive.db", + "seedPotential.db", + new File(networkRoot, SeedDB.DBFILE_OWN_SEED), + 1, + 4, + false, + false); + } + + @After + public void tearDown() { + if (this.seedDB != null) this.seedDB.close(); + } + + @Test + public void iamHashDisambiguatesPeersSharingOneNatAddress() throws Exception { + final Seed firstPeer = seed("AAAAAAAAAAAA", "first-nat-peer", "198.51.100.10", 8090); + final Seed secondPeer = seed("BBBBBBBBBBBB", "second-nat-peer", "198.51.100.10", 8091); + this.seedDB.addConnected(firstPeer); + this.seedDB.addConnected(secondPeer); + + final Seed resolved = search.resolveRemotePeer(this.seedDB, secondPeer.hash, "198.51.100.10"); + + assertEquals(secondPeer.hash, resolved.hash); + } + + @Test + public void unknownOrInvalidIamHashFallsBackToClientIp() throws Exception { + final Seed peer = seed("CCCCCCCCCCCC", "fallback-peer", "198.51.100.20", 8090); + this.seedDB.addConnected(peer); + + assertEquals(peer.hash, + search.resolveRemotePeer(this.seedDB, "DDDDDDDDDDDD", "198.51.100.20").hash); + assertEquals(peer.hash, + search.resolveRemotePeer(this.seedDB, "not-a-hash", "198.51.100.20").hash); + assertEquals(peer.hash, + search.resolveRemotePeer(this.seedDB, "", "198.51.100.20").hash); + } + + private static Seed seed(final String hash, final String name, final String ip, final int port) { + final ConcurrentMap<String, String> dna = new ConcurrentHashMap<String, String>(); + final Seed seed = new Seed(hash, dna); + seed.setName(name); + seed.setType(Seed.PEERTYPE_SENIOR); + seed.setIP(ip); + seed.setPort(port); + return seed; + } +} diff --git a/test/java/net/yacy/http/Jetty12HttpServerTest.java b/test/java/net/yacy/http/Jetty12HttpServerTest.java index f591db7b9..39cf4e380 100644 --- a/test/java/net/yacy/http/Jetty12HttpServerTest.java +++ b/test/java/net/yacy/http/Jetty12HttpServerTest.java @@ -13,6 +13,8 @@ import java.io.OutputStream; import java.lang.reflect.Proxy; import java.net.Socket; import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; @@ -121,27 +123,60 @@ public class Jetty12HttpServerTest { final HttpServletRequest trustedProxyRequest = requestWithRemoteAddress( "127.0.0.1", "198.51.100.23"); assertEquals("198.51.100.23", - Jetty12HttpServer.AdminSecurityHandler.resolveTrackingClientIp( + Jetty12HttpServer.AdminSecurityHandler.resolveTrustedClientIp( trustedProxyRequest, SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED_DEFAULT)); final HttpServletRequest spoofedDirectRequest = requestWithRemoteAddress( "203.0.113.10", "127.0.0.1"); assertEquals("203.0.113.10", - Jetty12HttpServer.AdminSecurityHandler.resolveTrackingClientIp( + Jetty12HttpServer.AdminSecurityHandler.resolveTrustedClientIp( spoofedDirectRequest, SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED_DEFAULT)); final HttpServletRequest invalidForwardedAddress = requestWithRemoteAddress( "127.0.0.1", "unknown, 198.51.100.23"); assertEquals("127.0.0.1", - Jetty12HttpServer.AdminSecurityHandler.resolveTrackingClientIp( + Jetty12HttpServer.AdminSecurityHandler.resolveTrustedClientIp( invalidForwardedAddress, SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED_DEFAULT)); + + final HttpServletRequest trustedProxyIpv6Request = requestWithRemoteAddress( + "::1", "2001:db8::23"); + assertEquals("2001:db8::23", + Jetty12HttpServer.AdminSecurityHandler.resolveTrustedClientIp( + trustedProxyIpv6Request, + SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED_DEFAULT)); + } + + @Test + public void exposesOnlyServerValidatedProxyAddressToRequestHeader() { + final HttpServletRequest trustedProxyRequest = requestWithRemoteAddress( + "127.0.0.1", "198.51.100.23"); + trustedProxyRequest.setAttribute(RequestHeader.EFFECTIVE_CLIENT_IP_ATTRIBUTE, + Jetty12HttpServer.AdminSecurityHandler.resolveTrustedClientIp( + trustedProxyRequest, + SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED_DEFAULT)); + + final RequestHeader trustedProxyHeader = new RequestHeader(trustedProxyRequest); + assertEquals("198.51.100.23", trustedProxyHeader.getRemoteAddr()); + assertEquals("127.0.0.1", trustedProxyHeader.getRemoteSocketAddr()); + + final HttpServletRequest spoofedDirectRequest = requestWithRemoteAddress( + "203.0.113.10", "127.0.0.1"); + spoofedDirectRequest.setAttribute(RequestHeader.EFFECTIVE_CLIENT_IP_ATTRIBUTE, + Jetty12HttpServer.AdminSecurityHandler.resolveTrustedClientIp( + spoofedDirectRequest, + SwitchboardConstants.SERVER_REVERSE_PROXY_TRUSTED_DEFAULT)); + + final RequestHeader spoofedDirectHeader = new RequestHeader(spoofedDirectRequest); + assertEquals("203.0.113.10", spoofedDirectHeader.getRemoteAddr()); + assertEquals("203.0.113.10", spoofedDirectHeader.getRemoteSocketAddr()); } private static HttpServletRequest requestWithRemoteAddress(final String socketRemoteIp, final String forwardedRemoteIp) { + final Map<String, Object> attributes = new HashMap<String, Object>(); return (HttpServletRequest) Proxy.newProxyInstance( Jetty12HttpServerTest.class.getClassLoader(), new Class<?>[]{HttpServletRequest.class}, @@ -149,10 +184,20 @@ public class Jetty12HttpServerTest { if ("getRemoteAddr".equals(method.getName())) { return socketRemoteIp; } + if ("getRemoteHost".equals(method.getName())) { + return socketRemoteIp; + } if ("getHeader".equals(method.getName()) && RequestHeader.X_Real_IP.equals(args[0])) { return forwardedRemoteIp; } + if ("setAttribute".equals(method.getName())) { + attributes.put((String) args[0], args[1]); + return null; + } + if ("getAttribute".equals(method.getName())) { + return attributes.get(args[0]); + } return null; }); } diff --git a/test/java/net/yacy/peers/ProtocolTest.java b/test/java/net/yacy/peers/ProtocolTest.java new file mode 100644 index 000000000..9c994ca38 --- /dev/null +++ b/test/java/net/yacy/peers/ProtocolTest.java @@ -0,0 +1,45 @@ +package net.yacy.peers; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.junit.Test; + +public class ProtocolTest { + + @Test + public void transferTargetIpUsesRecordedAttemptAddress() { + final Seed seed = seed(); + seed.setIP("192.0.2.10"); + seed.setIP("2001:db8::10"); + final Map<String, String> response = new HashMap<String, String>(); + response.put(Seed.IP, "2001:db8::10"); + + assertEquals("2001:db8::10", Protocol.transferTargetIP(response, seed)); + } + + @Test + public void transferTargetIpFallsBackToCanonicalIpv6OnlyAddress() { + final Seed seed = seed(); + seed.put(Seed.IP, ""); + seed.put(Seed.IP6, "2001:db8::20"); + + assertEquals("2001:db8::20", Protocol.transferTargetIP(Collections.emptyMap(), seed)); + } + + @Test + public void transferTargetIpIsNullWhenPeerHasNoAddress() { + assertNull(Protocol.transferTargetIP(Collections.emptyMap(), seed())); + } + + private static Seed seed() { + final ConcurrentMap<String, String> dna = new ConcurrentHashMap<String, String>(); + return new Seed("testseedhash", dna); + } +} diff --git a/test/java/net/yacy/server/serverSwitchTest.java b/test/java/net/yacy/server/serverSwitchTest.java new file mode 100644 index 000000000..4d6efe044 --- /dev/null +++ b/test/java/net/yacy/server/serverSwitchTest.java @@ -0,0 +1,66 @@ +package net.yacy.server; + +import static org.junit.Assert.assertEquals; + +import java.io.File; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import net.yacy.search.SwitchboardConstants; + +public class serverSwitchTest { + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void publicPortOverridesUpnpAndLocalPortAndSurvivesReload() throws Exception { + final File root = this.temporaryFolder.newFolder("peer"); + serverSwitch peer = new serverSwitch(root, root, "missing.init", "settings/yacy.conf"); + peer.setConfig(SwitchboardConstants.SERVER_PORT, "8090"); + peer.setConfig(SwitchboardConstants.SERVER_SSLPORT, "8443"); + peer.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, "443"); + peer.setConnectedViaUpnp(true); + peer.setUpnpPorts(SwitchboardConstants.SERVER_PORT, 18090); + peer.setUpnpPorts(SwitchboardConstants.SERVER_SSLPORT, 18443); + + assertEquals(443, peer.getPublicPort(SwitchboardConstants.SERVER_PORT, 8090)); + assertEquals(18443, peer.getPublicPort(SwitchboardConstants.SERVER_SSLPORT, 8443)); + + peer = new serverSwitch(root, root, "missing.init", "settings/yacy.conf"); + assertEquals(443, peer.getPublicPort(SwitchboardConstants.SERVER_PORT, 8090)); + } + + @Test + public void missingOrInvalidPublicPortFallsBackToUpnpThenLocalPort() throws Exception { + final File root = this.temporaryFolder.newFolder("fallback-peer"); + serverSwitch peer = + new serverSwitch(root, root, "missing.init", "settings/yacy.conf"); + peer.setConfig(SwitchboardConstants.SERVER_PORT, "8090"); + peer.setConnectedViaUpnp(true); + peer.setUpnpPorts(SwitchboardConstants.SERVER_PORT, 18090); + + peer.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, "443"); + assertEquals(443, peer.getPublicPort(SwitchboardConstants.SERVER_PORT, 8090)); + + peer.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, ""); + assertEquals(18090, peer.getPublicPort(SwitchboardConstants.SERVER_PORT, 8090)); + + peer.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, "0"); + assertEquals(18090, peer.getPublicPort(SwitchboardConstants.SERVER_PORT, 8090)); + + peer.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, "65535"); + assertEquals(65535, peer.getPublicPort(SwitchboardConstants.SERVER_PORT, 8090)); + + peer.setConfig(SwitchboardConstants.SERVER_PUBLICPORT, "65536"); + assertEquals(18090, peer.getPublicPort(SwitchboardConstants.SERVER_PORT, 8090)); + + peer.setConnectedViaUpnp(false); + assertEquals(8090, peer.getPublicPort(SwitchboardConstants.SERVER_PORT, 8090)); + + peer = new serverSwitch(root, root, "missing.init", "settings/yacy.conf"); + assertEquals(8090, peer.getPublicPort(SwitchboardConstants.SERVER_PORT, 8090)); + } +} |