diff options
33 files changed, 1077 insertions, 130 deletions
diff --git a/.classpath b/.classpath index 495d71af5..3430bee69 100644 --- a/.classpath +++ b/.classpath @@ -22,5 +22,7 @@ <classpathentry kind="lib" path="lib/solr9-bridge-http2-client-9.4.58.v20250814.jar"/> <classpathentry kind="lib" path="lib/solr9-bridge-http2-common-9.4.58.v20250814.jar"/> <classpathentry kind="lib" path="lib/solr9-bridge-http2-http-client-transport-9.4.58.v20250814.jar"/> + <classpathentry kind="lib" path="lib/solr9-bridge-slf4j-api-1.7.36.jar"/> + <classpathentry kind="lib" path="lib/solr9-bridge-slf4j-jdk14-1.7.36.jar"/> <classpathentry kind="output" path="gen"/> </classpath> diff --git a/.externalToolBuilders/solr9-bridge.launch b/.externalToolBuilders/solr9-bridge.launch index 012878bbd..1cfa5a6f2 100644 --- a/.externalToolBuilders/solr9-bridge.launch +++ b/.externalToolBuilders/solr9-bridge.launch @@ -1,10 +1,14 @@ <?xml version="1.0" encoding="UTF-8" standalone="no"?> <launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType"> - <booleanAttribute key="org.eclipse.ant.ui.ATTR_TARGETS_UPDATED" value="true"/> <stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_FULL_TARGETS" value="build-solr9-bridge,"/> <stringAttribute key="org.eclipse.ant.ui.ATTR_ANT_MANUAL_TARGETS" value="build-solr9-bridge,"/> - <booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/> + <booleanAttribute key="org.eclipse.ant.ui.ATTR_TARGETS_UPDATED" value="true"/> + <booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="false"/> + <stringAttribute key="org.eclipse.jdt.launching.CLASSPATH_PROVIDER" value="org.eclipse.ant.ui.AntClasspathProvider"/> + <booleanAttribute key="org.eclipse.jdt.launching.DEFAULT_CLASSPATH" value="true"/> + <stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${project_loc}/build.xml"/> + <stringAttribute key="org.eclipse.ui.externaltools.ATTR_REFRESH_SCOPE" value="${project}"/> <stringAttribute key="org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS" value="full,"/> - <stringAttribute key="org.eclipse.ui.externaltools.ATTR_LOCATION" value="${workspace_loc:/yacy/build.xml}"/> - <stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${workspace_loc:/yacy}"/> + <booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/> + <stringAttribute key="org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY" value="${project_loc}"/> </launchConfiguration> diff --git a/JETTY12_MIGRATION.md b/JETTY12_MIGRATION.md index 72d867f98..22122cfa4 100644 --- a/JETTY12_MIGRATION.md +++ b/JETTY12_MIGRATION.md @@ -167,6 +167,51 @@ Jetty version on the resolved classpath and the private Solr island: 6. Proxy traffic including CONNECT 7. `test/jetty-smoke-test.sh` +The live gates are split by the environment they require: + +- `test/jetty-smoke-test.sh` checks HTTP methods, ranges, conditional requests, + error dispatch, and gzip request/response handling against a running peer; +- `test/jetty-auth-smoke-test.sh` checks localhost, `bin/apicall.sh`, optional + credentials, and an optional real non-loopback path; +- `test/jetty-peer-start-smoke-test.sh` starts and stops an isolated peer through + explicit harness commands and queries its embedded Solr core; +- `test/remote-solr-smoke-test.sh` queries an explicitly configured external + Solr instance through YaCy's Apache-HttpClient-backed `RemoteInstance`; +- `test/proxy-smoke-test.sh` checks HTTP proxy traffic and an HTTPS CONNECT + tunnel against explicitly configured controlled targets. + +The environment-dependent gates exit with status 2 when their required target +or isolated-peer harness has not been supplied. This is a reported skip, not a +successful verification. + +The final migration acceptance must run the authentication gate with no skips: + +```sh +YACY_SMOKE_REQUIRE_COMPLETE=true \ +YACY_SMOKE_ADMIN_USER=admin \ +YACY_SMOKE_ADMIN_PASSWORD='the configured password' \ +YACY_SMOKE_REMOTE_BASE_URL='http://a-real-non-loopback-peer-address:8090' \ +test/jetty-auth-smoke-test.sh +``` + +The HTTP range contract includes a single satisfiable range (`206`), multiple +satisfiable ranges as `multipart/byteranges`, and an unsatisfiable range +(`416`). + +### Switch-time logging tests + +`Slf4jJulBridgeTest` is version-neutral and must pass both before and after the +server switch. It proves that the public SLF4J 2 provider routes the +`org.eclipse.jetty` logger namespace into `java.util.logging` and therefore the +YaCy logging configuration. + +`Jetty9LoggingFacadeTest` is deliberately a Jetty 9 baseline test. It imports +Jetty 9's removed `org.eclipse.jetty.util.log.Log` API and asserts the old +`Slf4jLog` facade. Remove it together with `Jetty9HttpServerImpl` during the +switch and replace it with a Jetty 12 integration test that starts and stops a +real server while capturing an `org.eclipse.jetty` record through JUL. The +Jetty 12 test must not assert an internal logger implementation class. + The following implementation phase may then replace `Jetty9HttpServerImpl` and the remaining Jetty adapter APIs without changing the Solr dependency graph. @@ -201,3 +246,88 @@ The portable address/path syntax of `serverClient` is represented by `InetPathAccessRule`. `InetPathAccessHandler` remains the Jetty 9 matcher adapter; Jetty 12 can consume the normalized `address|path` rules with its native path-aware access handler. + +## P2.3 Handler Boundaries + +Proxy request processing and cache processing no longer receive Jetty's +`Request`. `RequestCompletion` is the container-neutral signal that processing +is complete; `AbstractRemoteHandler` adapts it to Jetty 9's +`Request.setHandled(true)`. Consequently `ProxyHandler` and +`ProxyCacheHandler` have no Jetty imports. + +The `proxyClient` regular-expression list is evaluated by the pure +`ProxyAccessPolicy`. The local virtual-host cache used by proxy detection is a +concurrent set because it is populated by both the discovery thread and +request threads. + +The remaining Jetty handler classes now have explicit migration roles: + +| Jetty 9 adapter | Responsibility to reproduce with Jetty 12 | +| --- | --- | +| `AbstractRemoteHandler` | detect proxy traffic and delegate CONNECT tunnelling | +| `CrashProtectionHandler` | outer exception barrier around proxy and servlet handlers | +| `YacyDomainHandler` | rewrite `.yacy` destinations and redispatch into the proxy chain | +| `YaCyErrorHandler` | render the container error page | +| `YaCyQoSFilter` | optional request prioritization when enabled in `web.xml` | + +These classes intentionally remain container adapters. They must be ported +against the corresponding Jetty 12 APIs rather than replaced with servlet-only +approximations that would change CONNECT, error dispatch, or prioritization. + +## P2.4 Embedded Server Bootstrap Contract + +`HttpServerBootstrapConfig` is the common immutable input for Jetty 9 and the +future Jetty 12 implementation. It fixes the following startup values: + +| Concern | Contract | +| --- | --- | +| HTTP binding | constructor host and port | +| Acceptor threads | half the available processors, clamped to 1 through 4 | +| Request header limit | 16,384 bytes | +| Connector idle timeout | 9,000 ms | +| HTTP accept queue | 128 | +| HTTPS | `server.https`, configured SSL port, initialized SSL context only | +| Web root | configured `htRootPath` below the application directory | +| Descriptors | `defaults/web.xml`, optionally `DATA/SETTINGS/web.xml` | +| Request decompression | Gzip inflate buffer of 4,096 bytes | +| Response compression | controlled by `server.response.compress.gzip` | +| Form limit | unlimited at the proxy-handler context boundary | +| Proxy handlers | present only when transparent proxy is enabled | +| Network access | configured `serverClient` address/path rules plus loopback | +| Authentication realm | configured administrator realm, unchanged for DIGEST hashes | + +TLS preparation remains a YaCy bootstrap responsibility because it may import +a configured PKCS#12 file, create/update the JKS file, clear the one-shot +import settings, and construct the JDK `SSLContext`. The container adapter only +attaches that context to its HTTPS connector. + +The request pipeline order is a behavioral requirement: + +1. optional server-client address/path gate; +2. outer crash-protection barrier; +3. `.yacy` domain rewrite; +4. cached proxy response, when transparent proxy is enabled; +5. live HTTP proxy and CONNECT tunnel, when enabled; +6. root web application with monitor filter, admin security, gzip/inflate, and + `YaCyDefaultServlet`; +7. container default handler for requests left unhandled. + +The connection-close listener must remove the matching `ConnectionInfo` entry +created by `MonitorFilter`. The default servlet and monitor filter remain +hard-coded mandatory components; additional servlet mappings come from the +merged web descriptors. + +`YaCyHttpServer` defines the runtime contract used outside the adapter: + +- synchronous start; +- synchronous stop followed by join; +- asynchronous delayed port reconnect without rebuilding the handler graph; +- HTTPS availability and bound-port reporting; +- administrator identity eviction/reload after credential changes; +- container version reporting; +- current non-idle worker-thread count. + +A Jetty 12 implementation must first be added beside `Jetty9HttpServerImpl` +and satisfy this complete contract before the construction site in `yacy.java` +is switched. No caller outside the HTTP package should need a Jetty type or a +Jetty-version condition. @@ -38,6 +38,7 @@ <property name="lib" location="lib"/> <property name="libt" location="libt"/> <property name="build" location="build/classes/java/main"/> <!-- reuse Gradle build path --> + <property name="ivy.compile.stage" location="build/ivy-retrieve/compile"/> <property name="solr9.bridge.build" location="build/solr9-bridge"/> <property name="solr9.bridge.input" location="${solr9.bridge.build}/input"/> <property name="solr9.bridge.tool" location="${solr9.bridge.build}/tool"/> @@ -93,7 +94,23 @@ We can move there in a following step. ${lib} just happens to be equal to ${ivy.lib.dir}. --> - <ivy:retrieve conf="compile" pathid="compile.path" pattern="${ivy.lib.dir}/[artifact]-[revision].[ext]" /> + <!-- + Ivy retrieve does not remove stale revisions or replaced coordinates. + Resolve the complete public classpath into staging before touching lib/; + a failed resolve therefore leaves the last usable runtime intact. Only + after staging succeeds are stale public Jetty/Servlet jars removed and + the resolved classpath published. Private Solr jars use the distinct + solr9-bridge-* prefix and are rebuilt by build-solr9-bridge. + --> + <delete dir="${ivy.compile.stage}" failonerror="false" /> + <mkdir dir="${ivy.compile.stage}" /> + <ivy:retrieve conf="compile" pathid="compile.path" pattern="${ivy.compile.stage}/[artifact]-[revision].[ext]" /> + <delete failonerror="false"> + <fileset dir="${lib}" includes="javax.servlet-api-*.jar,jetty-*.jar" /> + </delete> + <copy todir="${ivy.lib.dir}" overwrite="true"> + <fileset dir="${ivy.compile.stage}" includes="**/*" /> + </copy> <ivy:retrieve conf="test" pathid="test.path" pattern="${libt}/[artifact]-[revision].[ext]" /> <ivy:retrieve conf="solr9-bridge" pathid="solr9.bridge.input.path" pattern="${solr9.bridge.input}/[artifact]-[revision].[ext]" /> <ivy:retrieve conf="solr9-bridge-tool" pathid="solr9.bridge.tool.path" pattern="${solr9.bridge.tool.libs}/[artifact]-[revision].[ext]" /> @@ -180,10 +197,10 @@ <fileset dir="${lib}" includes="solr9-bridge-*.jar" /> </path> - <target name="build-solr9-bridge" depends="init,resolve" description="build the private Solr 9.0 Jetty client island"> + <target name="build-solr9-bridge" depends="init,resolve" description="build the private Solr 9.0 Jetty and SLF4J 1.7 island"> <delete dir="${solr9.bridge.tool}" failonerror="false"/> <delete failonerror="false"> - <fileset dir="${lib}" includes="solr9-bridge-*.jar,solr-core-9.0.0.jar,solr-solrj-9.0.0.jar,solr-scripting-9.0.0.jar,http2-*.jar" /> + <fileset dir="${lib}" includes="solr9-bridge-*.jar,solr-core-9.0.0.jar,solr-solrj-9.0.0.jar,solr-scripting-9.0.0.jar,http2-*.jar,slf4j-*-1.7.36.jar" /> </delete> <mkdir dir="${solr9.bridge.tool}"/> <javac srcdir="test/jetty" destdir="${solr9.bridge.tool}" @@ -450,6 +467,7 @@ <fileset dir="." includes="TEST-*" /> </delete> <delete dir="test/DATA" failonerror="false"/> + <delete dir="${ivy.compile.stage}" failonerror="false"/> <delete dir="${solr9.bridge.build}" failonerror="false"/> </target> @@ -31,7 +31,12 @@ <dependency org="io.dropwizard.metrics" name="metrics-jmx" rev="4.2.39" conf="compile->master"/> <dependency org="io.opentracing" name="opentracing-noop" rev="0.33.0"/> <dependency org="io.opentracing" name="opentracing-util" rev="0.33.0"/> - <dependency org="javax.servlet" name="javax.servlet-api" rev="4.0.1"/> + <!-- + Jetty's EE8-compatible Servlet 4 API also contains the web.xml schemas + used by WebAppContext. Keep it as the single public javax.servlet API + before and after the Jetty 12 migration. + --> + <dependency org="org.eclipse.jetty.toolchain" name="jetty-servlet-api" rev="4.0.9" conf="compile->master"/> <dependency org="javainetlocator" name="inetaddresslocator" rev="2.18" /> <dependency org="eu.agno3.jcifs" name="jcifs-ng" rev="2.1.10" /> <dependency org="net.arnx" name="jsonic" rev="1.3.10"/> @@ -115,11 +120,18 @@ </dependency> <dependency org="org.locationtech.spatial4j" name="spatial4j" rev="0.8"/> <dependency org="org.rrd4j" name="rrd4j" rev="3.10"/> - <!-- Solr 9.0.0 directly references the SLF4J 1.7 StaticLoggerBinder API. --> - <dependency org="org.slf4j" name="slf4j-api" rev="1.7.36" force="true"/> - <dependency org="org.slf4j" name="jcl-over-slf4j" rev="1.7.36"/> - <dependency org="org.slf4j" name="log4j-over-slf4j" rev="1.7.36"/> - <dependency org="org.slf4j" name="slf4j-jdk14" rev="1.7.36"/> + <!-- + YaCy and the public Jetty runtime use SLF4J 2 with JUL as their single + provider. Solr 9.0.0 directly references the removed SLF4J 1.7 + StaticLoggerBinder API, so its API and JUL binding are relocated into + the private Solr bridge instead of being exposed on this classpath. + --> + <dependency org="org.slf4j" name="slf4j-api" rev="2.0.17" conf="compile->master" force="true"/> + <dependency org="org.slf4j" name="jcl-over-slf4j" rev="2.0.17" conf="compile->master"/> + <dependency org="org.slf4j" name="log4j-over-slf4j" rev="2.0.17" conf="compile->master"/> + <dependency org="org.slf4j" name="slf4j-jdk14" rev="2.0.17" conf="compile->master"/> + <dependency org="org.slf4j" name="slf4j-api" rev="1.7.36" conf="solr9-bridge->master"/> + <dependency org="org.slf4j" name="slf4j-jdk14" rev="1.7.36" conf="solr9-bridge->master"/> <!-- This does not match langdetect.jar from pre-ivy --> <dependency org="org.tukaani" name="xz" rev="1.12"/> <dependency org="oro" name="oro" rev="2.0.8"/> @@ -131,5 +143,7 @@ <dependency org="org.ow2.asm" name="asm-commons" rev="9.9" conf="solr9-bridge-tool->master"/> <dependency org="com.github.luben" name="zstd-jni" rev="1.5.7-11"></dependency> <dependency org="ai.djl" name ="api" rev="0.36.0"></dependency> + <!-- Different coordinates prevent Ivy from evicting duplicate Servlet API jars. --> + <exclude org="javax.servlet" module="javax.servlet-api"/> </dependencies> </ivy-module> diff --git a/source/net/yacy/http/AbstractRemoteHandler.java b/source/net/yacy/http/AbstractRemoteHandler.java index 826162c98..8f791c2a7 100644 --- a/source/net/yacy/http/AbstractRemoteHandler.java +++ b/source/net/yacy/http/AbstractRemoteHandler.java @@ -26,10 +26,9 @@ package net.yacy.http; import java.io.IOException; import java.net.InetAddress; -import java.util.HashSet; import java.util.Locale; import java.util.Set; -import java.util.StringTokenizer; +import java.util.concurrent.ConcurrentHashMap; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -42,17 +41,16 @@ import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import org.eclipse.jetty.proxy.ConnectHandler; -import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Request; /** * abstract jetty http handler * only request to remote hosts (proxy requests) are processed by derived classes */ -abstract public class AbstractRemoteHandler extends ConnectHandler implements Handler { +abstract public class AbstractRemoteHandler extends ConnectHandler { protected Switchboard sb = null; - private final Set<String> localVirtualHostNames = new HashSet<String>(); // list for quick check for req to local peer + private final Set<String> localVirtualHostNames = ConcurrentHashMap.newKeySet(); // updated by discovery thread and request threads @Override protected void doStart() throws Exception { @@ -99,7 +97,7 @@ abstract public class AbstractRemoteHandler extends ConnectHandler implements Ha }.start(); } - abstract public void handleRemote(String target, Request baseRequest, HttpServletRequest request, + abstract public void handleRemote(String target, RequestCompletion completion, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException; @Override @@ -135,7 +133,8 @@ abstract public class AbstractRemoteHandler extends ConnectHandler implements Ha } final String remoteHost = request.getRemoteHost(); - if (!proxyippatternmatch(remoteHost)) { + if (!ProxyAccessPolicy.isClientAllowed( + Switchboard.getSwitchboard().getConfig("proxyClient", "*"), remoteHost)) { // TODO: handle proxy account response.sendError(HttpServletResponse.SC_FORBIDDEN, "proxy use not granted for IP " + remoteHost + " (see Advanced Settings -> Proxy Access Settings -> IP-Number filter)."); @@ -157,28 +156,8 @@ abstract public class AbstractRemoteHandler extends ConnectHandler implements Ha return; } - handleRemote(target, baseRequest, request, response); + handleRemote(target, () -> baseRequest.setHandled(true), request, response); } - /** - * helper for proxy IP config pattern check - */ - private boolean proxyippatternmatch(final String key) { - // the cfgippattern is a comma-separated list of patterns - // each pattern may contain one wildcard-character '*' which matches anything - final String cfgippattern = Switchboard.getSwitchboard().getConfig("proxyClient", "*"); - if (cfgippattern.equals("*")) { - return true; - } - final StringTokenizer st = new StringTokenizer(cfgippattern, ","); - String pattern; - while (st.hasMoreTokens()) { - pattern = st.nextToken(); - if (key.matches(pattern)) { - return true; - } - } - return false; - } } diff --git a/source/net/yacy/http/AdminAccessPolicy.java b/source/net/yacy/http/AdminAccessPolicy.java index e4bad1350..5560200ec 100644 --- a/source/net/yacy/http/AdminAccessPolicy.java +++ b/source/net/yacy/http/AdminAccessPolicy.java @@ -1,3 +1,23 @@ +/** + * AdminAccessPolicy + * Copyright 2026 by Michael Peter Christen + * First released 12.07.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + package net.yacy.http; import java.net.MalformedURLException; diff --git a/source/net/yacy/http/AdminAuthenticationContext.java b/source/net/yacy/http/AdminAuthenticationContext.java index e276d4cbd..5080c9b05 100644 --- a/source/net/yacy/http/AdminAuthenticationContext.java +++ b/source/net/yacy/http/AdminAuthenticationContext.java @@ -1,3 +1,23 @@ +/** + * AdminAuthenticationContext + * Copyright 2026 by Michael Peter Christen + * First released 12.07.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + package net.yacy.http; import net.yacy.cora.protocol.Domains; diff --git a/source/net/yacy/http/CrashProtectionHandler.java b/source/net/yacy/http/CrashProtectionHandler.java index 1cbf27bcd..1456ba778 100644 --- a/source/net/yacy/http/CrashProtectionHandler.java +++ b/source/net/yacy/http/CrashProtectionHandler.java @@ -1,3 +1,23 @@ +/** + * CrashProtectionHandler + * Copyright 2026 by Michael Peter Christen + * First released 12.07.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + package net.yacy.http; import java.io.IOException; @@ -8,7 +28,6 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.eclipse.jetty.server.Handler; -import org.eclipse.jetty.server.HandlerContainer; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.handler.HandlerWrapper; @@ -25,7 +44,7 @@ import net.yacy.cora.util.ConcurrentLog; * trace. Its purpose is to catch failures outside the servlet context, e.g. * in the transparent proxy handlers. */ -public class CrashProtectionHandler extends HandlerWrapper implements Handler, HandlerContainer { +public class CrashProtectionHandler extends HandlerWrapper implements Handler { public CrashProtectionHandler() { super(); diff --git a/source/net/yacy/http/HttpServerBootstrapConfig.java b/source/net/yacy/http/HttpServerBootstrapConfig.java new file mode 100644 index 000000000..f8ee311ef --- /dev/null +++ b/source/net/yacy/http/HttpServerBootstrapConfig.java @@ -0,0 +1,103 @@ +/** + * HttpServerBootstrapConfig + * Copyright 2026 by Michael Peter Christen + * First released 12.07.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.http; + +import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; + +/** Immutable, servlet-container-neutral input for the embedded HTTP server. */ +public final class HttpServerBootstrapConfig { + + public static final int REQUEST_HEADER_SIZE = 16_384; + public static final long CONNECTOR_IDLE_TIMEOUT_MILLIS = 9_000L; + public static final int ACCEPT_QUEUE_SIZE = 128; + public static final int REQUEST_INFLATE_BUFFER_SIZE = 4_096; + public static final int MAX_FORM_CONTENT_SIZE = -1; + + private final int httpPort; + private final String bindHost; + private final int acceptorCount; + private final boolean httpsEnabled; + private final int httpsPort; + private final String htrootPath; + private final String defaultsWebXml; + private final String overrideWebXml; + private final boolean gzipResponsesEnabled; + private final boolean transparentProxyEnabled; + private final String serverClientRules; + private final String adminRealm; + + private HttpServerBootstrapConfig(final int httpPort, final String bindHost, + final int acceptorCount, final boolean httpsEnabled, final int httpsPort, + final String htrootPath, final String defaultsWebXml, final String overrideWebXml, + final boolean gzipResponsesEnabled, final boolean transparentProxyEnabled, + final String serverClientRules, final String adminRealm) { + this.httpPort = httpPort; + this.bindHost = bindHost; + this.acceptorCount = acceptorCount; + this.httpsEnabled = httpsEnabled; + this.httpsPort = httpsPort; + this.htrootPath = htrootPath; + this.defaultsWebXml = defaultsWebXml; + this.overrideWebXml = overrideWebXml; + this.gzipResponsesEnabled = gzipResponsesEnabled; + this.transparentProxyEnabled = transparentProxyEnabled; + this.serverClientRules = serverClientRules; + this.adminRealm = adminRealm; + } + + public static HttpServerBootstrapConfig from(final Switchboard switchboard, + final int httpPort, final String bindHost) { + final int cores = Runtime.getRuntime().availableProcessors(); + return new HttpServerBootstrapConfig( + httpPort, + bindHost, + acceptorCountFor(cores), + switchboard.getConfigBool("server.https", false), + switchboard.getConfigInt(SwitchboardConstants.SERVER_SSLPORT, 8443), + switchboard.appPath + "/" + switchboard.getConfig( + SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT), + switchboard.appPath + "/defaults/web.xml", + switchboard.dataPath + "/DATA/SETTINGS/web.xml", + switchboard.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP, + SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT), + switchboard.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false), + switchboard.getConfig("serverClient", "*"), + switchboard.getConfig(SwitchboardConstants.ADMIN_REALM, "YaCy")); + } + + static int acceptorCountFor(final int availableProcessors) { + return Math.max(1, Math.min(4, availableProcessors / 2)); + } + + public int httpPort() { return this.httpPort; } + public String bindHost() { return this.bindHost; } + public int acceptorCount() { return this.acceptorCount; } + public boolean httpsEnabled() { return this.httpsEnabled; } + public int httpsPort() { return this.httpsPort; } + public String htrootPath() { return this.htrootPath; } + public String defaultsWebXml() { return this.defaultsWebXml; } + public String overrideWebXml() { return this.overrideWebXml; } + public boolean gzipResponsesEnabled() { return this.gzipResponsesEnabled; } + public boolean transparentProxyEnabled() { return this.transparentProxyEnabled; } + public String serverClientRules() { return this.serverClientRules; } + public String adminRealm() { return this.adminRealm; } +} diff --git a/source/net/yacy/http/InetPathAccessRule.java b/source/net/yacy/http/InetPathAccessRule.java index f93f6336b..c93200829 100644 --- a/source/net/yacy/http/InetPathAccessRule.java +++ b/source/net/yacy/http/InetPathAccessRule.java @@ -1,3 +1,23 @@ +/** + * InetPathAccessRule + * Copyright 2026 by Michael Peter Christen + * First released 12.07.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + package net.yacy.http; /** Container-neutral representation of a server-client address/path rule. */ diff --git a/source/net/yacy/http/Jetty9HttpServerImpl.java b/source/net/yacy/http/Jetty9HttpServerImpl.java index 5bf942e06..137330acc 100644 --- a/source/net/yacy/http/Jetty9HttpServerImpl.java +++ b/source/net/yacy/http/Jetty9HttpServerImpl.java @@ -52,7 +52,6 @@ import org.eclipse.jetty.server.handler.InetAccessHandler; import org.eclipse.jetty.server.handler.gzip.GzipHandler; import org.eclipse.jetty.servlet.FilterHolder; import org.eclipse.jetty.servlet.ServletHolder; -import org.eclipse.jetty.util.log.Log; import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.webapp.WebAppContext; @@ -78,6 +77,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { */ public Jetty9HttpServerImpl(final int port, final String host) { final Switchboard sb = Switchboard.getSwitchboard(); + final HttpServerBootstrapConfig bootstrap = HttpServerBootstrapConfig.from(sb, port, host); this.server = new Server(); @@ -96,32 +96,29 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { } }; - final int cores = Runtime.getRuntime().availableProcessors(); - final int acceptors = Math.max(1, Math.min(4, cores/2)); // original: Math.max(1, Math.min(4,cores/8)); - final HttpConfiguration httpConfig = new HttpConfiguration(); - httpConfig.setRequestHeaderSize(16384); + httpConfig.setRequestHeaderSize(HttpServerBootstrapConfig.REQUEST_HEADER_SIZE); final HttpConnectionFactory hcf = new HttpConnectionFactory(httpConfig); - final ServerConnector connector = new ServerConnector(this.server, null, null, null, acceptors, -1, hcf); - connector.setPort(port); - connector.setHost(host); - connector.setName("httpd-" + host + ":" + Integer.toString(port)); - connector.setIdleTimeout(9000); // timout in ms when no bytes send / received - connector.setAcceptQueueSize(128); + final ServerConnector connector = new ServerConnector(this.server, null, null, null, bootstrap.acceptorCount(), -1, hcf); + connector.setPort(bootstrap.httpPort()); + connector.setHost(bootstrap.bindHost()); + connector.setName("httpd-" + bootstrap.bindHost() + ":" + Integer.toString(bootstrap.httpPort())); + connector.setIdleTimeout(HttpServerBootstrapConfig.CONNECTOR_IDLE_TIMEOUT_MILLIS); + connector.setAcceptQueueSize(HttpServerBootstrapConfig.ACCEPT_QUEUE_SIZE); connector.addBean(connectionCloseMonitor); this.server.addConnector(connector); // add ssl/https connector - final boolean useSSL = sb.getConfigBool("server.https", false); + final boolean useSSL = bootstrap.httpsEnabled(); if (useSSL) { final SslContextFactory sslContextFactory = new SslContextFactory.Server(); final SSLContext sslContext = this.initSslContext(sb); if (sslContext != null) { - final int sslport = sb.getConfigInt(SwitchboardConstants.SERVER_SSLPORT, 8443); + final int sslport = bootstrap.httpsPort(); sslContextFactory.setSslContext(sslContext); // SSL HTTP Configuration @@ -134,7 +131,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { new HttpConnectionFactory(https_config)); sslConnector.setPort(sslport); sslConnector.setName("ssld:" + Integer.toString(sslport)); // name must start with ssl (for withSSL() to work correctly) - sslConnector.setIdleTimeout(9000); // timout in ms when no bytes send / received + sslConnector.setIdleTimeout(HttpServerBootstrapConfig.CONNECTOR_IDLE_TIMEOUT_MILLIS); sslConnector.addBean(connectionCloseMonitor); this.server.addConnector(sslConnector); @@ -148,7 +145,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { // configure root context final WebAppContext htrootContext = new WebAppContext(); htrootContext.setContextPath("/"); - final String htrootpath = sb.appPath + "/" + sb.getConfig(SwitchboardConstants.HTROOT_PATH, SwitchboardConstants.HTROOT_PATH_DEFAULT); + final String htrootpath = bootstrap.htrootPath(); ConcurrentLog.info("Jetty9HttpServerImpl", "htrootpath = " + htrootpath); htrootContext.setErrorHandler(new YaCyErrorHandler()); // handler for custom error page try { @@ -158,8 +155,8 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { // make use of Jetty feature to define web.xml other as default WEB-INF/web.xml // and to use a DefaultsDescriptor merged with a individual web.xml // use defaults/web.xml as default and look in DATA/SETTINGS for local addition/changes - htrootContext.setDefaultsDescriptor(sb.appPath + "/defaults/web.xml"); - final Resource webxml = Resource.newResource(sb.dataPath + "/DATA/SETTINGS/web.xml"); + htrootContext.setDefaultsDescriptor(bootstrap.defaultsWebXml()); + final Resource webxml = Resource.newResource(bootstrap.overrideWebXml()); if (webxml.exists()) { htrootContext.setDescriptor(webxml.getName()); } @@ -191,10 +188,9 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { * APIs /yacy/transferRWI.html and /yacy/transferURL.html This was previously * handled by a GZIPRequestWrapper in the YaCyDefaultServlet. */ - gzipHandler.setInflateBufferSize(4096); + gzipHandler.setInflateBufferSize(HttpServerBootstrapConfig.REQUEST_INFLATE_BUFFER_SIZE); - if (!sb.getConfigBool(SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP, - SwitchboardConstants.SERVER_RESPONSE_COMPRESS_GZIP_DEFAULT)) { + if (!bootstrap.gzipResponsesEnabled()) { /* Gzip compression of responses can be disabled by user configuration */ gzipHandler.setExcludedMethods(HttpMethod.GET.asString(), HttpMethod.POST.asString()); } @@ -224,7 +220,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { // define list of YaCy specific general handlers final HandlerList handlers = new HandlerList(); - if (sb.getConfigBool(SwitchboardConstants.PROXY_TRANSPARENT_PROXY, false)) { + if (bootstrap.transparentProxyEnabled()) { // Proxyhandlers are only needed if feature activated (save resources if not used) ConcurrentLog.info("SERVER", "load Jetty handler for transparent proxy"); handlers.setHandlers(new Handler[]{domainHandler, new ProxyCacheHandler(), new ProxyHandler()}); @@ -236,9 +232,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { context.setServer(this.server); context.setContextPath("/"); context.setHandler(handlers); - context.setMaxFormContentSize(-1); - final org.eclipse.jetty.util.log.Logger log = Log.getRootLogger(); - context.setLogger(log); + context.setMaxFormContentSize(HttpServerBootstrapConfig.MAX_FORM_CONTENT_SIZE); // make YaCy handlers (in context) and servlet context handlers available (both contain root context "/") // logic: 1. YaCy handlers are called if request not handled (e.g. proxy) then servlets handle it final ContextHandlerCollection allrequesthandlers = new ContextHandlerCollection(); @@ -250,7 +244,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { final YaCyLoginService loginService = new YaCyLoginService(); // This is part of the built-in administrator's DIGEST password hash. // Changing it invalidates the configured administrator password hash. - loginService.setName(sb.getConfig(SwitchboardConstants.ADMIN_REALM,"YaCy")); + loginService.setName(bootstrap.adminRealm()); final YaCySecurityHandler securityHandler = new YaCySecurityHandler(); securityHandler.setLoginService(loginService); @@ -261,7 +255,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer { final Handler crashHandler = new CrashProtectionHandler(this.server, allrequesthandlers); // check server access restriction and add InetAccessHandler if restrictions are needed // otherwise don't (to save performance) - final String white = sb.getConfig("serverClient", "*"); + final String white = bootstrap.serverClientRules(); if (!white.equals("*")) { // full ip (allowed ranges 0-255 or prefix 10.0-255,0,0-100 or CIDR notation 192.168.1.0/24) final StringTokenizer st = new StringTokenizer(white, ","); final InetAccessHandler whiteListHandler; diff --git a/source/net/yacy/http/ProxyAccessPolicy.java b/source/net/yacy/http/ProxyAccessPolicy.java new file mode 100644 index 000000000..f0fab158d --- /dev/null +++ b/source/net/yacy/http/ProxyAccessPolicy.java @@ -0,0 +1,43 @@ +/** + * ProxyAccessPolicy + * Copyright 2026 by Michael Peter Christen + * First released 12.07.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.http; + +/** Container-neutral checks for the configured transparent-proxy client list. */ +public final class ProxyAccessPolicy { + + private ProxyAccessPolicy() { + } + + public static boolean isClientAllowed(final String configuredPatterns, final String clientHost) { + if ("*".equals(configuredPatterns)) { + return true; + } + if (configuredPatterns == null || configuredPatterns.isEmpty() || clientHost == null) { + return false; + } + for (final String pattern : configuredPatterns.split(",")) { + if (!pattern.isEmpty() && clientHost.matches(pattern)) { + return true; + } + } + return false; + } +} diff --git a/source/net/yacy/http/ProxyCacheHandler.java b/source/net/yacy/http/ProxyCacheHandler.java index 7c8597795..27e043191 100644 --- a/source/net/yacy/http/ProxyCacheHandler.java +++ b/source/net/yacy/http/ProxyCacheHandler.java @@ -31,8 +31,6 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.eclipse.jetty.server.Handler; -import org.eclipse.jetty.server.Request; import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.protocol.RequestHeader; @@ -43,7 +41,7 @@ import net.yacy.crawler.retrieval.Response; /** * jetty http handler serves pages from cache if available and valid */ -public class ProxyCacheHandler extends AbstractRemoteHandler implements Handler { +public class ProxyCacheHandler extends AbstractRemoteHandler { private void handleRequestFromCache(@SuppressWarnings("unused") HttpServletRequest request, HttpServletResponse response, ResponseHeader cachedResponseHeader, byte[] content) throws IOException { @@ -57,7 +55,7 @@ public class ProxyCacheHandler extends AbstractRemoteHandler implements Handler } @Override - public void handleRemote(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { + public void handleRemote(String target, RequestCompletion completion, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { if (request.getMethod().equals("GET")) { String queryString = request.getQueryString() != null ? "?" + request.getQueryString() : ""; DigestURL url = new DigestURL(request.getRequestURL().toString() + queryString); @@ -86,7 +84,7 @@ public class ProxyCacheHandler extends AbstractRemoteHandler implements Handler byte[] cacheContent = Cache.getContent(url.hash()); if (cacheContent != null && cachedResponse.isFreshForProxy()) { handleRequestFromCache(request, response, cachedResponseHeader, cacheContent); - baseRequest.setHandled(true); + completion.complete(); } } diff --git a/source/net/yacy/http/ProxyHandler.java b/source/net/yacy/http/ProxyHandler.java index 95f4dca56..9c86a6b09 100644 --- a/source/net/yacy/http/ProxyHandler.java +++ b/source/net/yacy/http/ProxyHandler.java @@ -52,14 +52,12 @@ import net.yacy.server.http.MultiOutputStream; import org.apache.http.Header; import org.apache.http.HttpResponse; -import org.eclipse.jetty.server.Handler; -import org.eclipse.jetty.server.Request; /** * jetty http handler * proxies request, caches responses and adds urls to crawler */ -public class ProxyHandler extends AbstractRemoteHandler implements Handler { +public class ProxyHandler extends AbstractRemoteHandler { protected int timeout = 10000; @@ -124,7 +122,7 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler { } @Override - public void handleRemote(String target, Request baseRequest, HttpServletRequest request, + public void handleRemote(String target, RequestCompletion completion, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { sb.proxyLastAccess = System.currentTimeMillis(); @@ -222,7 +220,7 @@ public class ProxyHandler extends AbstractRemoteHandler implements Handler { // we handled this request, break out of handler chain logProxyAccess(request); - baseRequest.setHandled(true); + completion.complete(); } /** diff --git a/source/net/yacy/http/RequestCompletion.java b/source/net/yacy/http/RequestCompletion.java new file mode 100644 index 000000000..7dce04d59 --- /dev/null +++ b/source/net/yacy/http/RequestCompletion.java @@ -0,0 +1,28 @@ +/** + * RequestCompletion + * Copyright 2026 by Michael Peter Christen + * First released 12.07.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + +package net.yacy.http; + +/** Container-neutral callback used when a handler has completed a request. */ +@FunctionalInterface +public interface RequestCompletion { + + void complete(); +} diff --git a/source/net/yacy/http/YaCyHttpServer.java b/source/net/yacy/http/YaCyHttpServer.java index a4bff7bbd..60cd9c219 100644 --- a/source/net/yacy/http/YaCyHttpServer.java +++ b/source/net/yacy/http/YaCyHttpServer.java @@ -28,51 +28,48 @@ package net.yacy.http; */ public interface YaCyHttpServer { - /** - * start the http server - */ + /** Start all configured connectors and handlers before returning. */ void startupServer() throws Exception; - /** - * stop the http server - */ + /** Stop all connectors and handlers and wait for complete termination. */ void stop() throws Exception; /** - * reconnect with new port settings (after waiting milsec) - routine returns immediately - * @param milsec wait time + * Apply current HTTP and HTTPS port settings asynchronously after a delay. + * Existing connectors are reused; implementations must not rebuild the handler graph. + * @param milsec non-negative delay before applying current configuration */ void reconnect(int milsec); /** - * @return true if the server runs a ssl/https connector + * @return true when a usable HTTPS connector was configured */ boolean withSSL(); /** - * @return the ssl/https port or -1 if not active + * @return the bound HTTPS port, or -1 when HTTPS is not active */ int getSslPort(); /** - * forces loginservice to reload user credentials + * Evict and immediately reload the named administrator identity from configuration. * @param username */ void resetUser(String username); /** - * removes user from the loginservice + * Evict the named administrator identity from the container login cache. * @param username */ void removeUser(String username); /** - * @return version string of the servlet container + * @return human-readable name and version of the servlet container */ String getVersion(); /** - * @return the number of currently active (busy) server threads + * @return current number of non-idle container worker threads */ int getServerThreads(); } diff --git a/source/net/yacy/http/YacyDomainHandler.java b/source/net/yacy/http/YacyDomainHandler.java index 747b0fe5d..a61a72fac 100644 --- a/source/net/yacy/http/YacyDomainHandler.java +++ b/source/net/yacy/http/YacyDomainHandler.java @@ -38,7 +38,6 @@ import javax.servlet.http.HttpServletResponse; import net.yacy.cora.protocol.Domains; import net.yacy.server.http.AlternativeDomainNames; -import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.AbstractHandler; @@ -53,7 +52,7 @@ import org.eclipse.jetty.server.handler.AbstractHandler; * host is picked up and forwarded to the peer by the transparent proxy * handlers, before the local servlet context would handle it. */ -public class YacyDomainHandler extends AbstractHandler implements Handler { +public class YacyDomainHandler extends AbstractHandler { private AlternativeDomainNames alternativeResolvers; diff --git a/source/net/yacy/http/servlets/Jetty9ServletResource.java b/source/net/yacy/http/servlets/Jetty9ServletResource.java index ba1d3d576..0a4054e45 100644 --- a/source/net/yacy/http/servlets/Jetty9ServletResource.java +++ b/source/net/yacy/http/servlets/Jetty9ServletResource.java @@ -1,3 +1,23 @@ +/** + * Jetty9ServletResource + * Copyright 2026 by Michael Peter Christen + * First released 12.07.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + package net.yacy.http.servlets; import java.io.File; diff --git a/source/net/yacy/http/servlets/ServletResource.java b/source/net/yacy/http/servlets/ServletResource.java index 74e35113a..e5ee160fa 100644 --- a/source/net/yacy/http/servlets/ServletResource.java +++ b/source/net/yacy/http/servlets/ServletResource.java @@ -1,3 +1,23 @@ +/** + * ServletResource + * Copyright 2026 by Michael Peter Christen + * First released 12.07.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + package net.yacy.http.servlets; import java.io.IOException; diff --git a/source/net/yacy/http/servlets/UrlProxyServlet.java b/source/net/yacy/http/servlets/UrlProxyServlet.java index 14ab7d863..378664448 100644 --- a/source/net/yacy/http/servlets/UrlProxyServlet.java +++ b/source/net/yacy/http/servlets/UrlProxyServlet.java @@ -1,3 +1,23 @@ +/** + * UrlProxyServlet + * Copyright 2026 by Michael Peter Christen + * First released 12.07.2026 at https://yacy.net + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program in the file lgpl21.txt + * If not, see <http://www.gnu.org/licenses/>. + */ + package net.yacy.http.servlets; import java.io.ByteArrayInputStream; diff --git a/test/java/net/yacy/cora/federate/solr/connector/RemoteSolrSmoke.java b/test/java/net/yacy/cora/federate/solr/connector/RemoteSolrSmoke.java new file mode 100644 index 000000000..480579150 --- /dev/null +++ b/test/java/net/yacy/cora/federate/solr/connector/RemoteSolrSmoke.java @@ -0,0 +1,32 @@ +package net.yacy.cora.federate.solr.connector; + +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.params.ModifiableSolrParams; + +import net.yacy.cora.federate.solr.instance.RemoteInstance; + +/** Command-line integration probe for YaCY's Apache-based remote Solr path. */ +public final class RemoteSolrSmoke { + + private RemoteSolrSmoke() { + } + + public static void main(final String[] args) throws Exception { + if (args.length != 1) { + throw new IllegalArgumentException("usage: RemoteSolrSmoke SOLR_BASE_URL"); + } + final RemoteInstance instance = new RemoteInstance( + args[0], null, "collection1", 10_000, false, Long.MAX_VALUE, false); + try { + final RemoteSolrConnector connector = new RemoteSolrConnector(instance, false); + final QueryResponse response = connector.getResponseByParams( + new ModifiableSolrParams().set("q", "*:*").set("rows", 0)); + if (response.getResults() == null) { + throw new IllegalStateException("remote Solr response has no result list"); + } + System.out.println("Remote Solr numFound=" + response.getResults().getNumFound()); + } finally { + instance.close(); + } + } +} diff --git a/test/java/net/yacy/http/HttpServerBootstrapConfigTest.java b/test/java/net/yacy/http/HttpServerBootstrapConfigTest.java new file mode 100644 index 000000000..7f2f71120 --- /dev/null +++ b/test/java/net/yacy/http/HttpServerBootstrapConfigTest.java @@ -0,0 +1,25 @@ +package net.yacy.http; + +import org.junit.Assert; +import org.junit.Test; + +public class HttpServerBootstrapConfigTest { + + @Test + public void testAcceptorCountIsClamped() { + Assert.assertEquals(1, HttpServerBootstrapConfig.acceptorCountFor(1)); + Assert.assertEquals(1, HttpServerBootstrapConfig.acceptorCountFor(2)); + Assert.assertEquals(2, HttpServerBootstrapConfig.acceptorCountFor(4)); + Assert.assertEquals(4, HttpServerBootstrapConfig.acceptorCountFor(8)); + Assert.assertEquals(4, HttpServerBootstrapConfig.acceptorCountFor(64)); + } + + @Test + public void testFixedConnectorLimits() { + Assert.assertEquals(16_384, HttpServerBootstrapConfig.REQUEST_HEADER_SIZE); + Assert.assertEquals(9_000L, HttpServerBootstrapConfig.CONNECTOR_IDLE_TIMEOUT_MILLIS); + Assert.assertEquals(128, HttpServerBootstrapConfig.ACCEPT_QUEUE_SIZE); + Assert.assertEquals(4_096, HttpServerBootstrapConfig.REQUEST_INFLATE_BUFFER_SIZE); + Assert.assertEquals(-1, HttpServerBootstrapConfig.MAX_FORM_CONTENT_SIZE); + } +} diff --git a/test/java/net/yacy/http/Jetty9LoggingFacadeTest.java b/test/java/net/yacy/http/Jetty9LoggingFacadeTest.java new file mode 100644 index 000000000..bba2b6924 --- /dev/null +++ b/test/java/net/yacy/http/Jetty9LoggingFacadeTest.java @@ -0,0 +1,16 @@ +package net.yacy.http; + +import static org.junit.Assert.assertEquals; + +import org.eclipse.jetty.util.log.Log; +import org.junit.Test; + +/** Jetty 9 baseline only; replace this test when Jetty9HttpServerImpl is removed. */ +public class Jetty9LoggingFacadeTest { + + @Test + public void jetty9UsesItsSlf4jFacade() { + assertEquals("org.eclipse.jetty.util.log.Slf4jLog", + Log.getLogger("org.eclipse.jetty.yacy.logging.test").getClass().getName()); + } +} diff --git a/test/java/net/yacy/http/ProxyAccessPolicyTest.java b/test/java/net/yacy/http/ProxyAccessPolicyTest.java new file mode 100644 index 000000000..ce67c6ce3 --- /dev/null +++ b/test/java/net/yacy/http/ProxyAccessPolicyTest.java @@ -0,0 +1,20 @@ +package net.yacy.http; + +import org.junit.Assert; +import org.junit.Test; + +public class ProxyAccessPolicyTest { + + @Test + public void testConfiguredClientPatterns() { + Assert.assertTrue(ProxyAccessPolicy.isClientAllowed("*", "198.51.100.7")); + Assert.assertTrue(ProxyAccessPolicy.isClientAllowed( + "localhost,127\\.0\\.0\\.1,192\\.168\\..*", "127.0.0.1")); + Assert.assertTrue(ProxyAccessPolicy.isClientAllowed( + "localhost,127\\.0\\.0\\.1,192\\.168\\..*", "192.168.2.15")); + Assert.assertFalse(ProxyAccessPolicy.isClientAllowed( + "localhost,127\\.0\\.0\\.1,192\\.168\\..*", "198.51.100.7")); + Assert.assertFalse(ProxyAccessPolicy.isClientAllowed(null, "127.0.0.1")); + Assert.assertFalse(ProxyAccessPolicy.isClientAllowed("127\\.0\\.0\\.1", null)); + } +} diff --git a/test/java/net/yacy/http/Slf4jJulBridgeTest.java b/test/java/net/yacy/http/Slf4jJulBridgeTest.java new file mode 100644 index 000000000..b1448eae6 --- /dev/null +++ b/test/java/net/yacy/http/Slf4jJulBridgeTest.java @@ -0,0 +1,59 @@ +package net.yacy.http; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; + +import org.junit.Test; +import org.slf4j.LoggerFactory; + +/** Version-neutral contract for the public SLF4J 2 to JUL logging path. */ +public class Slf4jJulBridgeTest { + + @Test + public void publicSlf4jProviderRoutesJettyNamespaceToJul() { + assertEquals("org.slf4j.jul.JDK14LoggerFactory", + LoggerFactory.getILoggerFactory().getClass().getName()); + + final String loggerName = "org.eclipse.jetty.yacy.logging.test"; + final Logger julLogger = Logger.getLogger(loggerName); + final Level previousLevel = julLogger.getLevel(); + final boolean previousUseParentHandlers = julLogger.getUseParentHandlers(); + final AtomicReference<LogRecord> received = new AtomicReference<>(); + final Handler capture = new Handler() { + @Override + public void publish(final LogRecord record) { + received.set(record); + } + + @Override + public void flush() { + } + + @Override + public void close() { + } + }; + + try { + julLogger.setUseParentHandlers(false); + julLogger.setLevel(Level.INFO); + capture.setLevel(Level.ALL); + julLogger.addHandler(capture); + + LoggerFactory.getLogger(loggerName).info("Jetty SLF4J to JUL bridge test"); + + assertTrue("SLF4J log record did not reach java.util.logging", received.get() != null); + assertEquals("Jetty SLF4J to JUL bridge test", received.get().getMessage()); + } finally { + julLogger.removeHandler(capture); + julLogger.setLevel(previousLevel); + julLogger.setUseParentHandlers(previousUseParentHandlers); + } + } +} diff --git a/test/jetty-auth-smoke-test.sh b/test/jetty-auth-smoke-test.sh new file mode 100755 index 000000000..4970a7e47 --- /dev/null +++ b/test/jetty-auth-smoke-test.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env sh + +# Authentication contract for the embedded server. A remote URL must really +# reach the peer through a non-loopback socket; forwarded headers are not used. + +set -eu + +base_url=${YACY_SMOKE_BASE_URL:-http://127.0.0.1:8090} +remote_base_url=${YACY_SMOKE_REMOTE_BASE_URL:-} +protected_path=${YACY_SMOKE_PROTECTED_PATH:-/ConfigAccounts_p.html} +conf=${YACY_SMOKE_CONF:-DATA/SETTINGS/yacy.conf} +timeout=${YACY_SMOKE_TIMEOUT:-30} +require_complete=${YACY_SMOKE_REQUIRE_COMPLETE:-false} + +case "$require_complete" in + true|false) ;; + *) echo "FAIL: YACY_SMOKE_REQUIRE_COMPLETE must be true or false" >&2; exit 2 ;; +esac + +[ -f "$conf" ] || { echo "FAIL: missing peer configuration: $conf" >&2; exit 2; } + +status() { + curl --silent --show-error --max-time "$timeout" --output /dev/null \ + --write-out '%{http_code}' "$@" +} + +localhost_access=$(sed -n 's/^adminAccountForLocalhost=//p' "$conf" | head -n 1) +local_status=$(status "$base_url$protected_path") +if [ "$localhost_access" = "true" ]; then + [ "$local_status" = 200 ] || { echo "FAIL: localhost bypass returned $local_status" >&2; exit 1; } +else + [ "$local_status" = 401 ] || { echo "FAIL: protected localhost request returned $local_status" >&2; exit 1; } +fi +echo "ok 1 - configured localhost access rule" + +YACY_DATA_PATH=$(CDPATH= cd -- "$(dirname "$conf")/.." && pwd) +export YACY_DATA_PATH +bin/apicall.sh 'ConfigAccounts_p.html' >/dev/null +echo "ok 2 - bin/apicall.sh localhost authentication" + +if [ -n "${YACY_SMOKE_ADMIN_USER:-}" ] && [ -n "${YACY_SMOKE_ADMIN_PASSWORD:-}" ]; then + authenticated_status=$(status --user "$YACY_SMOKE_ADMIN_USER:$YACY_SMOKE_ADMIN_PASSWORD" \ + "$base_url$protected_path") + [ "$authenticated_status" = 200 ] || { echo "FAIL: administrator login returned $authenticated_status" >&2; exit 1; } + echo "ok 3 - administrator credentials" +else + if [ "$require_complete" = true ]; then + echo "FAIL: complete acceptance requires YACY_SMOKE_ADMIN_USER and YACY_SMOKE_ADMIN_PASSWORD" >&2 + exit 1 + fi + echo "ok 3 # SKIP - set YACY_SMOKE_ADMIN_USER and YACY_SMOKE_ADMIN_PASSWORD" +fi + +if [ -n "$remote_base_url" ]; then + remote_status=$(status "$remote_base_url$protected_path") + [ "$remote_status" = 401 ] || { echo "FAIL: remote protected request returned $remote_status" >&2; exit 1; } + echo "ok 4 - remote request does not receive localhost bypass" +else + if [ "$require_complete" = true ]; then + echo "FAIL: complete acceptance requires YACY_SMOKE_REMOTE_BASE_URL over a real non-loopback path" >&2 + exit 1 + fi + echo "ok 4 # SKIP - set YACY_SMOKE_REMOTE_BASE_URL to a real non-loopback path" +fi + +echo "PASS: embedded-server authentication contract" diff --git a/test/jetty-peer-start-smoke-test.sh b/test/jetty-peer-start-smoke-test.sh new file mode 100755 index 000000000..148eb8678 --- /dev/null +++ b/test/jetty-peer-start-smoke-test.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env sh + +# Lifecycle gate owned by an external isolated-peer harness. Commands are +# explicit because this script must never start or stop the developer's normal +# DATA directory by guessing. + +set -eu + +[ -n "${YACY_SMOKE_START_COMMAND:-}" ] && [ -n "${YACY_SMOKE_STOP_COMMAND:-}" ] || { + echo "SKIP: set YACY_SMOKE_START_COMMAND and YACY_SMOKE_STOP_COMMAND for an isolated DATA directory" >&2 + exit 2 +} + +base_url=${YACY_SMOKE_BASE_URL:-http://127.0.0.1:8090} +timeout=${YACY_SMOKE_TIMEOUT:-60} +started=false +cleanup() { + if [ "$started" = true ]; then + sh -c "$YACY_SMOKE_STOP_COMMAND" + fi +} +trap cleanup EXIT HUP INT TERM + +sh -c "$YACY_SMOKE_START_COMMAND" +started=true + +attempt=0 +while [ "$attempt" -lt "$timeout" ]; do + if curl --silent --fail --max-time 2 "$base_url/api/version.xml" >/dev/null 2>&1; then + break + fi + attempt=$((attempt + 1)) + sleep 1 +done +[ "$attempt" -lt "$timeout" ] || { echo "FAIL: isolated peer did not start" >&2; exit 1; } + +test/jetty-smoke-test.sh "$base_url" +curl --silent --show-error --fail --max-time 30 \ + "$base_url/solr/collection1/select?q=*:*&rows=0&wt=json" \ + | grep -F 'numFound' >/dev/null || { + echo "FAIL: embedded Solr query failed" >&2 + exit 1 + } +echo "ok - embedded Solr query" + +cleanup +started=false +trap - EXIT HUP INT TERM +echo "PASS: isolated peer start, embedded Solr and clean stop" diff --git a/test/jetty-smoke-test.sh b/test/jetty-smoke-test.sh index ab53a0d8c..0ff61ccc2 100755 --- a/test/jetty-smoke-test.sh +++ b/test/jetty-smoke-test.sh @@ -53,21 +53,35 @@ fail() { request() { method=$1 path=$2 - shift 2 + expected_status=$3 + shift 3 : > "$response_headers" : > "$response_body" - status=$(curl --silent --show-error \ - --max-time "$curl_timeout" \ - --request "$method" \ - --dump-header "$response_headers" \ - --output "$response_body" \ - --write-out '%{http_code}' \ - "$@" \ - "$base_url$path") || fail "$method $path could not be requested" + if [ "$method" = HEAD ]; then + status=$(curl --silent --show-error \ + --max-time "$curl_timeout" \ + --head \ + --dump-header "$response_headers" \ + --output /dev/null \ + --write-out '%{http_code}' \ + "$@" \ + "$base_url$path") || fail "$method $path could not be requested" + : > "$response_body" + else + status=$(curl --silent --show-error \ + --max-time "$curl_timeout" \ + --request "$method" \ + --dump-header "$response_headers" \ + --output "$response_body" \ + --write-out '%{http_code}' \ + "$@" \ + "$base_url$path") || fail "$method $path could not be requested" + fi - [ "$status" = "200" ] || fail "$method $path returned HTTP $status" + [ "$status" = "$expected_status" ] || \ + fail "$method $path returned HTTP $status (expected $expected_status)" } assert_header_contains() { @@ -109,38 +123,101 @@ while [ "$attempt" -lt 30 ]; do done [ "$attempt" -lt 30 ] || fail "YaCy did not become ready within 30 seconds" -request GET /api/version.xml +request GET /api/version.xml 200 assert_header_contains Content-Type text/xml assert_body_contains '<version>' assert_body_contains '<buildVersion>' pass "reflection-backed XML endpoint" -request GET /env/grafics/YaCyLogo2012.svg +request GET /env/grafics/YaCyLogo2012.svg 200 assert_header_contains Content-Type image/svg+xml assert_body_contains '<svg' pass "static resource" -request GET /index.html +cp "$response_body" "$work_dir/static-full" +static_length=$(wc -c < "$work_dir/static-full" | tr -d ' ') +request HEAD /env/grafics/YaCyLogo2012.svg 200 +assert_header_contains Content-Type image/svg+xml +assert_header_contains Content-Length "$static_length" +[ ! -s "$response_body" ] || fail "HEAD response contains a body" +pass "HEAD without response body" + +request GET /env/grafics/YaCyLogo2012.svg 206 --header 'Range: bytes=0-9' +assert_header_contains Content-Range "bytes 0-9/$static_length" +[ "$(wc -c < "$response_body" | tr -d ' ')" -eq 10 ] || fail "range body is not 10 bytes" +head -c 10 "$work_dir/static-full" > "$work_dir/static-prefix" +cmp "$work_dir/static-prefix" "$response_body" >/dev/null 2>&1 || fail "range body has unexpected bytes" +pass "single byte range" + +request GET /env/grafics/YaCyLogo2012.svg 206 --header 'Range: bytes=0-4,48-51' +assert_header_contains Content-Type 'multipart/byteranges; boundary=' +assert_body_contains "Content-Range: bytes 0-4/$static_length" +assert_body_contains "Content-Range: bytes 48-51/$static_length" +assert_body_contains '<?xml' +assert_body_contains 'YaCy' +multipart_boundary=$(tr -d '\r' < "$response_headers" | awk -F 'boundary=' ' + tolower($0) ~ /^content-type: multipart\/byteranges/ { print $2 } +' | tail -n 1) +[ -n "$multipart_boundary" ] || fail "multipart range response has no boundary" +tail -c 80 "$response_body" | grep -F -- "--$multipart_boundary--" >/dev/null 2>&1 || \ + fail "multipart range response has no closing boundary" +pass "multipart byte ranges" + +request GET /env/grafics/YaCyLogo2012.svg 416 --header "Range: bytes=$static_length-" +assert_header_contains Content-Range "bytes */$static_length" +pass "unsatisfiable byte range" + +# YaCy deliberately suppresses Last-Modified on static responses to control its +# cache policy, but still implements If-Modified-Since against the resource. +request GET /env/grafics/YaCyLogo2012.svg 304 \ + --header 'If-Modified-Since: Thu, 31 Dec 2099 23:59:59 GMT' +[ ! -s "$response_body" ] || fail "304 response contains a body" +pass "If-Modified-Since" + +request GET /index.html 200 assert_header_contains Content-Type text/html assert_body_contains '<!DOCTYPE html>' assert_body_contains '<html' pass "rendered HTML template" -request GET '/suggest.json?q=jetty-smoke' +request GET '/suggest.json?q=jetty-smoke' 200 assert_header_contains Content-Type application/json assert_header_contains Access-Control-Allow-Origin '*' assert_body_contains '["jetty-smoke",[' pass "reflection-backed JSON endpoint and CORS header" -request POST /api/version.xml \ +request POST /api/version.xml 200 \ --header 'Content-Type: application/x-www-form-urlencoded' \ --data 'smoke=post' assert_header_contains Content-Type text/xml assert_body_contains '<version>' pass "URL-encoded POST dispatch" -request OPTIONS /api/version.xml +request OPTIONS /api/version.xml 200 assert_header_contains Allow 'GET,HEAD,POST,OPTIONS' pass "OPTIONS method contract" +request GET /this-resource-must-not-exist-yacy-jetty-smoke 404 +assert_header_contains Content-Type text/html +assert_body_contains 'YaCy ' +pass "YaCy 404 error page" + +request GET /env/grafics/YaCyLogo2012.svg 200 \ + --header 'Accept-Encoding: gzip' \ + --raw +assert_header_contains Content-Encoding gzip +gzip -dc "$response_body" > "$work_dir/gzip-decoded" || fail "gzip response cannot be decompressed" +cmp "$work_dir/static-full" "$work_dir/gzip-decoded" >/dev/null 2>&1 || \ + fail "decompressed response differs from the uncompressed resource" +pass "gzip response compression" + +printf 'q=jetty-gzip-smoke' > "$work_dir/gzip-request-form" +gzip -c "$work_dir/gzip-request-form" > "$work_dir/gzip-request-body" +request POST /suggest.json 200 \ + --header 'Content-Type: application/x-www-form-urlencoded' \ + --header 'Content-Encoding: gzip' \ + --data-binary "@$work_dir/gzip-request-body" +assert_body_contains '["jetty-gzip-smoke",[' +pass "gzip request decompression" + echo "PASS: $checks embedded-server checks succeeded." diff --git a/test/jetty-solr-dependency-guard.sh b/test/jetty-solr-dependency-guard.sh index 0f2fad8a2..21e208194 100755 --- a/test/jetty-solr-dependency-guard.sh +++ b/test/jetty-solr-dependency-guard.sh @@ -42,14 +42,72 @@ for artifact in http2-client http2-common http2-http-client-transport; do fail "$artifact must only be an input of the Solr 9 bridge" done +for artifact in slf4j-api slf4j-jdk14; do + grep -E "name=\"$artifact\" rev=\"1.7.36\" conf=\"solr9-bridge->master\"" ivy.xml >/dev/null 2>&1 || \ + fail "$artifact 1.7.36 must only be an input of the Solr 9 bridge" +done + +grep -E 'org="org.eclipse.jetty.toolchain" name="jetty-servlet-api" rev="4.0.9"' ivy.xml >/dev/null 2>&1 || \ + fail "Jetty's EE8 Servlet 4 API must be an explicit dependency" +grep -E 'exclude org="javax.servlet" module="javax.servlet-api"' ivy.xml >/dev/null 2>&1 || \ + fail "transitive javax.servlet-api artifacts must be excluded" + +expected_jetty_version=$(sed -n \ + 's/.*org="org.eclipse.jetty" name="jetty-server" rev="\([^"]*\)".*/\1/p' \ + ivy.xml) +[ -n "$expected_jetty_version" ] || \ + fail "could not determine the public Jetty version from jetty-server in ivy.xml" +[ "$(printf '%s\n' "$expected_jetty_version" | wc -l | tr -d ' ')" -eq 1 ] || \ + fail "jetty-server must declare exactly one public Jetty version" + if [ -d lib ]; then + public_jetty_count=0 + for artifact in lib/jetty-*.jar; do + [ -e "$artifact" ] || continue + case $(basename "$artifact") in + jetty-servlet-api-*.jar) + # Jetty's Servlet 4 toolchain has its own version line. + continue + ;; + esac + public_jetty_count=$((public_jetty_count + 1)) + case $(basename "$artifact") in + *-"$expected_jetty_version".jar) ;; + *) fail "public Jetty artifact is not on version $expected_jetty_version: $artifact" ;; + esac + done + [ "$public_jetty_count" -gt 0 ] || \ + fail "no public Jetty $expected_jetty_version artifacts found" + + case "$expected_jetty_version" in + 12.*) + for artifact in lib/jetty-continuation-*.jar; do + [ -e "$artifact" ] || continue + fail "Jetty 9-only artifact remains on the Jetty 12 classpath: $artifact" + done + ;; + esac + + servlet_api_count=0 + for artifact in lib/*servlet-api-*.jar; do + [ -e "$artifact" ] || continue + servlet_api_count=$((servlet_api_count + 1)) + [ "$(basename "$artifact")" = "jetty-servlet-api-4.0.9.jar" ] || \ + fail "unexpected Servlet API artifact: $artifact" + done + [ "$servlet_api_count" -eq 1 ] || \ + fail "expected exactly one public Servlet API artifact, found $servlet_api_count" + jar tf lib/jetty-servlet-api-4.0.9.jar | grep '^javax/servlet/resources/web-app_4_0.xsd$' >/dev/null 2>&1 || \ + fail "Jetty Servlet API is missing the EE8 web.xml schema" + for pattern in \ 'jetty-deploy-*.jar' \ 'jetty-jmx-*.jar' \ 'solr-core-*.jar' \ 'solr-solrj-*.jar' \ 'solr-scripting-*.jar' \ - 'http2-*.jar'; do + 'http2-*.jar' \ + 'slf4j-*-1.7.36.jar'; do for artifact in lib/$pattern; do [ -e "$artifact" ] || continue fail "forbidden resolved artifact: $artifact" @@ -66,7 +124,9 @@ if [ -d lib ]; then jetty-util-9.4.58.v20250814 \ http2-client-9.4.58.v20250814 \ http2-common-9.4.58.v20250814 \ - http2-http-client-transport-9.4.58.v20250814; do + http2-http-client-transport-9.4.58.v20250814 \ + slf4j-api-1.7.36 \ + slf4j-jdk14-1.7.36; do jar="lib/solr9-bridge-$artifact.jar" [ -f "$jar" ] || fail "missing generated bridge artifact: $jar" if jar tf "$jar" | grep '^org/eclipse/jetty/' >/dev/null 2>&1; then @@ -75,7 +135,13 @@ if [ -d lib ]; then if zipgrep -a -E 'org(/|\.)eclipse(/|\.)jetty' "$jar" >/dev/null 2>&1; then fail "unrelocated Jetty reference in $jar" fi + if jar tf "$jar" | grep '^org/slf4j/' >/dev/null 2>&1; then + fail "unrelocated SLF4J class in $jar" + fi + if zipgrep -a -E 'org(/|\.)slf4j' "$jar" >/dev/null 2>&1; then + fail "unrelocated SLF4J reference in $jar" + fi done fi -echo "PASS: Solr 9 uses only the relocated Jetty client island." +echo "PASS: Solr 9 uses only the relocated Jetty client and SLF4J 1.7 island." diff --git a/test/jetty/RelocateJettyPackages.java b/test/jetty/RelocateJettyPackages.java index 0766388e5..01939f5d2 100644 --- a/test/jetty/RelocateJettyPackages.java +++ b/test/jetty/RelocateJettyPackages.java @@ -1,9 +1,10 @@ /* - * Proof-of-concept bytecode relocator for the Jetty 12 migration. + * Build-time bytecode relocator for the Jetty 12 migration. * * This is deliberately a small build tool, not YaCy runtime code. It keeps * Solr's public packages unchanged while moving its private Jetty 9 linkage - * below net.yacy.solr9.jetty. + * below net.yacy.solr9.jetty and its SLF4J 1.7 linkage below + * net.yacy.solr9.slf4j. */ package net.yacy.test.jetty; @@ -26,10 +27,12 @@ import org.objectweb.asm.commons.Remapper; public final class RelocateJettyPackages { - private static final String SOURCE_INTERNAL = "org/eclipse/jetty"; - private static final String TARGET_INTERNAL = "net/yacy/solr9/jetty"; - private static final String SOURCE_BINARY = "org.eclipse.jetty"; - private static final String TARGET_BINARY = "net.yacy.solr9.jetty"; + private static final Relocation[] RELOCATIONS = { + new Relocation("org/eclipse/jetty", "net/yacy/solr9/jetty", + "org.eclipse.jetty", "net.yacy.solr9.jetty"), + new Relocation("org/slf4j", "net/yacy/solr9/slf4j", + "org.slf4j", "net.yacy.solr9.slf4j") + }; private RelocateJettyPackages() { } @@ -43,7 +46,7 @@ public final class RelocateJettyPackages { private static void relocate(final Path input, final Path output) throws IOException { Files.createDirectories(output.toAbsolutePath().getParent()); - final Remapper remapper = new JettyRemapper(); + final Remapper remapper = new BridgeRemapper(); try (JarFile source = new JarFile(input.toFile()); JarOutputStream target = new JarOutputStream(Files.newOutputStream(output))) { @@ -62,8 +65,7 @@ public final class RelocateJettyPackages { if (entry.getName().endsWith(".class")) { target.write(relocateClass(content, remapper)); } else if (entry.getName().startsWith("META-INF/services/")) { - target.write(new String(content, StandardCharsets.UTF_8) - .replace(SOURCE_BINARY, TARGET_BINARY) + target.write(relocateText(new String(content, StandardCharsets.UTF_8)) .getBytes(StandardCharsets.UTF_8)); } else { target.write(content); @@ -74,28 +76,42 @@ public final class RelocateJettyPackages { } } - private static final class JettyRemapper extends Remapper { + private static final class BridgeRemapper extends Remapper { - private JettyRemapper() { + private BridgeRemapper() { super(Opcodes.ASM9); } @Override public String map(final String internalName) { - return replacePrefix(internalName, SOURCE_INTERNAL, TARGET_INTERNAL); + return relocateInternalName(internalName); } @Override public Object mapValue(final Object value) { if (value instanceof String) { final String text = (String) value; - return text.replace(SOURCE_BINARY, TARGET_BINARY) - .replace(SOURCE_INTERNAL, TARGET_INTERNAL); + return relocateText(text); } return super.mapValue(value); } } + private static final class Relocation { + private final String sourceInternal; + private final String targetInternal; + private final String sourceBinary; + private final String targetBinary; + + private Relocation(final String sourceInternal, final String targetInternal, + final String sourceBinary, final String targetBinary) { + this.sourceInternal = sourceInternal; + this.targetInternal = targetInternal; + this.sourceBinary = sourceBinary; + this.targetBinary = targetBinary; + } + } + private static byte[] relocateClass(final byte[] content, final Remapper remapper) { final ClassReader reader = new ClassReader(content); final ClassWriter writer = new ClassWriter(0); @@ -120,13 +136,30 @@ public final class RelocateJettyPackages { } private static String relocateEntryName(final String name) { - final String internalName = replacePrefix(name, SOURCE_INTERNAL, TARGET_INTERNAL); + final String internalName = relocateInternalName(name); if (internalName.startsWith("META-INF/services/")) { - return internalName.replace(SOURCE_BINARY, TARGET_BINARY); + return relocateText(internalName); } return internalName; } + private static String relocateInternalName(final String name) { + String relocated = name; + for (final Relocation relocation : RELOCATIONS) { + relocated = replacePrefix(relocated, relocation.sourceInternal, relocation.targetInternal); + } + return relocated; + } + + private static String relocateText(final String text) { + String relocated = text; + for (final Relocation relocation : RELOCATIONS) { + relocated = relocated.replace(relocation.sourceBinary, relocation.targetBinary) + .replace(relocation.sourceInternal, relocation.targetInternal); + } + return relocated; + } + private static byte[] readAll(final InputStream stream) throws IOException { final ByteArrayOutputStream output = new ByteArrayOutputStream(); stream.transferTo(output); diff --git a/test/proxy-smoke-test.sh b/test/proxy-smoke-test.sh new file mode 100755 index 000000000..ccae2227b --- /dev/null +++ b/test/proxy-smoke-test.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env sh + +# Live HTTP proxy and CONNECT contract. The peer must be started with the +# transparent proxy enabled and both targets must be controlled test services. + +set -eu + +proxy=${YACY_SMOKE_PROXY:-} +http_target=${YACY_SMOKE_PROXY_HTTP_TARGET:-} +https_target=${YACY_SMOKE_PROXY_HTTPS_TARGET:-} +[ -n "$proxy" ] && [ -n "$http_target" ] && [ -n "$https_target" ] || { + echo "SKIP: set YACY_SMOKE_PROXY, YACY_SMOKE_PROXY_HTTP_TARGET and YACY_SMOKE_PROXY_HTTPS_TARGET" >&2 + exit 2 +} + +timeout=${YACY_SMOKE_TIMEOUT:-30} +curl --silent --show-error --fail --max-time "$timeout" --proxy "$proxy" \ + --noproxy '' \ + "$http_target" >/dev/null +echo "ok 1 - HTTP proxy traffic" + +# An HTTPS request through an HTTP proxy necessarily establishes a CONNECT tunnel. +curl --silent --show-error --fail --max-time "$timeout" --proxy "$proxy" \ + --noproxy '' \ + "$https_target" >/dev/null +echo "ok 2 - HTTPS CONNECT tunnel" + +echo "PASS: proxy and CONNECT contract" diff --git a/test/remote-solr-smoke-test.sh b/test/remote-solr-smoke-test.sh new file mode 100755 index 000000000..84bea117a --- /dev/null +++ b/test/remote-solr-smoke-test.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env sh + +# Exercises YaCy's Apache-HttpClient-backed RemoteInstance against an external +# Solr collection. This is an opt-in integration gate. + +set -eu + +[ -n "${YACY_REMOTE_SOLR_URL:-}" ] || { + echo "SKIP: set YACY_REMOTE_SOLR_URL (for example http://host:8983/solr/)" >&2 + exit 2 +} + +repo_root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) +cd "$repo_root" +[ -d build/classes/java/main ] || { echo "FAIL: run ant compile first" >&2; exit 1; } + +work=$(mktemp -d "${TMPDIR:-/tmp}/yacy-remote-solr-smoke.XXXXXX") +trap 'rm -rf "$work"' EXIT HUP INT TERM +classpath=build/classes/java/main +for archive in lib/*.jar; do + case $(basename "$archive") in yacycore.jar) continue ;; esac + classpath="$classpath:$archive" +done + +javac --release 17 -cp "$classpath" -d "$work" \ + test/java/net/yacy/cora/federate/solr/connector/RemoteSolrSmoke.java +java -cp "$work:$classpath" net.yacy.cora.federate.solr.connector.RemoteSolrSmoke \ + "$YACY_REMOTE_SOLR_URL" + +echo "PASS: remote Solr request through YaCy RemoteInstance" |