diff options
37 files changed, 2886 insertions, 410 deletions
@@ -161,13 +161,15 @@ <path id="project.class.path"> <pathelement location="${build}" /> <pathelement location="${htroot}" /> - <pathelement location="${lib}/jetty-http-9.0.5.v20130815.jar" /> - <pathelement location="${lib}/jetty-io-9.0.5.v20130815.jar" /> - <pathelement location="${lib}/jetty-security-9.0.5.v20130815.jar" /> - <pathelement location="${lib}/jetty-server-9.0.5.v20130815.jar" /> - <pathelement location="${lib}/jetty-servlet-9.0.5.v20130815.jar" /> - <pathelement location="${lib}/jetty-servlets-9.0.5.v20130815.jar" /> - <pathelement location="${lib}/jetty-util-9.0.5.v20130815.jar" /> + <pathelement location="${lib}/jetty-continuation-8.1.13.v20130916.jar" /> + <pathelement location="${lib}/jetty-http-8.1.13.v20130916.jar" /> + <pathelement location="${lib}/jetty-io-8.1.13.v20130916.jar" /> + <pathelement location="${lib}/jetty-security-8.1.13.v20130916.jar" /> + <pathelement location="${lib}/jetty-server-8.1.13.v20130916.jar" /> + <pathelement location="${lib}/jetty-servlet-8.1.13.v20130916.jar" /> + <pathelement location="${lib}/jetty-servlets-8.1.13.v20130916.jar" /> + <pathelement location="${lib}/jetty-util-8.1.13.v20130916.jar" /> + <pathelement location="${lib}/javax.servlet-3.0.0.v201112011016.jar" /> <pathelement location="${lib}/activation.jar" /> <pathelement location="${lib}/apache-mime4j-0.6.jar" /> diff --git a/htroot/ConfigBasic.java b/htroot/ConfigBasic.java index efa74a8be..db0394c11 100644 --- a/htroot/ConfigBasic.java +++ b/htroot/ConfigBasic.java @@ -38,12 +38,11 @@ import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.data.Translator; import net.yacy.data.WorkTables; -import net.yacy.http.HttpServer; +import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.workflow.InstantBusyThread; import net.yacy.peers.Seed; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; -import net.yacy.server.serverCore; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.server.http.HTTPDFileHandler; @@ -132,7 +131,7 @@ public class ConfigBasic { final boolean reconnect; if (!(env.getConfigLong("port", port) == port) || env.getConfigBool("server.https", false) != ssl) { // validate port - final HttpServer theServerCore = env.getHttpServer(); + final YaCyHttpServer theServerCore = env.getHttpServer(); env.setConfig("port", port); env.setConfig("server.https", ssl); diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index a35ec8921..e78f501c3 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -31,7 +31,7 @@ import java.util.Map; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.http.HttpServer; +import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.rwi.IndexCell; import net.yacy.kelondro.util.FileUtils; @@ -39,11 +39,9 @@ import net.yacy.kelondro.util.Formatter; import net.yacy.kelondro.util.MemoryControl; import net.yacy.kelondro.util.OS; import net.yacy.kelondro.workflow.BusyThread; -import net.yacy.kelondro.workflow.WorkflowThread; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.Segment; -import net.yacy.server.serverCore; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -267,7 +265,7 @@ public class PerformanceQueues_p { /* * configuring the http pool */ - final HttpServer httpd = sb.getHttpServer(); + final YaCyHttpServer httpd = sb.getHttpServer(); try { maxBusy = post.getInt("httpd Session Pool_maxActive", 8); } catch (final NumberFormatException e) { @@ -311,7 +309,7 @@ public class PerformanceQueues_p { prop.put("pool_0_maxActive", sb.getConfigLong("crawler.MaxActiveThreads", 0)); prop.put("pool_0_numActive",sb.crawlQueues.workerSize()); - final HttpServer httpd = sb.getHttpServer(); + final YaCyHttpServer httpd = sb.getHttpServer(); prop.put("pool_1_name", "httpd Session Pool"); prop.put("pool_1_maxActive", httpd.getMaxSessionCount()); prop.put("pool_1_numActive", httpd.getJobCount()); diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index ae5b02743..218df8b32 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -40,7 +40,7 @@ import net.yacy.cora.order.Base64Order; import net.yacy.cora.order.Digest; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.RequestHeader; -import net.yacy.http.HttpServer; +import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.util.Formatter; import net.yacy.peers.Network; import net.yacy.peers.Seed; @@ -109,7 +109,7 @@ public class SettingsAck_p { prop.putHTML("info_port", port); if (!env.getConfig("port", port).equals(port)) { // validation port - final HttpServer theServerCore = env.getHttpServer(); + final YaCyHttpServer theServerCore = env.getHttpServer(); try { final InetSocketAddress theNewAddress = theServerCore.generateSocketAddress(port); final String hostName = Domains.getHostName(theNewAddress.getAddress()); diff --git a/htroot/Status.java b/htroot/Status.java index faf84d0f1..3caa37dfc 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -33,7 +33,7 @@ import java.util.Date; import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.util.Memory;
-import net.yacy.http.HttpServer;
+import net.yacy.http.YaCyHttpServer;
import net.yacy.kelondro.io.ByteCount;
import net.yacy.kelondro.util.Formatter;
import net.yacy.kelondro.util.MemoryControl;
@@ -327,7 +327,7 @@ public class Status prop.put("trafficCrawler", Formatter.bytesToString(ByteCount.getAccountCount(ByteCount.CRAWLER)));
// connection information
- final HttpServer httpd = sb.getHttpServer();
+ final YaCyHttpServer httpd = sb.getHttpServer();
prop.putNum("connectionsActive", httpd.getJobCount());
prop.putNum("connectionsMax", httpd.getMaxSessionCount());
diff --git a/lib/jetty-continuation-8.1.13.v20130916.jar b/lib/jetty-continuation-8.1.13.v20130916.jar Binary files differnew file mode 100644 index 000000000..7d9990f75 --- /dev/null +++ b/lib/jetty-continuation-8.1.13.v20130916.jar diff --git a/lib/jetty-http-8.1.13.v20130916.jar b/lib/jetty-http-8.1.13.v20130916.jar Binary files differnew file mode 100644 index 000000000..e12cfe031 --- /dev/null +++ b/lib/jetty-http-8.1.13.v20130916.jar diff --git a/lib/jetty-io-8.1.13.v20130916.jar b/lib/jetty-io-8.1.13.v20130916.jar Binary files differnew file mode 100644 index 000000000..51e527b15 --- /dev/null +++ b/lib/jetty-io-8.1.13.v20130916.jar diff --git a/lib/jetty-security-8.1.13.v20130916.jar b/lib/jetty-security-8.1.13.v20130916.jar Binary files differnew file mode 100644 index 000000000..d1d7c2737 --- /dev/null +++ b/lib/jetty-security-8.1.13.v20130916.jar diff --git a/lib/jetty-server-8.1.13.v20130916.jar b/lib/jetty-server-8.1.13.v20130916.jar Binary files differnew file mode 100644 index 000000000..d2273bb16 --- /dev/null +++ b/lib/jetty-server-8.1.13.v20130916.jar diff --git a/lib/jetty-servlet-8.1.13.v20130916.jar b/lib/jetty-servlet-8.1.13.v20130916.jar Binary files differnew file mode 100644 index 000000000..4346d768c --- /dev/null +++ b/lib/jetty-servlet-8.1.13.v20130916.jar diff --git a/lib/jetty-servlets-8.1.13.v20130916.jar b/lib/jetty-servlets-8.1.13.v20130916.jar Binary files differnew file mode 100644 index 000000000..44e30dcb9 --- /dev/null +++ b/lib/jetty-servlets-8.1.13.v20130916.jar diff --git a/lib/jetty-util-8.1.13.v20130916.jar b/lib/jetty-util-8.1.13.v20130916.jar Binary files differnew file mode 100644 index 000000000..86e4d881c --- /dev/null +++ b/lib/jetty-util-8.1.13.v20130916.jar diff --git a/lib/jetty-continuation-9.0.5.v20130815.jar b/libt/jetty-continuation-9.0.5.v20130815.jar Binary files differindex a79294670..a79294670 100644 --- a/lib/jetty-continuation-9.0.5.v20130815.jar +++ b/libt/jetty-continuation-9.0.5.v20130815.jar diff --git a/lib/jetty-http-9.0.5.v20130815.jar b/libt/jetty-http-9.0.5.v20130815.jar Binary files differindex ff4975d74..ff4975d74 100644 --- a/lib/jetty-http-9.0.5.v20130815.jar +++ b/libt/jetty-http-9.0.5.v20130815.jar diff --git a/lib/jetty-io-9.0.5.v20130815.jar b/libt/jetty-io-9.0.5.v20130815.jar Binary files differindex 5459012fa..5459012fa 100644 --- a/lib/jetty-io-9.0.5.v20130815.jar +++ b/libt/jetty-io-9.0.5.v20130815.jar diff --git a/lib/jetty-security-9.0.5.v20130815.jar b/libt/jetty-security-9.0.5.v20130815.jar Binary files differindex 7ff8202da..7ff8202da 100644 --- a/lib/jetty-security-9.0.5.v20130815.jar +++ b/libt/jetty-security-9.0.5.v20130815.jar diff --git a/lib/jetty-server-9.0.5.v20130815.jar b/libt/jetty-server-9.0.5.v20130815.jar Binary files differindex da9bf3e71..da9bf3e71 100644 --- a/lib/jetty-server-9.0.5.v20130815.jar +++ b/libt/jetty-server-9.0.5.v20130815.jar diff --git a/lib/jetty-servlet-9.0.5.v20130815.jar b/libt/jetty-servlet-9.0.5.v20130815.jar Binary files differindex 961f23e53..961f23e53 100644 --- a/lib/jetty-servlet-9.0.5.v20130815.jar +++ b/libt/jetty-servlet-9.0.5.v20130815.jar diff --git a/lib/jetty-servlets-9.0.5.v20130815.jar b/libt/jetty-servlets-9.0.5.v20130815.jar Binary files differindex 3cecab6c2..3cecab6c2 100644 --- a/lib/jetty-servlets-9.0.5.v20130815.jar +++ b/libt/jetty-servlets-9.0.5.v20130815.jar diff --git a/lib/jetty-util-9.0.5.v20130815.jar b/libt/jetty-util-9.0.5.v20130815.jar Binary files differindex 207eab459..207eab459 100644 --- a/lib/jetty-util-9.0.5.v20130815.jar +++ b/libt/jetty-util-9.0.5.v20130815.jar diff --git a/source/net/yacy/http/Jetty8HttpServerImpl.java b/source/net/yacy/http/Jetty8HttpServerImpl.java new file mode 100644 index 000000000..16131e239 --- /dev/null +++ b/source/net/yacy/http/Jetty8HttpServerImpl.java @@ -0,0 +1,190 @@ +// +// Jetty8HttpServerImpl +// Copyright 2011 by Florian Richter +// First released 13.04.2011 at http://yacy.net +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program in the file lgpl21.txt +// If not, see <http://www.gnu.org/licenses/>. +// + +package net.yacy.http; + +import java.net.InetSocketAddress; +import java.net.SocketException; +import java.util.EnumSet; +import javax.servlet.DispatcherType; +import net.yacy.cora.federate.solr.SolrServlet; +import net.yacy.cora.federate.solr.SolrServlet.Servlet404; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.search.Switchboard; + +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.server.handler.ContextHandlerCollection; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.server.nio.SelectChannelConnector; +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; + +/** + * class to embedded Jetty 8 http server into YaCy + */ +public class Jetty8HttpServerImpl implements YaCyHttpServer { + + private Server server; + + /** + * @param port TCP Port to listen for http requests + */ + public Jetty8HttpServerImpl(int port) { + Switchboard sb = Switchboard.getSwitchboard(); + + server = new Server(); + SelectChannelConnector connector = new SelectChannelConnector(); + connector.setPort(port); + connector.setName("httpd:"+Integer.toString(port)); + //connector.setThreadPool(new QueuedThreadPool(20)); + server.addConnector(connector); + + YacyDomainHandler domainHandler = new YacyDomainHandler(); + domainHandler.setAlternativeResolver(sb.peers); + + /* this is now handled by YaCyDefaultServlet + ResourceHandler resource_handler = new ResourceHandler(); + resource_handler.setDirectoriesListed(true); + resource_handler.setWelcomeFiles(new String[]{"index.html"}); + resource_handler.setResourceBase("htroot/"); + */ + + //add SolrServlet + ServletContextHandler solrContext = new ServletContextHandler(ServletContextHandler.SESSIONS); + solrContext.setContextPath("/solr"); + solrContext.addServlet(new ServletHolder(Servlet404.class),"/*"); + + SolrServlet.initCore(sb.index.fulltext().getDefaultEmbeddedConnector()); + solrContext.addFilter(new FilterHolder(SolrServlet.class), "/*", EnumSet.of(DispatcherType.REQUEST)); + + // configure root context + ServletContextHandler htrootContext = new ServletContextHandler(ServletContextHandler.SESSIONS); + htrootContext.setContextPath("/"); + ServletHolder sholder = new ServletHolder(Jetty8YaCyDefaultServlet.class); + sholder.setInitParameter("resourceBase", "htroot"); + //sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html + sholder.setInitParameter("gzip","false"); + htrootContext.addServlet(sholder,"/*"); + + // assemble the servlet handlers + ContextHandlerCollection servletContext = new ContextHandlerCollection(); + servletContext.setHandlers(new Handler[] { solrContext, htrootContext }); + + // define list of YaCy specific general handlers + HandlerList handlers = new HandlerList(); + handlers.setHandlers(new Handler[] + {domainHandler, new ProxyCacheHandler(), new ProxyHandler() + /*, resource_handler, new DefaultHandler() */}); + + // context handler for dispatcher and security (hint: dispatcher requires a context) + ContextHandler context = new ContextHandler(); + context.setContextPath("/"); + context.setHandler(handlers); + + // make YaCy handlers (in context) and servlet context handlers available (both contain root context "/") + // logic: 1. YaCy handlers are called if request not handled (e.g. proxy) then servlets handle it + ContextHandlerCollection allrequesthandlers = new ContextHandlerCollection(); + allrequesthandlers.addHandler(context); + allrequesthandlers.addHandler(servletContext); + allrequesthandlers.addHandler(new DefaultHandler()); // if not handled by other handler + + // wrap all handlers by security handler + Jetty8YaCySecurityHandler securityHandler = new Jetty8YaCySecurityHandler(); + securityHandler.setLoginService(new YaCyLoginService()); + securityHandler.setRealmName("YaCy Admin Interface"); + securityHandler.setHandler(new CrashProtectionHandler(allrequesthandlers)); + + server.setHandler(securityHandler); + } + + /** + * start http server + */ + @Override + public void startupServer() throws Exception { + server.start(); + } + + /** + * stop http server and wait for it + */ + @Override + public void stop() throws Exception { + server.stop(); + server.join(); + } + + @Override + public void setMaxSessionCount(int maxBusy) { + // TODO: + } + + @Override + public boolean withSSL() { + return false; // TODO: + } + + @Override + public void reconnect(int milsec) { + try { + Thread.sleep(milsec); + } catch (final InterruptedException e) { + ConcurrentLog.logException(e); + } catch (final Exception e) { + ConcurrentLog.logException(e); + } + try { + server.stop(); + server.join(); + server.start(); + } catch (Exception ex) { + ConcurrentLog.logException(ex); + } + } + + @Override + public InetSocketAddress generateSocketAddress(String port) throws SocketException { + return null; // TODO: + } + + @Override + public int getMaxSessionCount() { + return server.getThreadPool().getThreads(); + } + + @Override + public int getJobCount() { + return getMaxSessionCount() - server.getThreadPool().getIdleThreads(); // TODO: + } + + @Override + public String getVersion() { + return "Jetty " + server.getVersion(); + } + +} diff --git a/source/net/yacy/http/Jetty8YaCyDefaultServlet.java b/source/net/yacy/http/Jetty8YaCyDefaultServlet.java new file mode 100644 index 000000000..b5a9c5041 --- /dev/null +++ b/source/net/yacy/http/Jetty8YaCyDefaultServlet.java @@ -0,0 +1,691 @@ +// +// ======================================================================== +// Copyright (c) 1995-2013 Mort Bay Consulting Pty. Ltd. +// ------------------------------------------------------------------------ +// All rights reserved. This program and the accompanying materials +// are made available under the terms of the Eclipse Public License v1.0 +// and Apache License v2.0 which accompanies this distribution. +// +// The Eclipse Public License is available at +// http://www.eclipse.org/legal/epl-v10.html +// +// The Apache License v2.0 is available at +// http://www.opensource.org/licenses/apache2.0.php +// +// You may elect to redistribute this code under either of these licenses. +// ======================================================================== +// +package net.yacy.http; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URL; +import java.util.Enumeration; +import java.util.List; + +import javax.servlet.RequestDispatcher; +import javax.servlet.ServletException; +import javax.servlet.UnavailableException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import net.yacy.cora.protocol.HeaderFramework; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.kelondro.util.FileUtils; + +import org.eclipse.jetty.http.HttpContent; +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpHeaderValues; +import org.eclipse.jetty.http.HttpHeaders; +import org.eclipse.jetty.http.HttpMethods; +import org.eclipse.jetty.io.Buffer; +import org.eclipse.jetty.io.WriterOutputStream; +import org.eclipse.jetty.server.AbstractHttpConnection; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Dispatcher; +import org.eclipse.jetty.server.HttpOutput; +import org.eclipse.jetty.server.InclusiveByteRange; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.server.nio.NIOConnector; +import org.eclipse.jetty.server.ssl.SslConnector; +import org.eclipse.jetty.util.MultiPartOutputStream; +import org.eclipse.jetty.util.QuotedStringTokenizer; +import org.eclipse.jetty.util.URIUtil; +import org.eclipse.jetty.util.resource.Resource; +import org.eclipse.jetty.util.resource.ResourceFactory; + +/* ------------------------------------------------------------ */ +/** + * YaCyDefaultServlet base on Jetty DefaultServlet.java + * handles static files and the YaCy servlets. + */ + +/** + * The default servlet. This servlet, normally mapped to /, provides the + * handling for static content, OPTION and TRACE methods for the context. The + * following initParameters are supported, these can be set either on the + * servlet itself or as ServletContext initParameters : + * <PRE> + * acceptRanges If true, range requests and responses are + * supported + * + * dirAllowed If true, directory listings are returned if no + * welcome file is found. Else 403 Forbidden. + * + * welcomeFile name of the welcome file (default is "index.html", "welcome.html") + * + * gzip If set to true, then static content will be served as + * gzip content encoded if a matching resource is + * found ending with ".gz" + * + * resourceBase Set to replace the context resource base + * + * resourceCache If set, this is a context attribute name, which the servlet + * will use to look for a shared ResourceCache instance. + * + * relativeResourceBase + * Set with a pathname relative to the base of the + * servlet context root. Useful for only serving static content out + * of only specific subdirectories. + * + * pathInfoOnly If true, only the path info will be applied to the resourceBase + * + * + * etags If True, weak etags will be generated and handled. + * + * </PRE> + * + * + * + * + */ +public class Jetty8YaCyDefaultServlet extends YaCyDefaultServlet implements ResourceFactory { + + private static final long serialVersionUID = 4900000000000001110L; + + private boolean _gzip = true; + + /* ------------------------------------------------------------ */ + @Override + public void init() throws UnavailableException { + super.init(); + + _gzip=getInitBoolean("gzip",_gzip); + + } + + /* ------------------------------------------------------------ */ + /** + * get Resource to serve. Map a path to a resource. The default + * implementation calls HttpContext.getResource but derived servlets may + * provide their own mapping. + * + * @param pathInContext The path to find a resource for. + * @return The resource to serve. + */ + @Override + public Resource getResource(String pathInContext) { + Resource r = null; + if (_relativeResourceBase != null) { + pathInContext = URIUtil.addPaths(_relativeResourceBase, pathInContext); + } + + try { + if (_resourceBase != null) { + r = _resourceBase.addPath(pathInContext); + } else { + URL u = _servletContext.getResource(pathInContext); + r = Resource.newResource(u); + } + + if (ConcurrentLog.isFine("YaCyDefaultServlet")) { + ConcurrentLog.fine("YaCyDefaultServlet","Resource " + pathInContext + "=" + r); + } + } catch (IOException e) { + // ConcurrentLog.logException(e); + } + + return r; + } + + /* ------------------------------------------------------------ */ + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + String servletPath = null; + String pathInfo = null; + Enumeration<String> reqRanges = null; + Boolean included = request.getAttribute(Dispatcher.INCLUDE_REQUEST_URI) != null; + if (included != null && included.booleanValue()) { + servletPath = (String) request.getAttribute(Dispatcher.INCLUDE_SERVLET_PATH); + pathInfo = (String) request.getAttribute(Dispatcher.INCLUDE_PATH_INFO); + if (servletPath == null) { + servletPath = request.getServletPath(); + pathInfo = request.getPathInfo(); + } + } else { + included = Boolean.FALSE; + servletPath = _pathInfoOnly ? "/" : request.getServletPath(); + pathInfo = request.getPathInfo(); + + // Is this a Range request? + reqRanges = request.getHeaders(HeaderFramework.RANGE); + if (!hasDefinedRange(reqRanges)) { + reqRanges = null; + } + } + + if (pathInfo.startsWith("/currentyacypeer/")) pathInfo = pathInfo.substring(16); + String pathInContext = URIUtil.addPaths(servletPath, pathInfo); + boolean endsWithSlash = (pathInfo == null ? request.getServletPath() : pathInfo).endsWith(URIUtil.SLASH); + + // Find the resource and content + Resource resource = null; + HttpContent content = null; + try { + // is gzip enabled? + String pathInContextGz = null; + boolean gzip = false; + if (!included.booleanValue() && _gzip && reqRanges == null && !endsWithSlash) { + // Look for a gzip resource + pathInContextGz = pathInContext + ".gz"; + resource = getResource(pathInContextGz); + + // Does a gzip resource exist? + if (resource != null && resource.exists() && !resource.isDirectory()) { + // Tell caches that response may vary by accept-encoding + response.addHeader(HttpHeaders.VARY, HttpHeaders.ACCEPT_ENCODING); + + // Does the client accept gzip? + String accept = request.getHeader(HttpHeaders.ACCEPT_ENCODING); + if (accept != null && accept.indexOf("gzip") >= 0) { + gzip = true; + } + } + } + + // find resource + if (!gzip) resource = getResource(pathInContext); + + // Look for a class resource + boolean hasClass = false; + if (reqRanges == null && !endsWithSlash) { + final int p = pathInContext.lastIndexOf('.'); + if (p >= 0) { + String pathofClass = pathInContext.substring(0, p) + ".class"; + Resource classresource = getResource(pathofClass); + // Does a class resource exist? + if (classresource != null && classresource.exists() && !classresource.isDirectory()) { + hasClass = true; + } + } + } + + if (ConcurrentLog.isFine("YaCyDefaultServlet")) { + ConcurrentLog.fine("YaCyDefaultServlet","uri=" + request.getRequestURI() + " resource=" + resource + (content != null ? " content" : "")); + } + + // Handle resource + if (!hasClass && (resource == null || !resource.exists())) { + if (included) { + throw new FileNotFoundException("!" + pathInContext); + } + response.sendError(HttpServletResponse.SC_NOT_FOUND); + } else if (!resource.isDirectory()) { + if (endsWithSlash && pathInContext.length() > 1) { + String q = request.getQueryString(); + pathInContext = pathInContext.substring(0, pathInContext.length() - 1); + if (q != null && q.length() != 0) { + pathInContext += "?" + q; + } + response.sendRedirect(response.encodeRedirectURL(URIUtil.addPaths(_servletContext.getContextPath(), pathInContext))); + } else { + // ensure we have content + if (content == null) { + content = new HttpContent.ResourceAsHttpContent(resource, _mimeTypes.getMimeByExtension(resource.toString()), response.getBufferSize(), _etags); + } + + if (hasClass) { // this is a YaCy servlet, handle the template + handleTemplate(pathInfo, request, response); + } else { + if (included.booleanValue() || passConditionalHeaders(request, response, resource, content)) { + if (gzip) { + response.setHeader(HttpHeaders.CONTENT_ENCODING, "gzip"); + String mt = _servletContext.getMimeType(pathInContext); + if (mt != null) { + response.setContentType(mt); + } + } + sendData(request, response, included.booleanValue(), resource, content, reqRanges); + } + } + } + } else { + String welcome = null; + + if (!endsWithSlash || (pathInContext.length() == 1 && request.getAttribute("org.eclipse.jetty.server.nullPathInfo") != null)) { + StringBuffer buf = request.getRequestURL(); + synchronized (buf) { + int param = buf.lastIndexOf(";"); + if (param < 0) { + buf.append('/'); + } else { + buf.insert(param, '/'); + } + String q = request.getQueryString(); + if (q != null && q.length() != 0) { + buf.append('?'); + buf.append(q); + } + response.setContentLength(0); + response.sendRedirect(response.encodeRedirectURL(buf.toString())); + } + } // else look for a welcome file + else if (null != (welcome = getWelcomeFile(pathInContext))) { + ConcurrentLog.fine("welcome={}", welcome); + + + // Forward to the index + RequestDispatcher dispatcher = request.getRequestDispatcher(welcome); + if (dispatcher != null) { + if (included.booleanValue()) { + dispatcher.include(request, response); + } else { + request.setAttribute("org.eclipse.jetty.server.welcome", welcome); + dispatcher.forward(request, response); + } + } + + } else { + content = new HttpContent.ResourceAsHttpContent(resource, _mimeTypes.getMimeByExtension(resource.toString()), _etags); + if (included.booleanValue() || passConditionalHeaders(request, response, resource, content)) { + sendDirectory(request, response, resource, pathInContext); + } + } + } + } catch (IllegalArgumentException e) { + ConcurrentLog.logException(e); + if (!response.isCommitted()) { + response.sendError(500, e.getMessage()); + } + } finally { + if (content != null) { + content.release(); + } else if (resource != null) { + resource.release(); + } + } + } + + /* ------------------------------------------------------------ */ + @Override + protected boolean hasDefinedRange(Enumeration<String> reqRanges) { + return (reqRanges != null && reqRanges.hasMoreElements()); + } + + /* ------------------------------------------------------------ */ + /* Check modification date headers. + */ + @Override + protected boolean passConditionalHeaders(HttpServletRequest request, HttpServletResponse response, Resource resource, HttpContent content) + throws IOException { + try { + if (!request.getMethod().equals(HttpMethods.HEAD)) { + if (_etags) { + String ifm = request.getHeader(HttpHeaders.IF_MATCH); + if (ifm != null) { + boolean match = false; + if (content != null && content.getETag() != null) { + QuotedStringTokenizer quoted = new QuotedStringTokenizer(ifm, ", ", false, true); + while (!match && quoted.hasMoreTokens()) { + String tag = quoted.nextToken(); + if (content.getETag().toString().equals(tag)) { + match = true; + } + } + } + + if (!match) { + Response r = Response.getResponse(response); + r.reset(true); + r.setStatus(HttpServletResponse.SC_PRECONDITION_FAILED); + return false; + } + } + + String ifnm = request.getHeader(HttpHeaders.IF_NONE_MATCH); + if (ifnm != null && content != null && content.getETag() != null) { + // Look for GzipFiltered version of etag + if (content.getETag().toString().equals(request.getAttribute("o.e.j.s.GzipFilter.ETag"))) { + Response r = Response.getResponse(response); + r.reset(true); + r.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + r.getHttpFields().put(HttpHeaders.ETAG_BUFFER, ifnm); + return false; + } + + + // Handle special case of exact match. + if (content.getETag().toString().equals(ifnm)) { + Response r = Response.getResponse(response); + r.reset(true); + r.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + r.getHttpFields().put(HttpHeaders.ETAG_BUFFER, content.getETag()); + return false; + } + + // Handle list of tags + QuotedStringTokenizer quoted = new QuotedStringTokenizer(ifnm, ", ", false, true); + while (quoted.hasMoreTokens()) { + String tag = quoted.nextToken(); + if (content.getETag().toString().equals(tag)) { + Response r = Response.getResponse(response); + r.reset(true); + r.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + r.getHttpFields().put(HttpHeaders.ETAG_BUFFER, content.getETag()); + return false; + } + } + + return true; + } + } + + String ifms = request.getHeader(HttpHeaders.IF_MODIFIED_SINCE); + if (ifms != null) { + //Get jetty's Response impl + Response r = Response.getResponse(response); + + if (content != null) { + Buffer mdlm = content.getLastModified(); + if (mdlm != null) { + if (ifms.equals(mdlm.toString())) { + r.reset(true); + r.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + if (_etags) { + r.getHttpFields().add(HttpHeaders.ETAG_BUFFER, content.getETag()); + } + r.flushBuffer(); + return false; + } + } + } + + long ifmsl = request.getDateHeader(HttpHeaders.IF_MODIFIED_SINCE); + if (ifmsl != -1) { + if (resource.lastModified() / 1000 <= ifmsl / 1000) { + r.reset(true); + r.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + if (_etags) { + r.getHttpFields().add(HttpHeaders.ETAG_BUFFER, content.getETag()); + } + r.flushBuffer(); + return false; + } + } + } + + // Parse the if[un]modified dates and compare to resource + long date = request.getDateHeader(HttpHeaders.IF_UNMODIFIED_SINCE); + + if (date != -1) { + if (resource.lastModified() / 1000 > date / 1000) { + response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED); + return false; + } + } + + } + } catch (IllegalArgumentException iae) { + if (!response.isCommitted()) { + response.sendError(400, iae.getMessage()); + } + throw iae; + } + return true; + } + + /* ------------------------------------------------------------ */ + protected void sendData(HttpServletRequest request, + HttpServletResponse response, + boolean include, + Resource resource, + HttpContent content, + Enumeration reqRanges) + throws IOException { + boolean direct; + long content_length; + if (content == null) { + direct = false; + content_length = resource.length(); + } else { + Connector connector = AbstractHttpConnection.getCurrentConnection().getConnector(); + direct = connector instanceof NIOConnector && ((NIOConnector) connector).getUseDirectBuffers() && !(connector instanceof SslConnector); + content_length = content.getContentLength(); + } + + + // Get the output stream (or writer) + OutputStream out = null; + boolean written; + try { + out = response.getOutputStream(); + + // has a filter already written to the response? + written = out instanceof HttpOutput + ? ((HttpOutput) out).isWritten() + : AbstractHttpConnection.getCurrentConnection().getGenerator().isWritten(); + } catch (IllegalStateException e) { + out = new WriterOutputStream(response.getWriter()); + written = true; // there may be data in writer buffer, so assume written + } + + if (reqRanges == null || !reqRanges.hasMoreElements() || content_length < 0) { + // if there were no ranges, send entire entity + if (include) { + resource.writeTo(out, 0, content_length); + } else { + // See if a direct methods can be used? + if (content != null && !written && out instanceof HttpOutput) { + if (response instanceof Response) { + writeOptionHeaders(((Response) response).getHttpFields()); + ((AbstractHttpConnection.Output) out).sendContent(content); + } else { + Buffer buffer = direct ? content.getDirectBuffer() : content.getIndirectBuffer(); + if (buffer != null) { + writeHeaders(response, content, content_length); + ((AbstractHttpConnection.Output) out).sendContent(buffer); + } else { + writeHeaders(response, content, content_length); + resource.writeTo(out, 0, content_length); + } + } + } else { + // Write headers normally + writeHeaders(response, content, written ? -1 : content_length); + + // Write content normally + Buffer buffer = (content == null) ? null : content.getIndirectBuffer(); + if (buffer != null) { + buffer.writeTo(out); + } else { + resource.writeTo(out, 0, content_length); + } + } + } + } else { + // Parse the satisfiable ranges + List ranges = InclusiveByteRange.satisfiableRanges(reqRanges, content_length); + + // if there are no satisfiable ranges, send 416 response + if (ranges == null || ranges.size() == 0) { + writeHeaders(response, content, content_length); + response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE); + response.setHeader(HttpHeaders.CONTENT_RANGE, + InclusiveByteRange.to416HeaderRangeString(content_length)); + resource.writeTo(out, 0, content_length); + return; + } + + // if there is only a single valid range (must be satisfiable + // since were here now), send that range with a 216 response + if (ranges.size() == 1) { + InclusiveByteRange singleSatisfiableRange = + (InclusiveByteRange) ranges.get(0); + long singleLength = singleSatisfiableRange.getSize(content_length); + writeHeaders(response, content, singleLength); + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + response.setHeader(HttpHeaders.CONTENT_RANGE, + singleSatisfiableRange.toHeaderRangeString(content_length)); + resource.writeTo(out, singleSatisfiableRange.getFirst(content_length), singleLength); + return; + } + + // multiple non-overlapping valid ranges cause a multipart + // 216 response which does not require an overall + // content-length header + // + writeHeaders(response, content, -1); + String mimetype = (content.getContentType() == null ? null : content.getContentType().toString()); + if (mimetype == null) { + ConcurrentLog.warn("YaCyDefaultServlet", "Unknown mimetype for " + request.getRequestURI()); + } + MultiPartOutputStream multi = new MultiPartOutputStream(out); + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + + // If the request has a "Request-Range" header then we need to + // send an old style multipart/x-byteranges Content-Type. This + // keeps Netscape and acrobat happy. This is what Apache does. + String ctp; + if (request.getHeader(HttpHeaders.REQUEST_RANGE) != null) { + ctp = "multipart/x-byteranges; boundary="; + } else { + ctp = "multipart/byteranges; boundary="; + } + response.setContentType(ctp + multi.getBoundary()); + + InputStream in = resource.getInputStream(); + long pos = 0; + + // calculate the content-length + int length = 0; + String[] header = new String[ranges.size()]; + for (int i = 0; i < ranges.size(); i++) { + InclusiveByteRange ibr = (InclusiveByteRange) ranges.get(i); + header[i] = ibr.toHeaderRangeString(content_length); + length += + ((i > 0) ? 2 : 0) + + 2 + multi.getBoundary().length() + 2 + + (mimetype == null ? 0 : HeaderFramework.CONTENT_TYPE.length() + 2 + mimetype.length()) + 2 + + HeaderFramework.CONTENT_RANGE.length() + 2 + header[i].length() + 2 + + 2 + + (ibr.getLast(content_length) - ibr.getFirst(content_length)) + 1; + } + length += 2 + 2 + multi.getBoundary().length() + 2 + 2; + response.setContentLength(length); + + for (int i = 0; i < ranges.size(); i++) { + InclusiveByteRange ibr = (InclusiveByteRange) ranges.get(i); + multi.startPart(mimetype, new String[]{HeaderFramework.CONTENT_RANGE + ": " + header[i]}); + + long start = ibr.getFirst(content_length); + long size = ibr.getSize(content_length); + if (in != null) { + // Handle non cached resource + if (start < pos) { + in.close(); + in = resource.getInputStream(); + pos = 0; + } + if (pos < start) { + in.skip(start - pos); + pos = start; + } + + FileUtils.copy(in, multi, size); + pos += size; + } else // Handle cached resource + { + (resource).writeTo(multi, start, size); + } + + } + if (in != null) { + in.close(); + } + multi.close(); + } + return; + } + + /* ------------------------------------------------------------ */ + @Override + protected void writeHeaders(HttpServletResponse response, HttpContent content, long count) { + if (content.getContentType() != null && response.getContentType() == null) { + response.setContentType(content.getContentType().toString()); + } + + if (response instanceof Response) { + Response r = (Response) response; + HttpFields fields = r.getHttpFields(); + + if (content.getLastModified() != null) { + fields.put(HttpHeaders.LAST_MODIFIED_BUFFER, content.getLastModified()); + } else if (content.getResource() != null) { + long lml = content.getResource().lastModified(); + if (lml != -1) { + fields.putDateField(HttpHeaders.LAST_MODIFIED_BUFFER, lml); + } + } + + if (count != -1) { + r.setLongContentLength(count); + } + + writeOptionHeaders(fields); + + if (_etags) { + fields.put(HttpHeaders.ETAG_BUFFER, content.getETag()); + } + } else { + long lml = content.getResource().lastModified(); + if (lml >= 0) { + response.setDateHeader(HeaderFramework.LAST_MODIFIED, lml); + } + + if (count != -1) { + if (count < Integer.MAX_VALUE) { + response.setContentLength((int) count); + } else { + response.setHeader(HeaderFramework.CONTENT_LENGTH, Long.toString(count)); + } + } + + writeOptionHeaders(response); + + if (_etags) { + response.setHeader(HeaderFramework.ETAG, content.getETag().toString()); + } + } + } + + /* ------------------------------------------------------------ */ + @Override + protected void writeOptionHeaders(HttpFields fields) { + if (_acceptRanges) { + fields.put(HttpHeaders.ACCEPT_RANGES_BUFFER, HttpHeaderValues.BYTES_BUFFER); + } + + } + + /* ------------------------------------------------------------ */ + /* + * @see javax.servlet.Servlet#destroy() + */ + @Override + public void destroy() { + + super.destroy(); + } +} diff --git a/source/net/yacy/http/Jetty8YaCySecurityHandler.java b/source/net/yacy/http/Jetty8YaCySecurityHandler.java new file mode 100644 index 000000000..1261342ec --- /dev/null +++ b/source/net/yacy/http/Jetty8YaCySecurityHandler.java @@ -0,0 +1,200 @@ +// +// YaCySecurityHandler +// Copyright 2011 by Florian Richter +// First released 16.04.2011 at http://yacy.net +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program in the file lgpl21.txt +// If not, see <http://www.gnu.org/licenses/>. +// + +package net.yacy.http; + +import java.io.IOException; +import java.net.MalformedURLException; +import net.yacy.cora.document.id.MultiProtocolURL; + +import net.yacy.cora.protocol.Domains; +import net.yacy.search.Switchboard; +import org.eclipse.jetty.http.HttpSchemes; +import org.eclipse.jetty.security.RoleInfo; + +import org.eclipse.jetty.security.SecurityHandler; +import org.eclipse.jetty.security.UserDataConstraint; +import org.eclipse.jetty.server.AbstractHttpConnection; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.server.UserIdentity; + +/** + * jetty security handler + * demands authentication for pages with _p. inside + * and updates AccessTracker + */ +public class Jetty8YaCySecurityHandler extends SecurityHandler { + + @Override + protected boolean checkUserDataPermissions(String pathInContext, Request request, Response response, Object constraintInfo) throws IOException + // check the SecurityHandler code, denying here does not provide authentication + // - identical with ConstraintSecurityHandler.checkUserDataPermissions implementation of Jetty source distribution + { + if (constraintInfo == null) + return true; + + RoleInfo roleInfo = (RoleInfo)constraintInfo; + if (roleInfo.isForbidden()) + return false; + + + UserDataConstraint dataConstraint = roleInfo.getUserDataConstraint(); + if (dataConstraint == null || dataConstraint == UserDataConstraint.None) + { + return true; + } + AbstractHttpConnection connection = AbstractHttpConnection.getCurrentConnection(); + Connector connector = connection.getConnector(); + + if (dataConstraint == UserDataConstraint.Integral) + { + if (connector.isIntegral(request)) + return true; + if (connector.getIntegralPort() > 0) + { + String scheme=connector.getIntegralScheme(); + int port=connector.getIntegralPort(); + String url = (HttpSchemes.HTTPS.equalsIgnoreCase(scheme) && port==443) + ? "https://"+request.getServerName()+request.getRequestURI() + : scheme + "://" + request.getServerName() + ":" + port + request.getRequestURI(); + if (request.getQueryString() != null) + url += "?" + request.getQueryString(); + response.setContentLength(0); + response.sendRedirect(url); + } + else + response.sendError(Response.SC_FORBIDDEN,"!Integral"); + + request.setHandled(true); + return false; + } + else if (dataConstraint == UserDataConstraint.Confidential) + { + if (connector.isConfidential(request)) + return true; + + if (connector.getConfidentialPort() > 0) + { + String scheme=connector.getConfidentialScheme(); + int port=connector.getConfidentialPort(); + String url = (HttpSchemes.HTTPS.equalsIgnoreCase(scheme) && port==443) + ? "https://"+request.getServerName()+request.getRequestURI() + : scheme + "://" + request.getServerName() + ":" + port + request.getRequestURI(); + if (request.getQueryString() != null) + url += "?" + request.getQueryString(); + response.setContentLength(0); + response.sendRedirect(url); + } + else + response.sendError(Response.SC_FORBIDDEN,"!Confidential"); + + request.setHandled(true); + return false; + } + else + { + throw new IllegalArgumentException("Invalid dataConstraint value: " + dataConstraint); + } + } + + @Override + protected boolean checkWebResourcePermissions(String pathInContext, Request request, + Response response, Object constraintInfo, UserIdentity userIdentity) throws IOException { + // deny and request for authentication, if necessary + // - identical with ConstraintSecurityHandler.checkWebResourcePermissions implementation of Jetty source distribution + if (constraintInfo == null) { + return true; + } + RoleInfo roleInfo = (RoleInfo) constraintInfo; + + if (!roleInfo.isChecked()) { + return true; + } + + if (roleInfo.isAnyRole() && request.getAuthType() != null) { + return true; + } + + for (String role : roleInfo.getRoles()) { + if (userIdentity.isUserInRole(role, null)) { + return true; + } + } + return false; + } + + @Override + protected boolean isAuthMandatory(Request baseRequest, Response base_response, Object constraintInfo) { + // identical with ConstraintSecurityHandler.isAuthMandatory implementation of Jetty source distribution + return constraintInfo != null && ((RoleInfo) constraintInfo).isChecked(); + } + + /** + * create the constraint for the given path + * for urls containing *_p. (like info_p.html) admin access is required, + * on localhost = admin setting no constraint is set + * @param pathInContext + * @param request + * @return RoleInfo with + * isChecked=true if any security contraint applies (compare reference implementation org.eclipse.jetty.security.ConstraintSecurityHandler) + * role = "admin" for resource name containint _p. + */ + @Override + protected RoleInfo prepareConstraintInfo(String pathInContext, Request request) { + final Switchboard sb = Switchboard.getSwitchboard(); + final boolean adminAccountForLocalhost = sb.getConfigBool("adminAccountForLocalhost", false); + //final String adminAccountBase64MD5 = sb.getConfig(YaCyLegacyCredential.ADMIN_ACCOUNT_B64MD5, ""); + + String refererHost; + // update AccessTracker + refererHost = request.getRemoteAddr(); + sb.track(refererHost, pathInContext); + + try { + refererHost = new MultiProtocolURL(request.getHeader("Referer")).getHost(); + } catch (MalformedURLException e) { + refererHost = null; + } + final boolean accessFromLocalhost = Domains.isLocalhost(request.getRemoteHost()) && (refererHost == null || refererHost.length() == 0 || Domains.isLocalhost(refererHost)); + // ! note : accessFromLocalhost compares localhost ip pattern ( ! currently also any intranet host is a local host) + final boolean grantedForLocalhost = adminAccountForLocalhost && accessFromLocalhost; + final boolean protectedPage = pathInContext.indexOf("_p.") > 0; + //final boolean accountEmpty = adminAccountBase64MD5.length() == 0; + //final boolean yacyBot = request.getHeader("User-Agent").startsWith("yacybot"); + + if (protectedPage) { // TODO: none public site + if (!grantedForLocalhost) { + RoleInfo roleinfo = new RoleInfo(); + roleinfo.setChecked(true); // RoleInfo.setChecked() : in Jetty this means - marked to have any security constraint + roleinfo.addRole("admin"); //YaCyLoginService assigns "admin" role to admin user + return roleinfo; + } // can omit else, as if grantedForLocalhost==true no constraint applies + // TODO: is this correct or adminAccountBase64MD5 not empty check neccessary ? + } + return null; + } + +} diff --git a/source/net/yacy/http/YaCyDefaultServlet.java b/source/net/yacy/http/YaCyDefaultServlet.java index 6cb3ebf32..6858b94ee 100644 --- a/source/net/yacy/http/YaCyDefaultServlet.java +++ b/source/net/yacy/http/YaCyDefaultServlet.java @@ -33,13 +33,11 @@ import java.lang.ref.SoftReference; import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.URL;
-import java.nio.ByteBuffer;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
-import javax.servlet.AsyncContext;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
@@ -72,24 +70,18 @@ import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.eclipse.jetty.http.HttpContent;
import org.eclipse.jetty.http.HttpFields;
-import org.eclipse.jetty.http.HttpHeader;
-import org.eclipse.jetty.http.HttpMethod;
import org.eclipse.jetty.http.MimeTypes;
-import org.eclipse.jetty.io.WriterOutputStream;
-import org.eclipse.jetty.server.HttpOutput;
-import org.eclipse.jetty.server.InclusiveByteRange;
-import org.eclipse.jetty.server.Response;
-import org.eclipse.jetty.util.BufferUtil;
-import org.eclipse.jetty.util.Callback;
-import org.eclipse.jetty.util.MultiPartOutputStream;
-import org.eclipse.jetty.util.QuotedStringTokenizer;
import org.eclipse.jetty.util.URIUtil;
import org.eclipse.jetty.util.resource.Resource;
import org.eclipse.jetty.util.resource.ResourceFactory;
/**
- * YaCyDefaultServlet base on Jetty DefaultServlet.java
+ * YaCyDefaultServlet based on Jetty DefaultServlet.java
* handles static files and the YaCy servlets.
+ *
+ * This interface impements the YaCy specific and standard Servlet routines
+ * which should not have a dependency on the implemented Jetty version.
+ * The Jetty version specific code is moved to the Jetty8HttpServerImpl.java implementation
*/
/**
@@ -123,29 +115,29 @@ import org.eclipse.jetty.util.resource.ResourceFactory; *
* </PRE>
*/
-public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory {
+public abstract class YaCyDefaultServlet extends HttpServlet implements ResourceFactory {
private static final long serialVersionUID = 4900000000000001110L;
- private ServletContext _servletContext;
- private boolean _acceptRanges = true;
- private boolean _dirAllowed = true;
- private boolean _pathInfoOnly = false;
- private boolean _etags = false;
- private Resource _resourceBase;
- private MimeTypes _mimeTypes;
- private String[] _welcomes;
- private String _relativeResourceBase;
+ protected ServletContext _servletContext;
+ protected boolean _acceptRanges = true;
+ protected boolean _dirAllowed = true;
+ protected boolean _pathInfoOnly = false;
+ protected boolean _etags = false;
+ protected Resource _resourceBase;
+ protected MimeTypes _mimeTypes;
+ protected String[] _welcomes;
+ protected String _relativeResourceBase;
- private File htLocalePath;
- private File htDocsPath;
- private static final serverClassLoader provider = new serverClassLoader(/*this.getClass().getClassLoader()*/);
- private ConcurrentHashMap<File, SoftReference<Method>> templateMethodCache = null;
+ protected File _htLocalePath;
+ protected File _htDocsPath;
+ protected static final serverClassLoader provider = new serverClassLoader(/*this.getClass().getClassLoader()*/);
+ protected ConcurrentHashMap<File, SoftReference<Method>> templateMethodCache = null;
/* ------------------------------------------------------------ */
@Override
public void init() throws UnavailableException {
- htDocsPath = Switchboard.getSwitchboard().htDocsPath;
- htLocalePath = Switchboard.getSwitchboard().getDataPath("locale.translated_html", "DATA/LOCALE/htroot");
+ _htDocsPath = Switchboard.getSwitchboard().htDocsPath;
+ _htLocalePath = Switchboard.getSwitchboard().getDataPath("locale.translated_html", "DATA/LOCALE/htroot");
_servletContext = getServletContext();
@@ -185,7 +177,7 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { /* ------------------------------------------------------------ */
- private boolean getInitBoolean(String name, boolean dft) {
+ protected boolean getInitBoolean(String name, boolean dft) {
String value = getInitParameter(name);
if (value == null || value.length() == 0) {
return dft;
@@ -357,13 +349,13 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { if (content != null) {
content.release();
} else if (resource != null) {
- resource.close();
+ resource.release();
}
}
}
/* ------------------------------------------------------------ */
- private boolean hasDefinedRange(Enumeration<String> reqRanges) {
+ protected boolean hasDefinedRange(Enumeration<String> reqRanges) {
return (reqRanges != null && reqRanges.hasMoreElements());
}
@@ -398,7 +390,7 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { * @param path in context
* @return The path of the matching welcome file in context or null.
*/
- private String getWelcomeFile(String pathInContext) {
+ protected String getWelcomeFile(String pathInContext) {
if (_welcomes == null) {
return null;
}
@@ -415,104 +407,8 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { /* ------------------------------------------------------------ */
/* Check modification date headers.
*/
- protected boolean passConditionalHeaders(HttpServletRequest request, HttpServletResponse response, Resource resource, HttpContent content)
- throws IOException {
- try {
- if (!HttpMethod.HEAD.is(request.getMethod())) {
- if (_etags) {
- String ifm = request.getHeader(HttpHeader.IF_MATCH.asString());
- if (ifm != null) {
- boolean match = false;
- if (content.getETag() != null) {
- QuotedStringTokenizer quoted = new QuotedStringTokenizer(ifm, ", ", false, true);
- while (!match && quoted.hasMoreTokens()) {
- String tag = quoted.nextToken();
- if (content.getETag().toString().equals(tag)) {
- match = true;
- }
- }
- }
-
- if (!match) {
- response.setStatus(HttpServletResponse.SC_PRECONDITION_FAILED);
- return false;
- }
- }
-
- String if_non_match_etag = request.getHeader(HttpHeader.IF_NONE_MATCH.asString());
- if (if_non_match_etag != null && content.getETag() != null) {
- // Look for GzipFiltered version of etag
- if (content.getETag().toString().equals(request.getAttribute("o.e.j.s.GzipFilter.ETag"))) {
- response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
- response.setHeader(HeaderFramework.ETAG, if_non_match_etag);
- return false;
- }
-
- // Handle special case of exact match.
- if (content.getETag().toString().equals(if_non_match_etag)) {
- response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
- response.setHeader(HeaderFramework.ETAG, content.getETag());
- return false;
- }
-
- // Handle list of tags
- QuotedStringTokenizer quoted = new QuotedStringTokenizer(if_non_match_etag, ", ", false, true);
- while (quoted.hasMoreTokens()) {
- String tag = quoted.nextToken();
- if (content.getETag().toString().equals(tag)) {
- response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
- response.setHeader(HeaderFramework.ETAG, content.getETag());
- return false;
- }
- }
-
- // If etag requires content to be served, then do not check if-modified-since
- return true;
- }
- }
-
- // Handle if modified since
- String ifms = request.getHeader(HttpHeader.IF_MODIFIED_SINCE.asString());
- if (ifms != null) {
- //Get jetty's Response impl
- String mdlm = content.getLastModified();
- if (mdlm != null && ifms.equals(mdlm)) {
- response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
- if (_etags) {
- response.setHeader(HeaderFramework.ETAG, content.getETag());
- }
- response.flushBuffer();
- return false;
- }
-
- long ifmsl = request.getDateHeader(HttpHeader.IF_MODIFIED_SINCE.asString());
- if (ifmsl != -1 && resource.lastModified() / 1000 <= ifmsl / 1000) {
- response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
- if (_etags) {
- response.setHeader(HeaderFramework.ETAG, content.getETag());
- }
- response.flushBuffer();
- return false;
- }
- }
-
- // Parse the if[un]modified dates and compare to resource
- long date = request.getDateHeader(HttpHeader.IF_UNMODIFIED_SINCE.asString());
- if (date != -1 && resource.lastModified() / 1000 > date / 1000) {
- response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED);
- return false;
- }
-
- }
- } catch (IllegalArgumentException iae) {
- if (!response.isCommitted()) {
- response.sendError(400, iae.getMessage());
- }
- throw iae;
- }
- return true;
- }
-
+ abstract protected boolean passConditionalHeaders(HttpServletRequest request, HttpServletResponse response, Resource resource, HttpContent content)
+ throws IOException;
/* ------------------------------------------------------------------- */
protected void sendDirectory(HttpServletRequest request,
@@ -542,228 +438,16 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { }
/* ------------------------------------------------------------ */
- protected void sendData(HttpServletRequest request,
+ abstract protected void sendData(HttpServletRequest request,
HttpServletResponse response,
boolean include,
Resource resource,
HttpContent content,
Enumeration<String> reqRanges)
- throws IOException {
- final long content_length = (content == null) ? resource.length() : content.getContentLength();
-
- // Get the output stream (or writer)
- OutputStream out = null;
- boolean written;
- try {
- out = response.getOutputStream();
-
- // has a filter already written to the response?
- written = out instanceof HttpOutput
- ? ((HttpOutput) out).isWritten()
- : true;
- } catch (IllegalStateException e) {
- out = new WriterOutputStream(response.getWriter());
- written = true; // there may be data in writer buffer, so assume written
- }
-
- if (reqRanges == null || !reqRanges.hasMoreElements() || content_length < 0) {
- // if there were no ranges, send entire entity
- if (include) {
- resource.writeTo(out, 0, content_length);
- } // else if we can't do a bypass write because of wrapping
- else if (content == null || written || !(out instanceof HttpOutput)) {
- // write normally
- writeHeaders(response, content, written ? -1 : content_length);
- ByteBuffer buffer = (content == null) ? null : content.getIndirectBuffer();
- if (buffer != null) {
- BufferUtil.writeTo(buffer, out);
- } else {
- resource.writeTo(out, 0, content_length);
- }
- } // else do a bypass write
- else {
- // write the headers
- if (response instanceof Response) {
- Response r = (Response) response;
- writeOptionHeaders(r.getHttpFields());
- r.setHeaders(content);
- } else {
- writeHeaders(response, content, content_length);
- }
-
- // write the content asynchronously if supported
- if (request.isAsyncSupported()) {
- final AsyncContext context = request.startAsync();
-
- ((HttpOutput) out).sendContent(content, new Callback() {
- @Override
- public void succeeded() {
- context.complete();
- }
-
- @Override
- public void failed(Throwable x) {
- ConcurrentLog.logException(x);
- context.complete();
- }
- });
- } // otherwise write content blocking
- else {
- ((HttpOutput) out).sendContent(content);
- }
- }
- } else {
- // Parse the satisfiable ranges
- List<InclusiveByteRange> ranges = InclusiveByteRange.satisfiableRanges(reqRanges, content_length);
-
- // if there are no satisfiable ranges, send 416 response
- if (ranges == null || ranges.size() == 0) {
- writeHeaders(response, content, content_length);
- response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
- response.setHeader(HeaderFramework.CONTENT_RANGE,
- InclusiveByteRange.to416HeaderRangeString(content_length));
- resource.writeTo(out, 0, content_length);
- return;
- }
-
- // if there is only a single valid range (must be satisfiable
- // since were here now), send that range with a 216 response
- if (ranges.size() == 1) {
- InclusiveByteRange singleSatisfiableRange = ranges.get(0);
- long singleLength = singleSatisfiableRange.getSize(content_length);
- writeHeaders(response, content, singleLength);
- response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
- response.setHeader(HeaderFramework.CONTENT_RANGE,
- singleSatisfiableRange.toHeaderRangeString(content_length));
- resource.writeTo(out, singleSatisfiableRange.getFirst(content_length), singleLength);
- return;
- }
-
- // multiple non-overlapping valid ranges cause a multipart
- // 216 response which does not require an overall
- // content-length header
- //
- writeHeaders(response, content, -1);
- String mimetype = (content == null || content.getContentType() == null ? null : content.getContentType().toString());
- if (mimetype == null) {
- ConcurrentLog.warn("YaCyDefaultServlet", "Unknown mimetype for " + request.getRequestURI());
- }
- MultiPartOutputStream multi = new MultiPartOutputStream(out);
- response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
-
- // If the request has a "Request-Range" header then we need to
- // send an old style multipart/x-byteranges Content-Type. This
- // keeps Netscape and acrobat happy. This is what Apache does.
- String ctp;
- if (request.getHeader(HttpHeader.REQUEST_RANGE.asString()) != null) {
- ctp = "multipart/x-byteranges; boundary=";
- } else {
- ctp = "multipart/byteranges; boundary=";
- }
- response.setContentType(ctp + multi.getBoundary());
-
- InputStream in = resource.getInputStream();
- long pos = 0;
-
- // calculate the content-length
- int length = 0;
- String[] header = new String[ranges.size()];
- for (int i = 0; i < ranges.size(); i++) {
- InclusiveByteRange ibr = ranges.get(i);
- header[i] = ibr.toHeaderRangeString(content_length);
- length +=
- ((i > 0) ? 2 : 0)
- + 2 + multi.getBoundary().length() + 2
- + (mimetype == null ? 0 : HeaderFramework.CONTENT_TYPE.length() + 2 + mimetype.length()) + 2
- + HeaderFramework.CONTENT_RANGE.length() + 2 + header[i].length() + 2
- + 2
- + (ibr.getLast(content_length) - ibr.getFirst(content_length)) + 1;
- }
- length += 2 + 2 + multi.getBoundary().length() + 2 + 2;
- response.setContentLength(length);
-
- for (int i = 0; i < ranges.size(); i++) {
- InclusiveByteRange ibr = ranges.get(i);
- multi.startPart(mimetype, new String[]{HeaderFramework.CONTENT_RANGE + ": " + header[i]});
-
- long start = ibr.getFirst(content_length);
- long size = ibr.getSize(content_length);
- if (in != null) {
- // Handle non cached resource
- if (start < pos) {
- in.close();
- in = resource.getInputStream();
- pos = 0;
- }
- if (pos < start) {
- in.skip(start - pos);
- pos = start;
- }
-
- FileUtils.copy(in, multi, size);
- pos += size;
- } else // Handle cached resource
- {
- (resource).writeTo(multi, start, size);
- }
-
- }
- if (in != null) {
- in.close();
- }
- multi.close();
- }
- }
+ throws IOException;
/* ------------------------------------------------------------ */
- protected void writeHeaders(HttpServletResponse response, HttpContent content, long count) {
- if (content.getContentType() != null && response.getContentType() == null) {
- response.setContentType(content.getContentType().toString());
- }
-
- if (response instanceof Response) {
- Response r = (Response) response;
- HttpFields fields = r.getHttpFields();
-
- if (content.getLastModified() != null) {
- fields.put(HeaderFramework.LAST_MODIFIED, content.getLastModified());
- } else if (content.getResource() != null) {
- long lml = content.getResource().lastModified();
- if (lml != -1) {
- fields.putDateField(HeaderFramework.LAST_MODIFIED, lml);
- }
- }
-
- if (count != -1) {
- r.setLongContentLength(count);
- }
-
- writeOptionHeaders(fields);
-
- if (_etags) {
- fields.put(HeaderFramework.ETAG, content.getETag());
- }
- } else {
- long lml = content.getResource().lastModified();
- if (lml >= 0) {
- response.setDateHeader(HeaderFramework.LAST_MODIFIED, lml);
- }
-
- if (count != -1) {
- if (count < Integer.MAX_VALUE) {
- response.setContentLength((int) count);
- } else {
- response.setHeader(HeaderFramework.CONTENT_LENGTH, Long.toString(count));
- }
- }
-
- writeOptionHeaders(response);
-
- if (_etags) {
- response.setHeader(HeaderFramework.ETAG, content.getETag().toString());
- }
- }
- }
+ abstract protected void writeHeaders(HttpServletResponse response, HttpContent content, long count);
/* ------------------------------------------------------------ */
protected void writeOptionHeaders(HttpFields fields) {
@@ -780,11 +464,11 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { }
- private Object invokeServlet(final File targetClass, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException {
+ protected Object invokeServlet(final File targetClass, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException {
return rewriteMethod(targetClass).invoke(null, new Object[]{request, args, Switchboard.getSwitchboard()}); // add switchboard
}
- private RequestHeader generateLegacyRequestHeader(HttpServletRequest request, String target, String targetExt) {
+ protected RequestHeader generateLegacyRequestHeader(HttpServletRequest request, String target, String targetExt) {
RequestHeader legacyRequestHeader = new RequestHeader();
@SuppressWarnings("unchecked")
Enumeration<String> headers = request.getHeaderNames();
@@ -814,20 +498,20 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { */
public File getLocalizedFile(final String path, final String localeSelection) throws IOException {
if (!(localeSelection.equals("default"))) {
- final File localePath = new File(htLocalePath, localeSelection + '/' + path);
+ final File localePath = new File(_htLocalePath, localeSelection + '/' + path);
if (localePath.exists()) {
return localePath; // avoid "NoSuchFile" troubles if the "localeSelection" is misspelled
}
}
- File docsPath = new File(htDocsPath, path);
+ File docsPath = new File(_htDocsPath, path);
if (docsPath.exists()) {
return docsPath;
}
return _resourceBase.addPath(path).getFile();
}
- private File rewriteClassFile(final File template) {
+ protected File rewriteClassFile(final File template) {
try {
String f = template.getCanonicalPath();
final int p = f.lastIndexOf('.');
@@ -845,7 +529,7 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { }
}
- private Method rewriteMethod(final File classFile) throws InvocationTargetException {
+ protected Method rewriteMethod(final File classFile) throws InvocationTargetException {
Method m = null;
// now make a class out of the stream
try {
@@ -879,7 +563,7 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { return m;
}
- public void handleTemplate(String target, HttpServletRequest request,
+ protected void handleTemplate(String target, HttpServletRequest request,
HttpServletResponse response) throws IOException, ServletException {
Switchboard sb = Switchboard.getSwitchboard();
@@ -1047,7 +731,7 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { }
// parse SSI line and include resource
- private void parseSSI(final net.yacy.cora.util.ByteBuffer in, final int off, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+ protected void parseSSI(final net.yacy.cora.util.ByteBuffer in, final int off, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
if (in.startsWith("<!--#include virtual=\"".getBytes(), off)) {
final int q = in.indexOf("\"".getBytes(), off + 22);
if (q > 0) {
@@ -1072,7 +756,7 @@ public class YaCyDefaultServlet extends HttpServlet implements ResourceFactory { * @param request
* @param args found fields/values are added to the map
*/
- public void parseMultipart(HttpServletRequest request, serverObjects args) {
+ protected void parseMultipart(HttpServletRequest request, serverObjects args) {
DiskFileItemFactory factory = new DiskFileItemFactory();
// maximum size that will be stored in memory
factory.setSizeThreshold(4096 * 16);
diff --git a/source/net/yacy/http/YaCyHttpServer.java b/source/net/yacy/http/YaCyHttpServer.java new file mode 100644 index 000000000..22c881aa7 --- /dev/null +++ b/source/net/yacy/http/YaCyHttpServer.java @@ -0,0 +1,29 @@ +
+package net.yacy.http;
+
+import java.net.InetSocketAddress;
+import java.net.SocketException;
+
+/**
+ * Isolation of HttpServer
+ *
+ * Development Goal: allow for individual implementation of a HttpServer
+ * to provide the routines and entry points required by the
+ * YaCy servlets
+ *
+ * currently Jetty implementation is ongoing
+ *
+ * Implementation Jetty8HttpServerImpl.java
+ */
+public interface YaCyHttpServer {
+
+ abstract public void startupServer() throws Exception;
+ abstract public void stop() throws Exception;
+ abstract public void setMaxSessionCount(int cnt);
+ abstract public InetSocketAddress generateSocketAddress(String port) throws SocketException;
+ abstract public int getMaxSessionCount();
+ abstract public int getJobCount();
+ abstract public boolean withSSL();
+ abstract public void reconnect(int milsec);
+ abstract public String getVersion();
+}
diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java index 2faa72b66..a4e532b50 100644 --- a/source/net/yacy/server/serverSwitch.java +++ b/source/net/yacy/server/serverSwitch.java @@ -47,7 +47,7 @@ import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.http.HTTPClient;
import net.yacy.cora.util.ConcurrentLog;
-import net.yacy.http.HttpServer;
+import net.yacy.http.YaCyHttpServer;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.workflow.BusyThread;
import net.yacy.kelondro.workflow.WorkflowThread;
@@ -70,7 +70,7 @@ public class serverSwitch private final ConcurrentMap<InetAddress, String> authorization;
private final NavigableMap<String, BusyThread> workerThreads;
private final serverAccessTracker accessTracker;
- private HttpServer httpserver;
+ private YaCyHttpServer httpserver; // implemented HttpServer
public serverSwitch(
final File dataPath,
@@ -645,10 +645,10 @@ public class serverSwitch * set/remember jetty server
* @param jettyserver
*/
- public void setHttpServer(HttpServer jettyserver) {
+ public void setHttpServer(YaCyHttpServer jettyserver) {
this.httpserver = jettyserver;
}
- public HttpServer getHttpServer() {
+ public YaCyHttpServer getHttpServer() {
return httpserver;
}
diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index ed588195c..9eac74edc 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -52,7 +52,8 @@ import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.Translator;
import net.yacy.gui.YaCyApp;
import net.yacy.gui.framework.Browser;
-import net.yacy.http.HttpServer;
+import net.yacy.http.YaCyHttpServer;
+import net.yacy.http.Jetty8HttpServerImpl;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.Formatter;
import net.yacy.kelondro.util.MemoryControl;
@@ -331,8 +332,10 @@ public final class yacy { try {
// start jetty http server
- HttpServer httpServer = new HttpServer(port);
- httpServer.start();
+ YaCyHttpServer httpServer = new Jetty8HttpServerImpl(port);
+ httpServer.startupServer();
+ sb.setHttpServer(httpServer);
+ ConcurrentLog.info("STARTUP",httpServer.getVersion());
//final HTTPDemon protocolHandler = new HTTPDemon(sb);
//final serverCore server = new serverCore(
// timeout /*control socket timeout in milliseconds*/,
@@ -345,8 +348,7 @@ public final class yacy { //server.setObeyIntermission(false);
// start the server
- //sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0);
- sb.setHttpServer(httpServer);
+ //sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0);
//server.start();
// open the browser window
diff --git a/test/de/anomic/document/ParserTest.java b/test/de/anomic/document/ParserTest.java index 66b5891b1..701506438 100644 --- a/test/de/anomic/document/ParserTest.java +++ b/test/de/anomic/document/ParserTest.java @@ -11,6 +11,7 @@ import java.io.InputStreamReader; import java.io.Reader; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; +import net.yacy.cora.document.id.AnchorURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; @@ -19,7 +20,6 @@ import net.yacy.document.parser.docParser; import net.yacy.document.parser.odtParser; import net.yacy.document.parser.ooxmlParser; import net.yacy.document.parser.pdfParser; -import net.yacy.kelondro.data.meta.DigestURI; import org.junit.Test; @@ -38,7 +38,7 @@ public class ParserTest { final String filename = "test/parsertest/" + testFile[0]; final File file = new File(filename); final String mimetype = testFile[1]; - final DigestURI url = new DigestURI("http://localhost/"+filename); + final AnchorURL url = new AnchorURL("http://localhost/"+filename); AbstractParser p = new ooxmlParser(); final Document[] docs = p.parse(url, mimetype, null, new FileInputStream(file)); @@ -72,7 +72,7 @@ public class ParserTest { final String filename = "test/parsertest/" + testFile[0]; final File file = new File(filename); final String mimetype = testFile[1]; - final DigestURI url = new DigestURI("http://localhost/"+filename); + final AnchorURL url = new AnchorURL("http://localhost/"+filename); AbstractParser p = new odtParser(); final Document[] docs = p.parse(url, mimetype, null, new FileInputStream(file)); @@ -104,7 +104,7 @@ public class ParserTest { final String filename = "test/parsertest/" + testFile[0]; final File file = new File(filename); final String mimetype = testFile[1]; - final DigestURI url = new DigestURI("http://localhost/"+filename); + final AnchorURL url = new AnchorURL("http://localhost/"+filename); AbstractParser p = new pdfParser(); final Document[] docs = p.parse(url, mimetype, null, new FileInputStream(file)); @@ -136,7 +136,7 @@ public class ParserTest { final String filename = "test/parsertest/" + testFile[0]; final File file = new File(filename); final String mimetype = testFile[1]; - final DigestURI url = new DigestURI("http://localhost/"+filename); + final AnchorURL url = new AnchorURL("http://localhost/"+filename); AbstractParser p = new docParser(); final Document[] docs = p.parse(url, mimetype, null, new FileInputStream(file)); diff --git a/test/de/anomic/yacy/yacyURLTest.java b/test/de/anomic/yacy/yacyURLTest.java index a0439a4e1..00bc8b78c 100644 --- a/test/de/anomic/yacy/yacyURLTest.java +++ b/test/de/anomic/yacy/yacyURLTest.java @@ -2,10 +2,10 @@ package de.anomic.yacy; import java.net.MalformedURLException; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.kelondro.data.meta.DigestURI; import junit.framework.TestCase; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; public class yacyURLTest extends TestCase { @@ -32,7 +32,7 @@ public class yacyURLTest extends TestCase { // conversion result String resolvedURL=""; try { - resolvedURL = (new MultiProtocolURI(testhost + testStrings[i][0])).toString(); + resolvedURL = (new MultiProtocolURL(testhost + testStrings[i][0])).toString(); } catch (final MalformedURLException ex) { fail ("malformed URL"); } @@ -57,7 +57,7 @@ public class yacyURLTest extends TestCase { String shouldBe = testStrings[i][1]; // conversion result - String resolvedURL = (new DigestURI(testStrings[i][0])).toString(); + String resolvedURL = (new DigestURL(testStrings[i][0])).toString(); // test if equal assertEquals(shouldBe,resolvedURL); diff --git a/test/net/yacy/cora/document/MultiProtocolURITest.java b/test/net/yacy/cora/document/MultiProtocolURITest.java index f69098727..ae6cf5ab1 100644 --- a/test/net/yacy/cora/document/MultiProtocolURITest.java +++ b/test/net/yacy/cora/document/MultiProtocolURITest.java @@ -4,8 +4,7 @@ import static org.junit.Assert.*; import java.net.MalformedURLException; import java.util.TreeSet; - -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; import org.junit.Test; @@ -23,10 +22,10 @@ public class MultiProtocolURITest { TreeSet<String> idNames = new TreeSet<String>(); idNames.add("phpsessionid"); - MultiProtocolURI.initSessionIDNames(idNames); + MultiProtocolURL.initSessionIDNames(idNames); for (int i=0; i<testURIs.length; i++) { - MultiProtocolURI uri = new MultiProtocolURI(testURIs[i][0]); + MultiProtocolURL uri = new MultiProtocolURL(testURIs[i][0]); assertEquals(uri.toNormalform(true, true), testURIs[i][1]); } diff --git a/test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java b/test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java index 69ae7abbf..ef43842e0 100644 --- a/test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java +++ b/test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java @@ -4,6 +4,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.EnumSet; +import javax.servlet.DispatcherType; import net.yacy.cora.federate.solr.SolrServlet; import net.yacy.cora.federate.solr.instance.EmbeddedInstance; import net.yacy.search.schema.CollectionSchema; @@ -13,8 +15,9 @@ import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.ServletContextHandler.Context; +import org.eclipse.jetty.servlet.ServletContextHandler; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -122,14 +125,19 @@ public class EmbeddedSolrConnectorTest { this.server.setConnectors(new Connector[] { connector }); this.server.setSessionIdManager(new HashSessionIdManager(new Random())); */ + ServerConnector connector = new ServerConnector(server); + connector.setPort(port); + + server.addConnector(connector); server.setStopAtShutdown(true); - Context root = new Context(server, context, Context.SESSIONS); + ServletContextHandler root = new ServletContextHandler(ServletContextHandler.SESSIONS); //new Context(server, context, Context.SESSIONS); root.addServlet(SolrServlet.Servlet404.class, "/*"); // attach org.apache.solr.response.XMLWriter to search requests SolrServlet.initCore(c); - FilterHolder dispatchFilter = root.addFilter(SolrServlet.class, "*", Handler.REQUEST); - + FilterHolder dispatchFilter = root.addFilter(SolrServlet.class, "*", EnumSet.of(DispatcherType.REQUEST)); + //root.addFilter(new FilterHolder(SolrServlet.class), "/*", EnumSet.of(DispatcherType.REQUEST)); + server.setHandler(root); if (!server.isRunning()) { try { server.start(); diff --git a/source/net/yacy/http/HttpServer.java b/test/net/yacy/http/Jetty9HttpServerImpl.java index 3ca32f05f..5247d4194 100644 --- a/source/net/yacy/http/HttpServer.java +++ b/test/net/yacy/http/Jetty9HttpServerImpl.java @@ -1,5 +1,5 @@ // -// HttpServer +// Jetty9HttpServerImpl // Copyright 2011 by Florian Richter // First released 13.04.2011 at http://yacy.net // @@ -45,16 +45,16 @@ import org.eclipse.jetty.servlet.ServletContextHandler; import org.eclipse.jetty.servlet.ServletHolder; /** - * class to embedded jetty http server into YaCy + * class to embedded Jetty 9 http server into YaCy */ -public class HttpServer { +public class Jetty9HttpServerImpl implements YaCyHttpServer { private Server server; /** * @param port TCP Port to listen for http requests */ - public HttpServer(int port) { + public Jetty9HttpServerImpl(int port) { Switchboard sb = Switchboard.getSwitchboard(); server = new Server(); @@ -85,10 +85,11 @@ public class HttpServer { // configure root context ServletContextHandler htrootContext = new ServletContextHandler(ServletContextHandler.SESSIONS); htrootContext.setContextPath("/"); - ServletHolder sholder = new ServletHolder(YaCyDefaultServlet.class); - sholder.setInitParameter("resourceBase", "htroot"); + ServletHolder sholder = new ServletHolder(Jetty9YaCyDefaultServlet.class); + sholder.setInitParameter("resourceBase", "htroot"); + //sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html + sholder.setInitParameter("gzip","false"); htrootContext.addServlet(sholder,"/*"); - //htrootContext.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html // assemble the servlet handlers ContextHandlerCollection servletContext = new ContextHandlerCollection(); @@ -113,7 +114,7 @@ public class HttpServer { allrequesthandlers.addHandler(new DefaultHandler()); // if not handled by other handler // wrap all handlers by security handler - YaCySecurityHandler securityHandler = new YaCySecurityHandler(); + Jetty9YaCySecurityHandler securityHandler = new Jetty9YaCySecurityHandler(); securityHandler.setLoginService(new YaCyLoginService()); securityHandler.setRealmName("YaCy Admin Interface"); securityHandler.setHandler(new CrashProtectionHandler(allrequesthandlers)); @@ -124,26 +125,31 @@ public class HttpServer { /** * start http server */ - public void start() throws Exception { + @Override + public void startupServer() throws Exception { server.start(); } /** * stop http server and wait for it */ + @Override public void stop() throws Exception { server.stop(); server.join(); } + @Override public void setMaxSessionCount(int maxBusy) { // TODO: } + @Override public boolean withSSL() { return false; // TODO: } + @Override public void reconnect(int milsec) { try { Thread.sleep(milsec); @@ -161,16 +167,24 @@ public class HttpServer { } } + @Override public InetSocketAddress generateSocketAddress(String port) throws SocketException { return null; // TODO: } + @Override public int getMaxSessionCount() { return server.getThreadPool().getThreads(); } + @Override public int getJobCount() { return getMaxSessionCount() - server.getThreadPool().getIdleThreads(); // TODO: } + @Override + public String getVersion() { + return "Jetty " + server.getVersion(); + } + } diff --git a/test/net/yacy/http/Jetty9YaCyDefaultServlet.java b/test/net/yacy/http/Jetty9YaCyDefaultServlet.java new file mode 100644 index 000000000..ab607cbbd --- /dev/null +++ b/test/net/yacy/http/Jetty9YaCyDefaultServlet.java @@ -0,0 +1,924 @@ +// YaCyDefaultServlet
+// Copyright 2013 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// First released 2013 at http://yacy.net
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with this program in the file lgpl21.txt
+// If not, see <http://www.gnu.org/licenses/>.
+//
+package net.yacy.http;
+
+import java.awt.Image;
+import java.awt.image.BufferedImage;
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.ref.SoftReference;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.Enumeration;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.servlet.AsyncContext;
+import javax.servlet.RequestDispatcher;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.UnavailableException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import net.yacy.cora.date.GenericFormatter;
+import net.yacy.cora.document.analysis.Classification;
+import net.yacy.cora.protocol.HeaderFramework;
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.kelondro.util.FileUtils;
+import net.yacy.kelondro.util.MemoryControl;
+import net.yacy.peers.Seed;
+import net.yacy.peers.graphics.EncodedImage;
+import net.yacy.peers.operation.yacyBuildProperties;
+import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
+import net.yacy.server.http.TemplateEngine;
+import net.yacy.server.serverClassLoader;
+import net.yacy.server.serverCore;
+import net.yacy.server.serverObjects;
+import net.yacy.server.serverSwitch;
+import net.yacy.server.servletProperties;
+import net.yacy.visualization.RasterPlotter;
+import org.apache.commons.fileupload.FileItem;
+import org.apache.commons.fileupload.disk.DiskFileItemFactory;
+import org.apache.commons.fileupload.servlet.ServletFileUpload;
+
+import org.eclipse.jetty.http.HttpContent;
+import org.eclipse.jetty.http.HttpFields;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpMethod;
+import org.eclipse.jetty.http.MimeTypes;
+import org.eclipse.jetty.io.WriterOutputStream;
+import org.eclipse.jetty.server.HttpOutput;
+import org.eclipse.jetty.server.InclusiveByteRange;
+import org.eclipse.jetty.server.Response;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.MultiPartOutputStream;
+import org.eclipse.jetty.util.QuotedStringTokenizer;
+import org.eclipse.jetty.util.URIUtil;
+import org.eclipse.jetty.util.resource.Resource;
+import org.eclipse.jetty.util.resource.ResourceFactory;
+
+/**
+ * YaCyDefaultServlet base on Jetty DefaultServlet.java
+ * handles static files and the YaCy servlets.
+ */
+
+/**
+ * The default servlet. This servlet, normally mapped to /, provides the
+ * handling for static content, OPTION and TRACE methods for the context. The
+ * following initParameters are supported, these can be set either on the
+ * servlet itself or as ServletContext initParameters :
+ * <PRE>
+ * acceptRanges If true, range requests and responses are
+ * supported
+ *
+ * dirAllowed If true, directory listings are returned if no
+ * welcome file is found. Else 403 Forbidden.
+ *
+ * welcomeFile name of the welcome file (default is "index.html", "welcome.html")
+ *
+ * gzip If set to true, then static content will be served as
+ * gzip content encoded if a matching resource is
+ * found ending with ".gz"
+ *
+ * resourceBase Set to replace the context resource base
+ *
+ * resourceCache If set, this is a context attribute name, which the servlet
+ * will use to look for a shared ResourceCache instance.
+ *
+ * relativeResourceBase
+ * Set with a pathname relative to the base of the
+ * servlet context root. Useful for only serving static content out
+ * of only specific subdirectories.
+ *
+ * pathInfoOnly If true, only the path info will be applied to the resourceBase
+ *
+ *
+ * etags If True, weak etags will be generated and handled.
+ *
+ * </PRE>
+ */
+public class Jetty9YaCyDefaultServlet extends YaCyDefaultServlet implements ResourceFactory {
+
+ private static final long serialVersionUID = 4900000000000001110L;
+
+ private boolean _gzip=true;
+
+ /* ------------------------------------------------------------ */
+ @Override
+ public void init() throws UnavailableException {
+ super.init();
+ _gzip=getInitBoolean("gzip",_gzip);
+ }
+
+ /* ------------------------------------------------------------ */
+ /**
+ * get Resource to serve. Map a path to a resource. The default
+ * implementation calls HttpContext.getResource but derived servlets may
+ * provide their own mapping.
+ *
+ * @param pathInContext The path to find a resource for.
+ * @return The resource to serve.
+ */
+ @Override
+ public Resource getResource(String pathInContext) {
+ Resource r = null;
+ if (_relativeResourceBase != null) {
+ pathInContext = URIUtil.addPaths(_relativeResourceBase, pathInContext);
+ }
+
+ try {
+ if (_resourceBase != null) {
+ r = _resourceBase.addPath(pathInContext);
+ } else {
+ URL u = _servletContext.getResource(pathInContext);
+ r = Resource.newResource(u);
+ }
+
+ if (ConcurrentLog.isFine("YaCyDefaultServlet")) {
+ ConcurrentLog.fine("YaCyDefaultServlet","Resource " + pathInContext + "=" + r);
+ }
+ } catch (IOException e) {
+ // ConcurrentLog.logException(e);
+ }
+
+ return r;
+ }
+
+ /* ------------------------------------------------------------ */
+ @Override
+ protected void doGet(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+ String servletPath = null;
+ String pathInfo = null;
+ Enumeration<String> reqRanges = null;
+ Boolean included = request.getAttribute(RequestDispatcher.INCLUDE_REQUEST_URI) != null;
+ if (included != null && included.booleanValue()) {
+ servletPath = (String) request.getAttribute(RequestDispatcher.INCLUDE_SERVLET_PATH);
+ pathInfo = (String) request.getAttribute(RequestDispatcher.INCLUDE_PATH_INFO);
+ if (servletPath == null) {
+ servletPath = request.getServletPath();
+ pathInfo = request.getPathInfo();
+ }
+ } else {
+ included = Boolean.FALSE;
+ servletPath = _pathInfoOnly ? "/" : request.getServletPath();
+ pathInfo = request.getPathInfo();
+
+ // Is this a Range request?
+ reqRanges = request.getHeaders(HeaderFramework.RANGE);
+ if (!hasDefinedRange(reqRanges)) {
+ reqRanges = null;
+ }
+ }
+
+ if (pathInfo.startsWith("/currentyacypeer/")) pathInfo = pathInfo.substring(16);
+ String pathInContext = URIUtil.addPaths(servletPath, pathInfo);
+ boolean endsWithSlash = (pathInfo == null ? request.getServletPath() : pathInfo).endsWith(URIUtil.SLASH);
+
+ // Find the resource and content
+ Resource resource = null;
+ HttpContent content = null;
+ try {
+ // Look for a class resource
+ boolean hasClass = false;
+ if (reqRanges == null && !endsWithSlash) {
+ final int p = pathInContext.lastIndexOf('.');
+ if (p >= 0) {
+ String pathofClass = pathInContext.substring(0, p) + ".class";
+ resource = getResource(pathofClass);
+ // Does a class resource exist?
+ if (resource != null && resource.exists() && !resource.isDirectory()) {
+ hasClass = true;
+ }
+ }
+ }
+ // is gzip enabled?
+ String pathInContextGz=null;
+ boolean gzip=false;
+ if (!included.booleanValue() && _gzip && reqRanges==null && !endsWithSlash )
+ {
+ // Look for a gzip resource
+ pathInContextGz=pathInContext+".gz";
+ resource=getResource(pathInContextGz);
+ // Does a gzip resource exist?
+ if (resource!=null && resource.exists() && !resource.isDirectory())
+ {
+ // Tell caches that response may vary by accept-encoding
+ response.addHeader(HttpHeader.VARY.asString(),HttpHeader.ACCEPT_ENCODING.asString());
+
+ // Does the client accept gzip?
+ String accept=request.getHeader(HttpHeader.ACCEPT_ENCODING.asString());
+ if (accept!=null && accept.indexOf("gzip")>=0)
+ gzip=true;
+ }
+ }
+
+ // find resource
+ if (!gzip) resource = getResource(pathInContext);
+
+ if (ConcurrentLog.isFine("YaCyDefaultServlet")) {
+ ConcurrentLog.fine("YaCyDefaultServlet","uri=" + request.getRequestURI() + " resource=" + resource + (content != null ? " content" : ""));
+ }
+
+ // Handle resource
+ if (!hasClass && (resource == null || !resource.exists())) {
+ if (included) {
+ throw new FileNotFoundException("!" + pathInContext);
+ }
+ response.sendError(HttpServletResponse.SC_NOT_FOUND);
+ } else if (!resource.isDirectory()) {
+ if (endsWithSlash && pathInContext.length() > 1) {
+ String q = request.getQueryString();
+ pathInContext = pathInContext.substring(0, pathInContext.length() - 1);
+ if (q != null && q.length() != 0) {
+ pathInContext += "?" + q;
+ }
+ response.sendRedirect(response.encodeRedirectURL(URIUtil.addPaths(_servletContext.getContextPath(), pathInContext)));
+ } else {
+ // ensure we have content
+ if (content == null) {
+ content = new HttpContent.ResourceAsHttpContent(resource, _mimeTypes.getMimeByExtension(resource.toString()), response.getBufferSize(), _etags);
+ }
+
+ if (hasClass) { // this is a YaCy servlet, handle the template
+ handleTemplate(pathInfo, request, response);
+ } else {
+ if (included.booleanValue() || passConditionalHeaders(request, response, resource, content)) {
+ //sendData(request, response, included.booleanValue(), resource, content, reqRanges);
+ if (gzip) {
+ response.setHeader(HttpHeader.CONTENT_ENCODING.asString(), "gzip");
+ String mt = _servletContext.getMimeType(pathInContext);
+ if (mt != null) {
+ response.setContentType(mt);
+ }
+ }
+ sendData(request, response, included.booleanValue(), resource, content, reqRanges);
+ }
+ }
+ }
+ } else {
+ if (!endsWithSlash || (pathInContext.length() == 1 && request.getAttribute("org.eclipse.jetty.server.nullPathInfo") != null)) {
+ StringBuffer buf = request.getRequestURL();
+ synchronized (buf) {
+ int param = buf.lastIndexOf(";");
+ if (param < 0) {
+ buf.append('/');
+ } else {
+ buf.insert(param, '/');
+ }
+ String q = request.getQueryString();
+ if (q != null && q.length() != 0) {
+ buf.append('?');
+ buf.append(q);
+ }
+ response.setContentLength(0);
+ response.sendRedirect(response.encodeRedirectURL(buf.toString()));
+ }
+ } else { // look for a welcome file
+ String welcomeFileName = getWelcomeFile (pathInContext);
+ if (welcomeFileName != null) {
+ RequestDispatcher rd = request.getRequestDispatcher(welcomeFileName);
+ rd.forward(request, response);
+ } else { // send directory listing
+ content = new HttpContent.ResourceAsHttpContent(resource, _mimeTypes.getMimeByExtension(resource.toString()), _etags);
+ if (included.booleanValue() || passConditionalHeaders(request, response, resource, content)) {
+ sendDirectory(request, response, resource, pathInContext);
+ }
+ }
+ }
+ }
+ } catch (IllegalArgumentException e) {
+ ConcurrentLog.logException(e);
+ if (!response.isCommitted()) {
+ response.sendError(500, e.getMessage());
+ }
+ } finally {
+ if (content != null) {
+ content.release();
+ } else if (resource != null) {
+ resource.close();
+ }
+ }
+ }
+
+ /* ------------------------------------------------------------ */
+ @Override
+ protected boolean hasDefinedRange(Enumeration<String> reqRanges) {
+ return (reqRanges != null && reqRanges.hasMoreElements());
+ }
+
+ /* ------------------------------------------------------------ */
+ @Override
+ protected void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+ doGet(request, response);
+ }
+
+ /* ------------------------------------------------------------ */
+ /* Check modification date headers.
+ */
+ @Override
+ protected boolean passConditionalHeaders(HttpServletRequest request, HttpServletResponse response, Resource resource, HttpContent content)
+ throws IOException {
+ try {
+ if (!HttpMethod.HEAD.is(request.getMethod())) {
+ if (_etags) {
+ String ifm = request.getHeader(HttpHeader.IF_MATCH.asString());
+ if (ifm != null) {
+ boolean match = false;
+ if (content.getETag() != null) {
+ QuotedStringTokenizer quoted = new QuotedStringTokenizer(ifm, ", ", false, true);
+ while (!match && quoted.hasMoreTokens()) {
+ String tag = quoted.nextToken();
+ if (content.getETag().toString().equals(tag)) {
+ match = true;
+ }
+ }
+ }
+
+ if (!match) {
+ response.setStatus(HttpServletResponse.SC_PRECONDITION_FAILED);
+ return false;
+ }
+ }
+
+ String if_non_match_etag = request.getHeader(HttpHeader.IF_NONE_MATCH.asString());
+ if (if_non_match_etag != null && content.getETag() != null) {
+ // Look for GzipFiltered version of etag
+ if (content.getETag().toString().equals(request.getAttribute("o.e.j.s.GzipFilter.ETag"))) {
+ response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
+ response.setHeader(HeaderFramework.ETAG, if_non_match_etag);
+ return false;
+ }
+
+ // Handle special case of exact match.
+ if (content.getETag().toString().equals(if_non_match_etag)) {
+ response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
+ response.setHeader(HeaderFramework.ETAG, content.getETag());
+ return false;
+ }
+
+ // Handle list of tags
+ QuotedStringTokenizer quoted = new QuotedStringTokenizer(if_non_match_etag, ", ", false, true);
+ while (quoted.hasMoreTokens()) {
+ String tag = quoted.nextToken();
+ if (content.getETag().toString().equals(tag)) {
+ response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
+ response.setHeader(HeaderFramework.ETAG, content.getETag());
+ return false;
+ }
+ }
+
+ // If etag requires content to be served, then do not check if-modified-since
+ return true;
+ }
+ }
+
+ // Handle if modified since
+ String ifms = request.getHeader(HttpHeader.IF_MODIFIED_SINCE.asString());
+ if (ifms != null) {
+ //Get jetty's Response impl
+ String mdlm = content.getLastModified();
+ if (mdlm != null && ifms.equals(mdlm)) {
+ response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
+ if (_etags) {
+ response.setHeader(HeaderFramework.ETAG, content.getETag());
+ }
+ response.flushBuffer();
+ return false;
+ }
+
+ long ifmsl = request.getDateHeader(HttpHeader.IF_MODIFIED_SINCE.asString());
+ if (ifmsl != -1 && resource.lastModified() / 1000 <= ifmsl / 1000) {
+ response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
+ if (_etags) {
+ response.setHeader(HeaderFramework.ETAG, content.getETag());
+ }
+ response.flushBuffer();
+ return false;
+ }
+ }
+
+ // Parse the if[un]modified dates and compare to resource
+ long date = request.getDateHeader(HttpHeader.IF_UNMODIFIED_SINCE.asString());
+ if (date != -1 && resource.lastModified() / 1000 > date / 1000) {
+ response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED);
+ return false;
+ }
+
+ }
+ } catch (IllegalArgumentException iae) {
+ if (!response.isCommitted()) {
+ response.sendError(400, iae.getMessage());
+ }
+ throw iae;
+ }
+ return true;
+ }
+
+
+ /* ------------------------------------------------------------------- */
+ @Override
+ protected void sendDirectory(HttpServletRequest request,
+ HttpServletResponse response,
+ Resource resource,
+ String pathInContext)
+ throws IOException {
+ if (!_dirAllowed) {
+ response.sendError(HttpServletResponse.SC_FORBIDDEN);
+ return;
+ }
+
+ byte[] data = null;
+ String base = URIUtil.addPaths(request.getRequestURI(), URIUtil.SLASH);
+
+ String dir = resource.getListHTML(base, pathInContext.length() > 1);
+ if (dir == null) {
+ response.sendError(HttpServletResponse.SC_FORBIDDEN,
+ "No directory");
+ return;
+ }
+
+ data = dir.getBytes("UTF-8");
+ response.setContentType("text/html; charset=UTF-8");
+ response.setContentLength(data.length);
+ response.getOutputStream().write(data);
+ }
+
+ /* ------------------------------------------------------------ */
+ @Override
+ protected void sendData(HttpServletRequest request,
+ HttpServletResponse response,
+ boolean include,
+ Resource resource,
+ HttpContent content,
+ Enumeration<String> reqRanges)
+ throws IOException {
+ final long content_length = (content == null) ? resource.length() : content.getContentLength();
+
+ // Get the output stream (or writer)
+ OutputStream out = null;
+ boolean written;
+ try {
+ out = response.getOutputStream();
+
+ // has a filter already written to the response?
+ written = out instanceof HttpOutput
+ ? ((HttpOutput) out).isWritten()
+ : true;
+ } catch (IllegalStateException e) {
+ out = new WriterOutputStream(response.getWriter());
+ written = true; // there may be data in writer buffer, so assume written
+ }
+
+ if (reqRanges == null || !reqRanges.hasMoreElements() || content_length < 0) {
+ // if there were no ranges, send entire entity
+ if (include) {
+ resource.writeTo(out, 0, content_length);
+ } // else if we can't do a bypass write because of wrapping
+ else if (content == null || written || !(out instanceof HttpOutput)) {
+ // write normally
+ writeHeaders(response, content, written ? -1 : content_length);
+ ByteBuffer buffer = (content == null) ? null : content.getIndirectBuffer();
+ if (buffer != null) {
+ BufferUtil.writeTo(buffer, out);
+ } else {
+ resource.writeTo(out, 0, content_length);
+ }
+ } // else do a bypass write
+ else {
+ // write the headers
+ if (response instanceof Response) {
+ Response r = (Response) response;
+ writeOptionHeaders(r.getHttpFields());
+ r.setHeaders(content);
+ } else {
+ writeHeaders(response, content, content_length);
+ }
+
+ // write the content asynchronously if supported
+ if (request.isAsyncSupported()) {
+ final AsyncContext context = request.startAsync();
+
+ ((HttpOutput) out).sendContent(content, new Callback() {
+ @Override
+ public void succeeded() {
+ context.complete();
+ }
+
+ @Override
+ public void failed(Throwable x) {
+ ConcurrentLog.logException(x);
+ context.complete();
+ }
+ });
+ } // otherwise write content blocking
+ else {
+ ((HttpOutput) out).sendContent(content);
+ }
+ }
+ } else {
+ // Parse the satisfiable ranges
+ List<InclusiveByteRange> ranges = InclusiveByteRange.satisfiableRanges(reqRanges, content_length);
+
+ // if there are no satisfiable ranges, send 416 response
+ if (ranges == null || ranges.size() == 0) {
+ writeHeaders(response, content, content_length);
+ response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
+ response.setHeader(HeaderFramework.CONTENT_RANGE,
+ InclusiveByteRange.to416HeaderRangeString(content_length));
+ resource.writeTo(out, 0, content_length);
+ return;
+ }
+
+ // if there is only a single valid range (must be satisfiable
+ // since were here now), send that range with a 216 response
+ if (ranges.size() == 1) {
+ InclusiveByteRange singleSatisfiableRange = ranges.get(0);
+ long singleLength = singleSatisfiableRange.getSize(content_length);
+ writeHeaders(response, content, singleLength);
+ response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
+ response.setHeader(HeaderFramework.CONTENT_RANGE,
+ singleSatisfiableRange.toHeaderRangeString(content_length));
+ resource.writeTo(out, singleSatisfiableRange.getFirst(content_length), singleLength);
+ return;
+ }
+
+ // multiple non-overlapping valid ranges cause a multipart
+ // 216 response which does not require an overall
+ // content-length header
+ //
+ writeHeaders(response, content, -1);
+ String mimetype = (content == null || content.getContentType() == null ? null : content.getContentType().toString());
+ if (mimetype == null) {
+ ConcurrentLog.warn("YaCyDefaultServlet", "Unknown mimetype for " + request.getRequestURI());
+ }
+ MultiPartOutputStream multi = new MultiPartOutputStream(out);
+ response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
+
+ // If the request has a "Request-Range" header then we need to
+ // send an old style multipart/x-byteranges Content-Type. This
+ // keeps Netscape and acrobat happy. This is what Apache does.
+ String ctp;
+ if (request.getHeader(HttpHeader.REQUEST_RANGE.asString()) != null) {
+ ctp = "multipart/x-byteranges; boundary=";
+ } else {
+ ctp = "multipart/byteranges; boundary=";
+ }
+ response.setContentType(ctp + multi.getBoundary());
+
+ InputStream in = resource.getInputStream();
+ long pos = 0;
+
+ // calculate the content-length
+ int length = 0;
+ String[] header = new String[ranges.size()];
+ for (int i = 0; i < ranges.size(); i++) {
+ InclusiveByteRange ibr = ranges.get(i);
+ header[i] = ibr.toHeaderRangeString(content_length);
+ length +=
+ ((i > 0) ? 2 : 0)
+ + 2 + multi.getBoundary().length() + 2
+ + (mimetype == null ? 0 : HeaderFramework.CONTENT_TYPE.length() + 2 + mimetype.length()) + 2
+ + HeaderFramework.CONTENT_RANGE.length() + 2 + header[i].length() + 2
+ + 2
+ + (ibr.getLast(content_length) - ibr.getFirst(content_length)) + 1;
+ }
+ length += 2 + 2 + multi.getBoundary().length() + 2 + 2;
+ response.setContentLength(length);
+
+ for (int i = 0; i < ranges.size(); i++) {
+ InclusiveByteRange ibr = ranges.get(i);
+ multi.startPart(mimetype, new String[]{HeaderFramework.CONTENT_RANGE + ": " + header[i]});
+
+ long start = ibr.getFirst(content_length);
+ long size = ibr.getSize(content_length);
+ if (in != null) {
+ // Handle non cached resource
+ if (start < pos) {
+ in.close();
+ in = resource.getInputStream();
+ pos = 0;
+ }
+ if (pos < start) {
+ in.skip(start - pos);
+ pos = start;
+ }
+
+ FileUtils.copy(in, multi, size);
+ pos += size;
+ } else // Handle cached resource
+ {
+ (resource).writeTo(multi, start, size);
+ }
+
+ }
+ if (in != null) {
+ in.close();
+ }
+ multi.close();
+ }
+ }
+
+ /* ------------------------------------------------------------ */
+ @Override
+ protected void writeHeaders(HttpServletResponse response, HttpContent content, long count) {
+ if (content.getContentType() != null && response.getContentType() == null) {
+ response.setContentType(content.getContentType().toString());
+ }
+
+ if (response instanceof Response) {
+ Response r = (Response) response;
+ HttpFields fields = r.getHttpFields();
+
+ if (content.getLastModified() != null) {
+ fields.put(HeaderFramework.LAST_MODIFIED, content.getLastModified());
+ } else if (content.getResource() != null) {
+ long lml = content.getResource().lastModified();
+ if (lml != -1) {
+ fields.putDateField(HeaderFramework.LAST_MODIFIED, lml);
+ }
+ }
+
+ if (count != -1) {
+ r.setLongContentLength(count);
+ }
+
+ writeOptionHeaders(fields);
+
+ if (_etags) {
+ fields.put(HeaderFramework.ETAG, content.getETag());
+ }
+ } else {
+ long lml = content.getResource().lastModified();
+ if (lml >= 0) {
+ response.setDateHeader(HeaderFramework.LAST_MODIFIED, lml);
+ }
+
+ if (count != -1) {
+ if (count < Integer.MAX_VALUE) {
+ response.setContentLength((int) count);
+ } else {
+ response.setHeader(HeaderFramework.CONTENT_LENGTH, Long.toString(count));
+ }
+ }
+
+ writeOptionHeaders(response);
+
+ if (_etags) {
+ response.setHeader(HeaderFramework.ETAG, content.getETag().toString());
+ }
+ }
+ }
+
+ @Override
+ public void handleTemplate(String target, HttpServletRequest request,
+ HttpServletResponse response) throws IOException, ServletException {
+ Switchboard sb = Switchboard.getSwitchboard();
+
+ String localeSelection = Switchboard.getSwitchboard().getConfig("locale.language", "default");
+ File targetFile = getLocalizedFile(target, localeSelection);
+ File targetClass = rewriteClassFile(_resourceBase.addPath(target).getFile());
+ String targetExt = target.substring(target.lastIndexOf('.') + 1, target.length());
+
+ if ((targetClass != null)) {
+ serverObjects args = new serverObjects();
+ @SuppressWarnings("unchecked")
+ Enumeration<String> argNames = request.getParameterNames();
+ while (argNames.hasMoreElements()) {
+ String argName = argNames.nextElement();
+ args.put(argName, request.getParameter(argName));
+ }
+ //TODO: for SSI request, local parameters are added as attributes, put them back as parameter for the legacy request
+ // likely this should be implemented via httpservletrequestwrapper to supply complete parameters
+ @SuppressWarnings("unchecked")
+ Enumeration<String> attNames = request.getAttributeNames();
+ while (attNames.hasMoreElements()) {
+ String argName = attNames.nextElement();
+ args.put(argName, request.getAttribute(argName).toString());
+ }
+
+ // add multipart-form fields to parameter
+ if (request.getContentType() != null && request.getContentType().startsWith("multipart/form-data")) {
+ parseMultipart(request, args);
+ }
+ // eof modification to read attribute
+ RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt);
+
+ Object tmp;
+ try {
+ tmp = invokeServlet(targetClass, legacyRequestHeader, args);
+ } catch (InvocationTargetException e) {
+ ConcurrentLog.logException(e);
+ throw new ServletException();
+ } catch (IllegalArgumentException e) {
+ ConcurrentLog.logException(e);
+ throw new ServletException();
+ } catch (IllegalAccessException e) {
+ ConcurrentLog.logException(e);
+ throw new ServletException();
+ }
+
+ if (tmp instanceof RasterPlotter || tmp instanceof EncodedImage || tmp instanceof Image) {
+
+ net.yacy.cora.util.ByteBuffer result = null;
+
+ if (tmp instanceof RasterPlotter) {
+ final RasterPlotter yp = (RasterPlotter) tmp;
+ // send an image to client
+ result = RasterPlotter.exportImage(yp.getImage(), "png");
+ }
+ if (tmp instanceof EncodedImage) {
+ final EncodedImage yp = (EncodedImage) tmp;
+ result = yp.getImage();
+ }
+
+ if (tmp instanceof Image) {
+ final Image i = (Image) tmp;
+
+ // generate an byte array from the generated image
+ int width = i.getWidth(null);
+ if (width < 0) {
+ width = 96; // bad hack
+ }
+ int height = i.getHeight(null);
+ if (height < 0) {
+ height = 96; // bad hack
+ }
+ final BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
+ bi.createGraphics().drawImage(i, 0, 0, width, height, null);
+ result = RasterPlotter.exportImage(bi, targetExt);
+ }
+
+ final String mimeType = Classification.ext2mime(targetExt, "text/html");
+ response.setContentType(mimeType);
+ response.setContentLength(result.length());
+ response.setStatus(HttpServletResponse.SC_OK);
+
+ result.writeTo(response.getOutputStream());
+
+ return;
+ }
+
+ servletProperties templatePatterns = null;
+ if (tmp == null) {
+ // if no args given, then tp will be an empty Hashtable object (not null)
+ templatePatterns = new servletProperties();
+ } else if (tmp instanceof servletProperties) {
+ templatePatterns = (servletProperties) tmp;
+ } else {
+ templatePatterns = new servletProperties((serverObjects) tmp);
+ }
+ // add the application version, the uptime and the client name to every rewrite table
+ templatePatterns.put(servletProperties.PEER_STAT_VERSION, yacyBuildProperties.getVersion());
+ templatePatterns.put(servletProperties.PEER_STAT_UPTIME, ((System.currentTimeMillis() - serverCore.startupTime) / 1000) / 60); // uptime in minutes
+ templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTNAME, sb.peers.mySeed().getName());
+ templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTID, sb.peers.myID());
+ templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format());
+ Seed myPeer = sb.peers.mySeed();
+ templatePatterns.put("newpeer", myPeer.getAge() >= 1 ? 0 : 1);
+ templatePatterns.putHTML("newpeer_peerhash", myPeer.hash);
+ templatePatterns.put("p2p", sb.getConfigBool(SwitchboardConstants.DHT_ENABLED, true) || !sb.isRobinsonMode() ? 1 : 0);
+
+ if (targetFile.exists() && targetFile.isFile() && targetFile.canRead()) {
+ String mimeType = Classification.ext2mime(targetExt, "text/html");
+
+ InputStream fis = null;
+ long fileSize = targetFile.length();
+
+ if (fileSize <= Math.min(4 * 1024 * 1204, MemoryControl.available() / 100)) {
+ // read file completely into ram, avoid that too many files are open at the same time
+ fis = new ByteArrayInputStream(FileUtils.read(targetFile));
+ } else {
+ fis = new BufferedInputStream(new FileInputStream(targetFile));
+ }
+
+ // set response header
+ response.setContentType(mimeType);
+ response.setStatus(HttpServletResponse.SC_OK);
+ ByteArrayOutputStream bas = new ByteArrayOutputStream(4096);
+ // apply templates
+ TemplateEngine.writeTemplate(fis, bas, templatePatterns, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
+ fis.close();
+ // handle SSI
+ doContentMod (bas.toByteArray(),request,response);
+ }
+ }
+ }
+
+ protected void doContentMod(final byte[] in, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
+ net.yacy.cora.util.ByteBuffer buffer = new net.yacy.cora.util.ByteBuffer(in);
+ OutputStream out = response.getOutputStream();
+
+
+ // remove virtual host "currentyacypeer"
+ int off = 0; // starting offset
+ int x = buffer.indexOf("/currentyacypeer/".getBytes(), off);
+ while (x >= 0) {
+ for (int i = 0; i < 16; i++) {
+ in[x + i] = 32;
+ }
+ off = x + 16;
+ x = buffer.indexOf("/currentyacypeer/".getBytes(), off);
+ }
+
+ // check and handle SSI (ServerSideIncludes)
+ off = 0;
+ int p = buffer.indexOf("<!--#".getBytes(), off);
+ int q;
+ while (p >= 0) {
+ q = buffer.indexOf("-->".getBytes(), p + 10);
+
+ out.write(in, off, p - off);
+ out.flush();
+ parseSSI(buffer, p, request, response);
+ off = q + 3;
+ p = buffer.indexOf("<!--#".getBytes(), off);
+ }
+ out.write(in, off, in.length - off);
+ out.flush();
+ }
+
+ // parse SSI line and include resource
+ @Override
+ protected void parseSSI(final net.yacy.cora.util.ByteBuffer in, final int off, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+ if (in.startsWith("<!--#include virtual=\"".getBytes(), off)) {
+ final int q = in.indexOf("\"".getBytes(), off + 22);
+ if (q > 0) {
+ final String path = in.toString(off + 22, q - off - 22);
+ try {
+ RequestDispatcher dispatcher = request.getRequestDispatcher("/" + path);
+ dispatcher.include(request, response);
+ response.flushBuffer();
+ } catch (Exception e) {
+ ConcurrentLog.logException(e);
+ throw new ServletException();
+ }
+ }
+ }
+ }
+ /**
+ * TODO: add same functionality & checks as in HTTPDemon.parseMultipart
+ *
+ * parse multi-part form data for formfields (only), see also original
+ * implementation in HTTPDemon.parseMultipart
+ *
+ * @param request
+ * @param args found fields/values are added to the map
+ */
+ public void parseMultipart(HttpServletRequest request, serverObjects args) {
+ DiskFileItemFactory factory = new DiskFileItemFactory();
+ // maximum size that will be stored in memory
+ factory.setSizeThreshold(4096 * 16);
+ // Location to save data that is larger than maxMemSize.
+ // factory.setRepository(new File("."));
+ // Create a new file upload handler
+ ServletFileUpload upload = new ServletFileUpload(factory);
+ upload.setSizeMax(4096 * 16);
+ try {
+ // Parse the request to get form field items
+ @SuppressWarnings("unchecked")
+ List<FileItem> fileItems = upload.parseRequest(request);
+ // Process the uploaded file items
+ Iterator<FileItem> i = fileItems.iterator();
+ while (i.hasNext()) {
+ FileItem fi = i.next();
+ if (fi.isFormField()) {
+ args.put(fi.getFieldName(), fi.getString());
+ }
+ }
+ } catch (Exception ex) {
+ ConcurrentLog.logException(ex);
+ }
+ }
+}
\ No newline at end of file diff --git a/source/net/yacy/http/YaCySecurityHandler.java b/test/net/yacy/http/Jetty9YaCySecurityHandler.java index 0649ec830..f40250c7f 100644 --- a/source/net/yacy/http/YaCySecurityHandler.java +++ b/test/net/yacy/http/Jetty9YaCySecurityHandler.java @@ -46,7 +46,7 @@ import org.eclipse.jetty.server.UserIdentity; * demands authentication for pages with _p. inside * and updates AccessTracker */ -public class YaCySecurityHandler extends SecurityHandler { +public class Jetty9YaCySecurityHandler extends SecurityHandler { @Override protected boolean checkUserDataPermissions(String pathInContext, Request request, Response response, RoleInfo roleInfo) throws IOException diff --git a/test/net/yacy/http/Jetty9_Implementation_Test.java b/test/net/yacy/http/Jetty9_Implementation_Test.java new file mode 100644 index 000000000..ea58fae2d --- /dev/null +++ b/test/net/yacy/http/Jetty9_Implementation_Test.java @@ -0,0 +1,34 @@ +package net.yacy.http;
+
+
+import static org.junit.Assert.*;
+import org.junit.Test;
+
+/**
+ *
+ * @author bubu
+ */
+public class Jetty9_Implementation_Test {
+
+ public Jetty9_Implementation_Test() {
+ }
+
+ /**
+ * Test of main method, of class yacy.
+ */
+ @Test
+ public void testYaCyMain() {
+ System.out.println("* * * * * * * * * * * * * * * * * * * * *");
+ System.err.println("* This is a Implementation test *");
+ System.out.println("* for Jetty 9 (not a normal test case) *");
+ System.out.println("* Shutdown YaCy to continue *");
+ System.out.println("* * * * * * * * * * * * * * * * * * * * *");
+
+ String[] args = new String[]{};
+ YacyMain.main(args);
+
+ assertTrue(true);
+
+ }
+
+}
\ No newline at end of file diff --git a/test/net/yacy/http/YacyMain.java b/test/net/yacy/http/YacyMain.java new file mode 100644 index 000000000..56659b28d --- /dev/null +++ b/test/net/yacy/http/YacyMain.java @@ -0,0 +1,702 @@ +// yacy.java
+// -----------------------
+// (C) by Michael Peter Christen; mc@yacy.net
+// first published on http://www.yacy.net
+// Frankfurt, Germany, 2004, 2005
+//
+// $LastChangedDate$
+// $LastChangedRevision$
+// $LastChangedBy$
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package net.yacy.http;
+
+import net.yacy.*;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.util.Properties;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ExecutionException;
+
+import net.yacy.cora.date.GenericFormatter;
+import net.yacy.cora.lod.JenaTripleStore;
+import net.yacy.cora.protocol.ClientIdentification;
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.protocol.TimeoutRequest;
+import net.yacy.cora.protocol.http.HTTPClient;
+import net.yacy.cora.sorting.Array;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.data.Translator;
+import net.yacy.gui.YaCyApp;
+import net.yacy.gui.framework.Browser;
+import net.yacy.http.YaCyHttpServer;
+import net.yacy.kelondro.util.FileUtils;
+import net.yacy.kelondro.util.Formatter;
+import net.yacy.kelondro.util.MemoryControl;
+import net.yacy.kelondro.util.OS;
+import net.yacy.peers.operation.yacyBuildProperties;
+import net.yacy.peers.operation.yacyRelease;
+import net.yacy.peers.operation.yacyVersion;
+import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
+
+import com.google.common.io.Files;
+import net.yacy.http.Jetty9HttpServerImpl;
+import net.yacy.migration;
+
+import net.yacy.server.serverCore;
+
+
+
+/**
+* This is the main class of YaCy. Several threads are started from here:
+* <ul>
+* <li>one single instance of the plasmaSwitchboard is generated, which itself
+* starts a thread with a plasmaHTMLCache object. This object simply counts
+* files sizes in the cache and terminates them. It also generates a
+* plasmaCrawlerLoader object, which may itself start some more httpc-calling
+* threads to load web pages. They terminate automatically when a page has
+* loaded.
+* <li>one serverCore - thread is started, which implements a multi-threaded
+* server. The process may start itself many more processes that handle
+* connections.lo
+* <li>finally, all idle-dependent processes are written in a queue in
+* plasmaSwitchboard which are worked off inside an idle-sensitive loop of the
+* main process. (here)
+* </ul>
+*
+* On termination, the following must be done:
+* <ul>
+* <li>stop feeding of the crawling process because it otherwise fills the
+* indexing queue.
+* <li>say goodbye to connected peers and disable new connections. Don't wait for
+* success.
+* <li>first terminate the serverCore thread. This prevents that new cache
+* objects are queued.
+* <li>wait that the plasmaHTMLCache terminates (it should be normal that this
+* process already has terminated).
+* <li>then wait for termination of all loader process of the
+* plasmaCrawlerLoader.
+* <li>work off the indexing and cache storage queue. These values are inside a
+* RAM cache and would be lost otherwise.
+* <li>write all settings.
+* <li>terminate.
+* </ul>
+*/
+
+public final class YacyMain {
+
+ // static objects
+ public static final String vString = yacyBuildProperties.getVersion();
+ public static float version = 0.1f;
+
+ public static final String vDATE = yacyBuildProperties.getBuildDate();
+ public static final String copyright = "[ YaCy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
+ public static final String hline = "-------------------------------------------------------------------------------";
+ public static final Semaphore shutdownSemaphore = new Semaphore(0);
+
+ /**
+ * a reference to the {@link Switchboard} created by the
+ * {@link yacy#startup(String, long, long)} method.
+ */
+ private static Switchboard sb = null;
+ public static String homedir;
+ public static File dataHome_g;
+
+ /**
+ * Starts up the whole application. Sets up all datastructures and starts
+ * the main threads.
+ *
+ * @param homePath Root-path where all information is to be found.
+ * @param startupFree free memory at startup time, to be used later for statistics
+ */
+ private static void startup(final File dataHome, final File appHome, final long startupMemFree, final long startupMemTotal, final boolean gui) {
+ try {
+ // start up
+ System.out.println(copyright);
+ System.out.println(hline);
+
+ // check java version
+ try {
+ "a".isEmpty(); // needs at least Java 1.6
+ } catch (final NoSuchMethodError e) {
+ System.err.println("STARTUP: Java Version too low. You need at least Java 1.6 to run YaCy");
+ System.exit(-1);
+ }
+
+ // ensure that there is a DATA directory, if not, create one and if that fails warn and die
+ mkdirsIfNeseccary(dataHome);
+ dataHome_g = dataHome;
+ mkdirsIfNeseccary(appHome);
+ File f = new File(dataHome, "DATA/");
+ mkdirsIfNeseccary(f);
+ if (!(f.exists())) {
+ System.err.println("Error creating DATA-directory in " + dataHome.toString() + " . Please check your write-permission for this folder. YaCy will now terminate.");
+ System.exit(-1);
+ }
+
+ homedir = appHome.toString();
+
+ // setting up logging
+ f = new File(dataHome, "DATA/LOG/");
+ mkdirsIfNeseccary(f);
+ f = new File(dataHome, "DATA/LOG/yacy.logging");
+ final File f0 = new File(appHome, "defaults/yacy.logging");
+ if (!f.exists() || f0.lastModified() > f.lastModified()) try {
+ Files.copy(f0, f);
+ } catch (final IOException e){
+ System.out.println("could not copy yacy.logging");
+ }
+ try{
+ ConcurrentLog.configureLogging(dataHome, appHome, new File(dataHome, "DATA/LOG/yacy.logging"));
+ } catch (final IOException e) {
+ System.out.println("could not find logging properties in homePath=" + dataHome);
+ ConcurrentLog.logException(e);
+ }
+ ConcurrentLog.config("STARTUP", "YaCy version: " + yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision());
+ ConcurrentLog.config("STARTUP", "Java version: " + System.getProperty("java.version", "no-java-version"));
+ ConcurrentLog.config("STARTUP", "Operation system: " + System.getProperty("os.name","unknown"));
+ ConcurrentLog.config("STARTUP", "Application root-path: " + appHome);
+ ConcurrentLog.config("STARTUP", "Data root-path: " + dataHome);
+ ConcurrentLog.config("STARTUP", "Time zone: UTC" + GenericFormatter.UTCDiffString() + "; UTC+0000 is " + System.currentTimeMillis());
+ ConcurrentLog.config("STARTUP", "Maximum file system path length: " + OS.maxPathLength);
+
+ f = new File(dataHome, "DATA/yacy.running");
+ final String conf = "DATA/SETTINGS/yacy.conf".replace("/", File.separator);
+ if (f.exists()) { // another instance running? VM crash? User will have to care about this
+ ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " exists, this usually means that a YaCy instance is still running. If you want to restart YaCy, try first ./stopYACY.sh, then ./startYACY.sh. If ./stopYACY.sh fails, try ./killYACY.sh");
+
+ // If YaCy is actually running, then we check if the server port is open.
+ // If yes, then we consider that a restart is a user mistake and then we just respond
+ // as the user expects and tell the browser to open the start page.
+ // That will especially happen if Windows Users double-Click the YaCy Icon on the desktop to simply
+ // open the web interface. (They don't think of 'servers' they just want to get to the search page).
+ // We need to parse the configuration file for that to get the host port
+ File dataFile = new File(dataHome, conf);
+ if (dataFile.exists()) {
+ Properties p = new Properties();
+ p.load(new FileInputStream(dataFile));
+ int port = Integer.parseInt(p.getProperty("port", "8090"));
+ try {
+ if (TimeoutRequest.ping("127.0.0.1", port, 1000)) {
+ Browser.openBrowser("http://localhost:" + port + "/" + p.getProperty(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html"));
+ // Thats it; YaCy was running, the user is happy, we can stop now.
+ ConcurrentLog.severe("STARTUP", "WARNING: YaCy instance was still running; just opening the browser and exit.");
+ System.exit(0);
+ }
+ } catch (final ExecutionException ex) {
+ ConcurrentLog.info("STARTUP", "INFO: delete old yacy.running file; likely previous YaCy session was not orderly shutdown!");
+ }
+ }
+
+ // YaCy is not running; thus delete the file an go on as nothing was wrong.
+ delete(f);
+ }
+ if (!f.createNewFile()) ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be created!");
+ try { new FileOutputStream(f).write(Integer.toString(OS.getPID()).getBytes()); } catch (final Exception e) { } // write PID
+ f.deleteOnExit();
+ FileChannel channel = null;
+ FileLock lock = null;
+ try {
+ channel = new RandomAccessFile(f,"rw").getChannel();
+ lock = channel.tryLock(); // lock yacy.running
+ } catch (final Exception e) { }
+
+ try {
+ sb = new Switchboard(dataHome, appHome, "defaults/yacy.init".replace("/", File.separator), conf);
+ } catch (final RuntimeException e) {
+ ConcurrentLog.severe("STARTUP", "YaCy cannot start: " + e.getMessage(), e);
+ System.exit(-1);
+ }
+ //sbSync.V(); // signal that the sb reference was set
+
+ // switch the memory strategy
+ MemoryControl.setStandardStrategy(sb.getConfigBool("memory.standardStrategy", true));
+
+ // save information about available memory at startup time
+ sb.setConfig("memoryFreeAfterStartup", startupMemFree);
+ sb.setConfig("memoryTotalAfterStartup", startupMemTotal);
+
+ // start gui if wanted
+ if (gui) YaCyApp.start("localhost", (int) sb.getConfigLong("port", 8090));
+
+ // hardcoded, forced, temporary value-migration
+ sb.setConfig("htTemplatePath", "htroot/env/templates");
+
+ int oldRev;
+ try {
+ oldRev = Integer.parseInt(sb.getConfig("svnRevision", "0"));
+ } catch (final NumberFormatException e) {
+ oldRev = 0;
+ }
+ final int newRev = Integer.parseInt(yacyBuildProperties.getSVNRevision());
+ sb.setConfig("svnRevision", yacyBuildProperties.getSVNRevision());
+ sb.setConfig("applicationRoot", appHome.toString());
+ sb.setConfig("dataRoot", dataHome.toString());
+ yacyVersion.latestRelease = version;
+
+ // read environment
+ final int timeout = Math.max(5000, Integer.parseInt(sb.getConfig("httpdTimeout", "5000")));
+
+ // create some directories
+ final File htRootPath = new File(appHome, sb.getConfig("htRootPath", "htroot"));
+ final File htDocsPath = sb.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT);
+ mkdirIfNeseccary(htDocsPath);
+ //final File htTemplatePath = new File(homePath, sb.getConfig("htTemplatePath","htdocs"));
+
+ // create default notifier picture
+ //TODO: Use templates instead of copying images ...
+ if (!((new File(htDocsPath, "notifier.gif")).exists())) try {
+ Files.copy(new File(htRootPath, "env/grafics/empty.gif"),
+ new File(htDocsPath, "notifier.gif"));
+ } catch (final IOException e) {}
+
+ final File htdocsReadme = new File(htDocsPath, "readme.txt");
+ if (!(htdocsReadme.exists())) try {FileUtils.copy((
+ "This is your root directory for individual Web Content\r\n" +
+ "\r\n" +
+ "Please place your html files into the www subdirectory.\r\n" +
+ "The URL of that path is either\r\n" +
+ "http://www.<your-peer-name>.yacy or\r\n" +
+ "http://<your-ip>:<your-port>/www\r\n" +
+ "\r\n" +
+ "Other subdirectories may be created; they map to corresponding sub-domains.\r\n" +
+ "This directory shares it's content with the applications htroot path, so you\r\n" +
+ "may access your yacy search page with\r\n" +
+ "http://<your-peer-name>.yacy/\r\n" +
+ "\r\n").getBytes(), htdocsReadme);} catch (final IOException e) {
+ System.out.println("Error creating htdocs readme: " + e.getMessage());
+ }
+
+ final File wwwDefaultPath = new File(htDocsPath, "www");
+ mkdirIfNeseccary(wwwDefaultPath);
+
+
+ final File shareDefaultPath = new File(htDocsPath, "share");
+ mkdirIfNeseccary(shareDefaultPath);
+
+ migration.migrate(sb, oldRev, newRev);
+
+ // delete old release files
+ final int deleteOldDownloadsAfterDays = (int) sb.getConfigLong("update.deleteOld", 30);
+ yacyRelease.deleteOldDownloads(sb.releasePath, deleteOldDownloadsAfterDays );
+
+ // set user-agent
+ HTTPClient.setDefaultUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent);
+
+ // initial fill of the triplestore
+ File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath()));
+ mkdirIfNeseccary(triplestore);
+ for (String s: triplestore.list()) {
+ if ((s.endsWith(".rdf") || s.endsWith(".nt")) && !s.equals("local.rdf") && !s.endsWith("_triplestore.rdf") && !s.startsWith("private_store_")) {
+ try {
+ JenaTripleStore.load(new File(triplestore, s).getAbsolutePath());
+ } catch (final IOException e) {
+ ConcurrentLog.logException(e);
+ }
+ }
+ }
+ if (sb.getConfigBool("triplestore.persistent", false)) {
+ File local = new File(triplestore, "local.rdf");
+ if (local.exists()) {
+ try {
+ JenaTripleStore.load(local.getAbsolutePath());
+ } catch (final IOException e) {
+ ConcurrentLog.logException(e);
+ }
+ }
+ }
+
+ // start main threads
+ final int port = sb.getConfigInt("port", 8090);
+ try {
+
+ // start jetty http server
+ YaCyHttpServer httpServer = new Jetty9HttpServerImpl(port);
+ httpServer.startupServer();
+ sb.setHttpServer(httpServer);
+ ConcurrentLog.info("STARTUP",httpServer.getVersion());
+ //final HTTPDemon protocolHandler = new HTTPDemon(sb);
+ //final serverCore server = new serverCore(
+ // timeout /*control socket timeout in milliseconds*/,
+ // true /* block attacks (wrong protocol) */,
+ // protocolHandler /*command class*/,
+ // sb,
+ // 30000 /*command max length incl. GET args*/);
+ //server.setName("httpd:"+port);
+ //server.setPriority(Thread.MAX_PRIORITY);
+ //server.setObeyIntermission(false);
+
+ // start the server
+ //sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0);
+ //server.start();
+
+ // open the browser window
+ final boolean browserPopUpTrigger = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_TRIGGER, "true").equals("true");
+ if (browserPopUpTrigger) try {
+ final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
+ //boolean properPW = (sb.getConfig("adminAccount", "").isEmpty()) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0);
+ //if (!properPW) browserPopUpPage = "ConfigBasic.html";
+ Browser.openBrowser((false?"https":"http") + "://localhost:" + port + "/" + browserPopUpPage);
+ // Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage);
+ } catch (final Throwable e) {
+ // cannot open browser. This may be normal in headless environments
+ //Log.logException(e);
+ }
+
+ // enable browser popup, http server is ready now
+ sb.tray.setReady();
+
+ //regenerate Locales from Translationlist, if needed
+ final File locale_source = sb.getAppPath("locale.source", "locales");
+ final String lang = sb.getConfig("locale.language", "");
+ if (!lang.equals("") && !lang.equals("default")) { //locale is used
+ String currentRev = "";
+ try{
+ final BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang+"/version" ))));
+ currentRev = br.readLine();
+ br.close();
+ }catch(final IOException e){
+ //Error
+ }
+
+ if (!currentRev.equals(sb.getConfig("svnRevision", ""))) try { //is this another version?!
+ final File sourceDir = new File(sb.getConfig("htRootPath", "htroot"));
+ final File destDir = new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang);
+ if (Translator.translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it
+ //write the new Versionnumber
+ final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version"))));
+ bw.write(sb.getConfig("svnRevision", "Error getting Version"));
+ bw.close();
+ }
+ } catch (final IOException e) {}
+ }
+ // initialize number formatter with this locale
+ Formatter.setLocale(lang);
+
+ // registering shutdown hook
+ ConcurrentLog.config("STARTUP", "Registering Shutdown Hook");
+ final Runtime run = Runtime.getRuntime();
+ run.addShutdownHook(new shutdownHookThread(Thread.currentThread(), sb));
+
+ // save information about available memory after all initializations
+ //try {
+ sb.setConfig("memoryFreeAfterInitBGC", MemoryControl.free());
+ sb.setConfig("memoryTotalAfterInitBGC", MemoryControl.total());
+ System.gc();
+ sb.setConfig("memoryFreeAfterInitAGC", MemoryControl.free());
+ sb.setConfig("memoryTotalAfterInitAGC", MemoryControl.total());
+ //} catch (final ConcurrentModificationException e) {}
+
+ // wait for server shutdown
+ try {
+ sb.waitForShutdown();
+ } catch (final Exception e) {
+ ConcurrentLog.severe("MAIN CONTROL LOOP", "PANIC: " + e.getMessage(),e);
+ }
+ // shut down
+ Array.terminate();
+ ConcurrentLog.config("SHUTDOWN", "caught termination signal");
+ httpServer.stop();
+
+ ConcurrentLog.config("SHUTDOWN", "server has terminated");
+ sb.close();
+ } catch (final Exception e) {
+ ConcurrentLog.severe("STARTUP", "Unexpected Error: " + e.getClass().getName(),e);
+ //System.exit(1);
+ }
+ if(lock != null) lock.release();
+ if(channel != null) channel.close();
+ } catch (final Exception ee) {
+ ConcurrentLog.severe("STARTUP", "FATAL ERROR: " + ee.getMessage(),ee);
+ } finally {
+ }
+
+ // save the triple store
+ if (sb.getConfigBool("triplestore.persistent", false)) {
+ JenaTripleStore.saveAll();
+ }
+
+ ConcurrentLog.config("SHUTDOWN", "goodbye. (this is the last line)");
+ ConcurrentLog.shutdown();
+ shutdownSemaphore.release(1000);
+ try {
+ System.exit(0);
+ } catch (final Exception e) {} // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790)
+ }
+
+ /**
+ * @param f
+ */
+ private static void delete(final File f) {
+ if(!f.delete())
+ ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be deleted!");
+ }
+
+ /**
+ * @see File#mkdir()
+ * @param path
+ */
+ private static void mkdirIfNeseccary(final File path) {
+ if (!(path.exists()))
+ if(!path.mkdir())
+ ConcurrentLog.warn("STARTUP", "could not create directory "+ path.toString());
+ }
+
+ /**
+ * @see File#mkdirs()
+ * @param path
+ */
+ public static void mkdirsIfNeseccary(final File path) {
+ if (!(path.exists()))
+ if(!path.mkdirs())
+ ConcurrentLog.warn("STARTUP", "could not create directories "+ path.toString());
+ }
+
+ /**
+ * Loads the configuration from the data-folder.
+ * FIXME: Why is this called over and over again from every method, instead
+ * of setting the configurationdata once for this class in main?
+ *
+ * @param mes Where are we called from, so that the errormessages can be
+ * more descriptive.
+ * @param homePath Root-path where all the information is to be found.
+ * @return Properties read from the configurationfile.
+ */
+ private static Properties configuration(final String mes, final File homePath) {
+ ConcurrentLog.config(mes, "Application Root Path: " + homePath.toString());
+
+ // read data folder
+ final File dataFolder = new File(homePath, "DATA");
+ if (!(dataFolder.exists())) {
+ ConcurrentLog.severe(mes, "Application was never started or root path wrong.");
+ System.exit(-1);
+ }
+
+ final Properties config = new Properties();
+ FileInputStream fis = null;
+ try {
+ fis = new FileInputStream(new File(homePath, "DATA/SETTINGS/yacy.conf"));
+ config.load(fis);
+ } catch (final FileNotFoundException e) {
+ ConcurrentLog.severe(mes, "could not find configuration file.");
+ System.exit(-1);
+ } catch (final IOException e) {
+ ConcurrentLog.severe(mes, "could not read configuration file.");
+ System.exit(-1);
+ } finally {
+ if(fis != null) {
+ try {
+ fis.close();
+ } catch (final IOException e) {
+ ConcurrentLog.logException(e);
+ }
+ }
+ }
+
+ return config;
+ }
+
+ /**
+ * Call the shutdown-page of YaCy to tell it to shut down. This method is
+ * called if you start yacy with the argument -shutdown.
+ *
+ * @param homePath Root-path where all the information is to be found.
+ */
+ public static void shutdown(final File homePath) {
+ // start up
+ System.out.println(copyright);
+ System.out.println(hline);
+ submitURL(homePath, "Steering.html?shutdown=", "Terminate YaCy");
+ }
+
+ public static void update(final File homePath) {
+ // start up
+ System.out.println(copyright);
+ System.out.println(hline);
+ submitURL(homePath, "ConfigUpdate_p.html?autoUpdate=", "Update YaCy to most recent version");
+ }
+
+ private static void submitURL(final File homePath, final String path, final String processdescription) {
+ final Properties config = configuration("COMMAND-STEERING", homePath);
+
+ // read port
+ final int port = serverCore.getPortNr(config.getProperty("port", "8090"));
+
+ // read password
+ String encodedPassword = (String) config.get(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5);
+ if (encodedPassword == null) encodedPassword = ""; // not defined
+
+ // send 'wget' to web interface
+ final RequestHeader requestHeader = new RequestHeader();
+ requestHeader.put(RequestHeader.AUTHORIZATION, "realm=" + encodedPassword); // for http-authentify
+// final Client con = new Client(10000, requestHeader);
+ final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
+ con.setHeader(requestHeader.entrySet());
+// ResponseContainer res = null;
+ try {
+// res = con.GET("http://localhost:"+ port +"/" + path);
+ con.GETbytes("http://localhost:"+ port +"/" + path);
+
+ // read response
+// if (res.getStatusLine().startsWith("2")) {
+ if (con.getStatusCode() > 199 && con.getStatusCode() < 300) {
+ ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);
+// final ByteArrayOutputStream bos = new ByteArrayOutputStream(); //This is stream is not used???
+// try {
+// FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(bos));
+// } finally {
+// res.closeStream();
+// }
+ } else {
+// Log.logSevere("COMMAND-STEERING", "error response from YACY socket: " + res.getStatusLine());
+ ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine());
+ System.exit(-1);
+ }
+ } catch (final IOException e) {
+ ConcurrentLog.severe("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage());
+ System.exit(-1);
+// } finally {
+// // release connection
+// if(res != null) {
+// res.closeStream();
+// }
+ }
+
+ try {
+ HTTPClient.closeConnectionManager();
+ } catch (final InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ // finished
+ ConcurrentLog.config("COMMAND-STEERING", "SUCCESSFULLY FINISHED COMMAND: " + processdescription);
+ }
+
+ /**
+ * Main-method which is started by java. Checks for special arguments or
+ * starts up the application.
+ *
+ * @param args
+ * Given arguments from the command line.
+ */
+ public static void main(String args[]) {
+
+ try {
+
+ // check assertion status
+ //ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true);
+ boolean assertionenabled = false;
+ assert (assertionenabled = true) == true; // compare to true to remove warning: "Possible accidental assignement"
+ if (assertionenabled) System.out.println("Asserts are enabled");
+
+ // check memory amount
+ System.gc();
+ final long startupMemFree = MemoryControl.free();
+ final long startupMemTotal = MemoryControl.total();
+
+ // maybe go into headless awt mode: we have three cases depending on OS and one exception:
+ // windows : better do not go into headless mode
+ // mac : go into headless mode because an application is shown in gui which may not be wanted
+ // linux : go into headless mode because this does not need any head operation
+ // exception : if the -gui option is used then do not go into headless mode since that uses a gui
+ boolean headless = true;
+ if (OS.isWindows) headless = false;
+ if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) headless = false;
+ System.setProperty("java.awt.headless", headless ? "true" : "false");
+ // System.setProperty("java.net.preferIPv4Stack", "true"); // DO NOT PREFER IPv6, i.e. freifunk uses ipv6 only and host resolving does not work
+
+ String s = ""; for (final String a: args) s += a + " ";
+ yacyRelease.startParameter = s.trim();
+
+ File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/'));
+ File dataRoot = applicationRoot;
+ //System.out.println("args.length=" + args.length);
+ //System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]");
+ if ((args.length >= 1) && (args[0].toLowerCase().equals("-startup") || args[0].equals("-start"))) {
+ // normal start-up of yacy
+ if (args.length > 1) dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
+ startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
+ } else if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) {
+ // start-up of yacy with gui
+ if (args.length > 1) dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
+ startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, true);
+ } else if ((args.length >= 1) && ((args[0].toLowerCase().equals("-shutdown")) || (args[0].equals("-stop")))) {
+ // normal shutdown of yacy
+ if (args.length == 2) applicationRoot= new File(args[1]);
+ shutdown(applicationRoot);
+ } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-update"))) {
+ // aut-update yacy
+ if (args.length == 2) applicationRoot= new File(args[1]);
+ update(applicationRoot);
+ } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-version"))) {
+ // show yacy version
+ System.out.println(copyright);
+ } else {
+ if (args.length == 1) applicationRoot= new File(args[0]);
+ startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
+ }
+ } finally {
+ ConcurrentLog.shutdown();
+ }
+ }
+}
+/**
+* This class is a helper class whose instance is started, when the java virtual
+* machine shuts down. Signals the plasmaSwitchboard to shut down.
+*/
+class shutdownHookThread extends Thread {
+ private final Switchboard sb;
+ private final Thread mainThread;
+
+ public shutdownHookThread(final Thread mainThread, final Switchboard sb) {
+ super();
+ this.sb = sb;
+ this.mainThread = mainThread;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (!this.sb.isTerminated()) {
+ ConcurrentLog.config("SHUTDOWN","Shutdown via shutdown hook.");
+
+ // sending the yacy main thread a shutdown signal
+ ConcurrentLog.fine("SHUTDOWN","Signaling shutdown to the switchboard.");
+ this.sb.terminate("shutdown hook");
+
+ // waiting for the yacy thread to finish execution
+ ConcurrentLog.fine("SHUTDOWN","Waiting for main thread to finish.");
+ if (this.mainThread.isAlive() && !this.sb.isTerminated()) {
+ this.mainThread.join();
+ }
+ }
+ } catch (final Exception e) {
+ ConcurrentLog.severe("SHUTDOWN","Unexpected error. " + e.getClass().getName(),e);
+ }
+ }
+}
|
