diff options
| author | reger <reger18@arcor.de> | 2013-10-09 00:40:48 +0200 |
|---|---|---|
| committer | reger <reger18@arcor.de> | 2013-10-09 00:40:48 +0200 |
| commit | 71d2655c02ca08e6b16528d3505ec616212240f8 (patch) | |
| tree | b06c7b4d9617f118f9075b33f033690fed3e6be3 /test | |
| parent | daebeb93aa9ffa850b8af445246e132cb73e398b (diff) | |
downgrade to Jetty 8 to assure support of JRE 1.6
- introduce a YaCyHttp interface to modulize/separate http server
- adjust the Jetty version specific implementation part (in package net.yacy.http)
- putting the version specific code in classes starting with Jetty8xxxx
- moved existing Jetty9xxx implementation into a test class (to keep the code)
- adjust build to the changed jars
- make use of the introduced YaCyHttpServer interface in related htroot servlets
- adjust other test cases/classes
Diffstat (limited to 'test')
| -rw-r--r-- | test/de/anomic/document/ParserTest.java | 10 | ||||
| -rw-r--r-- | test/de/anomic/yacy/yacyURLTest.java | 8 | ||||
| -rw-r--r-- | test/net/yacy/cora/document/MultiProtocolURITest.java | 7 | ||||
| -rw-r--r-- | test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java | 16 | ||||
| -rw-r--r-- | test/net/yacy/http/Jetty9HttpServerImpl.java | 190 | ||||
| -rw-r--r-- | test/net/yacy/http/Jetty9YaCyDefaultServlet.java | 924 | ||||
| -rw-r--r-- | test/net/yacy/http/Jetty9YaCySecurityHandler.java | 174 | ||||
| -rw-r--r-- | test/net/yacy/http/Jetty9_Implementation_Test.java | 34 | ||||
| -rw-r--r-- | test/net/yacy/http/YacyMain.java | 702 |
9 files changed, 2048 insertions, 17 deletions
diff --git a/test/de/anomic/document/ParserTest.java b/test/de/anomic/document/ParserTest.java index 66b5891b1..701506438 100644 --- a/test/de/anomic/document/ParserTest.java +++ b/test/de/anomic/document/ParserTest.java @@ -11,6 +11,7 @@ import java.io.InputStreamReader; import java.io.Reader; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; +import net.yacy.cora.document.id.AnchorURL; import net.yacy.document.AbstractParser; import net.yacy.document.Document; @@ -19,7 +20,6 @@ import net.yacy.document.parser.docParser; import net.yacy.document.parser.odtParser; import net.yacy.document.parser.ooxmlParser; import net.yacy.document.parser.pdfParser; -import net.yacy.kelondro.data.meta.DigestURI; import org.junit.Test; @@ -38,7 +38,7 @@ public class ParserTest { final String filename = "test/parsertest/" + testFile[0]; final File file = new File(filename); final String mimetype = testFile[1]; - final DigestURI url = new DigestURI("http://localhost/"+filename); + final AnchorURL url = new AnchorURL("http://localhost/"+filename); AbstractParser p = new ooxmlParser(); final Document[] docs = p.parse(url, mimetype, null, new FileInputStream(file)); @@ -72,7 +72,7 @@ public class ParserTest { final String filename = "test/parsertest/" + testFile[0]; final File file = new File(filename); final String mimetype = testFile[1]; - final DigestURI url = new DigestURI("http://localhost/"+filename); + final AnchorURL url = new AnchorURL("http://localhost/"+filename); AbstractParser p = new odtParser(); final Document[] docs = p.parse(url, mimetype, null, new FileInputStream(file)); @@ -104,7 +104,7 @@ public class ParserTest { final String filename = "test/parsertest/" + testFile[0]; final File file = new File(filename); final String mimetype = testFile[1]; - final DigestURI url = new DigestURI("http://localhost/"+filename); + final AnchorURL url = new AnchorURL("http://localhost/"+filename); AbstractParser p = new pdfParser(); final Document[] docs = p.parse(url, mimetype, null, new FileInputStream(file)); @@ -136,7 +136,7 @@ public class ParserTest { final String filename = "test/parsertest/" + testFile[0]; final File file = new File(filename); final String mimetype = testFile[1]; - final DigestURI url = new DigestURI("http://localhost/"+filename); + final AnchorURL url = new AnchorURL("http://localhost/"+filename); AbstractParser p = new docParser(); final Document[] docs = p.parse(url, mimetype, null, new FileInputStream(file)); diff --git a/test/de/anomic/yacy/yacyURLTest.java b/test/de/anomic/yacy/yacyURLTest.java index a0439a4e1..00bc8b78c 100644 --- a/test/de/anomic/yacy/yacyURLTest.java +++ b/test/de/anomic/yacy/yacyURLTest.java @@ -2,10 +2,10 @@ package de.anomic.yacy; import java.net.MalformedURLException; -import net.yacy.cora.document.MultiProtocolURI; -import net.yacy.kelondro.data.meta.DigestURI; import junit.framework.TestCase; +import net.yacy.cora.document.id.DigestURL; +import net.yacy.cora.document.id.MultiProtocolURL; public class yacyURLTest extends TestCase { @@ -32,7 +32,7 @@ public class yacyURLTest extends TestCase { // conversion result String resolvedURL=""; try { - resolvedURL = (new MultiProtocolURI(testhost + testStrings[i][0])).toString(); + resolvedURL = (new MultiProtocolURL(testhost + testStrings[i][0])).toString(); } catch (final MalformedURLException ex) { fail ("malformed URL"); } @@ -57,7 +57,7 @@ public class yacyURLTest extends TestCase { String shouldBe = testStrings[i][1]; // conversion result - String resolvedURL = (new DigestURI(testStrings[i][0])).toString(); + String resolvedURL = (new DigestURL(testStrings[i][0])).toString(); // test if equal assertEquals(shouldBe,resolvedURL); diff --git a/test/net/yacy/cora/document/MultiProtocolURITest.java b/test/net/yacy/cora/document/MultiProtocolURITest.java index f69098727..ae6cf5ab1 100644 --- a/test/net/yacy/cora/document/MultiProtocolURITest.java +++ b/test/net/yacy/cora/document/MultiProtocolURITest.java @@ -4,8 +4,7 @@ import static org.junit.Assert.*; import java.net.MalformedURLException; import java.util.TreeSet; - -import net.yacy.cora.document.MultiProtocolURI; +import net.yacy.cora.document.id.MultiProtocolURL; import org.junit.Test; @@ -23,10 +22,10 @@ public class MultiProtocolURITest { TreeSet<String> idNames = new TreeSet<String>(); idNames.add("phpsessionid"); - MultiProtocolURI.initSessionIDNames(idNames); + MultiProtocolURL.initSessionIDNames(idNames); for (int i=0; i<testURIs.length; i++) { - MultiProtocolURI uri = new MultiProtocolURI(testURIs[i][0]); + MultiProtocolURL uri = new MultiProtocolURL(testURIs[i][0]); assertEquals(uri.toNormalform(true, true), testURIs[i][1]); } diff --git a/test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java b/test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java index 69ae7abbf..ef43842e0 100644 --- a/test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java +++ b/test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java @@ -4,6 +4,8 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.EnumSet; +import javax.servlet.DispatcherType; import net.yacy.cora.federate.solr.SolrServlet; import net.yacy.cora.federate.solr.instance.EmbeddedInstance; import net.yacy.search.schema.CollectionSchema; @@ -13,8 +15,9 @@ import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrInputDocument; import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.servlet.FilterHolder; -import org.eclipse.jetty.servlet.ServletContextHandler.Context; +import org.eclipse.jetty.servlet.ServletContextHandler; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -122,14 +125,19 @@ public class EmbeddedSolrConnectorTest { this.server.setConnectors(new Connector[] { connector }); this.server.setSessionIdManager(new HashSessionIdManager(new Random())); */ + ServerConnector connector = new ServerConnector(server); + connector.setPort(port); + + server.addConnector(connector); server.setStopAtShutdown(true); - Context root = new Context(server, context, Context.SESSIONS); + ServletContextHandler root = new ServletContextHandler(ServletContextHandler.SESSIONS); //new Context(server, context, Context.SESSIONS); root.addServlet(SolrServlet.Servlet404.class, "/*"); // attach org.apache.solr.response.XMLWriter to search requests SolrServlet.initCore(c); - FilterHolder dispatchFilter = root.addFilter(SolrServlet.class, "*", Handler.REQUEST); - + FilterHolder dispatchFilter = root.addFilter(SolrServlet.class, "*", EnumSet.of(DispatcherType.REQUEST)); + //root.addFilter(new FilterHolder(SolrServlet.class), "/*", EnumSet.of(DispatcherType.REQUEST)); + server.setHandler(root); if (!server.isRunning()) { try { server.start(); diff --git a/test/net/yacy/http/Jetty9HttpServerImpl.java b/test/net/yacy/http/Jetty9HttpServerImpl.java new file mode 100644 index 000000000..5247d4194 --- /dev/null +++ b/test/net/yacy/http/Jetty9HttpServerImpl.java @@ -0,0 +1,190 @@ +// +// Jetty9HttpServerImpl +// Copyright 2011 by Florian Richter +// First released 13.04.2011 at http://yacy.net +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program in the file lgpl21.txt +// If not, see <http://www.gnu.org/licenses/>. +// + +package net.yacy.http; + +import java.net.InetSocketAddress; +import java.net.SocketException; +import java.util.EnumSet; +import javax.servlet.DispatcherType; +import net.yacy.cora.federate.solr.SolrServlet; +import net.yacy.cora.federate.solr.SolrServlet.Servlet404; +import net.yacy.cora.util.ConcurrentLog; +import net.yacy.search.Switchboard; + +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.server.handler.ContextHandler; +import org.eclipse.jetty.server.handler.ContextHandlerCollection; +import org.eclipse.jetty.server.handler.DefaultHandler; +import org.eclipse.jetty.server.handler.HandlerList; +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; + +/** + * class to embedded Jetty 9 http server into YaCy + */ +public class Jetty9HttpServerImpl implements YaCyHttpServer { + + private Server server; + + /** + * @param port TCP Port to listen for http requests + */ + public Jetty9HttpServerImpl(int port) { + Switchboard sb = Switchboard.getSwitchboard(); + + server = new Server(); + ServerConnector connector = new ServerConnector(server); + connector.setPort(port); + connector.setName("httpd:"+Integer.toString(port)); + //connector.setThreadPool(new QueuedThreadPool(20)); + server.addConnector(connector); + + YacyDomainHandler domainHandler = new YacyDomainHandler(); + domainHandler.setAlternativeResolver(sb.peers); + + /* this is now handled by YaCyDefaultServlet + ResourceHandler resource_handler = new ResourceHandler(); + resource_handler.setDirectoriesListed(true); + resource_handler.setWelcomeFiles(new String[]{"index.html"}); + resource_handler.setResourceBase("htroot/"); + */ + + //add SolrServlet + ServletContextHandler solrContext = new ServletContextHandler(ServletContextHandler.SESSIONS); + solrContext.setContextPath("/solr"); + solrContext.addServlet(new ServletHolder(Servlet404.class),"/*"); + + SolrServlet.initCore(sb.index.fulltext().getDefaultEmbeddedConnector()); + solrContext.addFilter(new FilterHolder(SolrServlet.class), "/*", EnumSet.of(DispatcherType.REQUEST)); + + // configure root context + ServletContextHandler htrootContext = new ServletContextHandler(ServletContextHandler.SESSIONS); + htrootContext.setContextPath("/"); + ServletHolder sholder = new ServletHolder(Jetty9YaCyDefaultServlet.class); + sholder.setInitParameter("resourceBase", "htroot"); + //sholder.setInitParameter("welcomeFile", "index.html"); // default is index.html, welcome.html + sholder.setInitParameter("gzip","false"); + htrootContext.addServlet(sholder,"/*"); + + // assemble the servlet handlers + ContextHandlerCollection servletContext = new ContextHandlerCollection(); + servletContext.setHandlers(new Handler[] { solrContext, htrootContext }); + + // define list of YaCy specific general handlers + HandlerList handlers = new HandlerList(); + handlers.setHandlers(new Handler[] + {domainHandler, new ProxyCacheHandler(), new ProxyHandler() + /*, resource_handler, new DefaultHandler() */}); + + // context handler for dispatcher and security (hint: dispatcher requires a context) + ContextHandler context = new ContextHandler(); + context.setContextPath("/"); + context.setHandler(handlers); + + // make YaCy handlers (in context) and servlet context handlers available (both contain root context "/") + // logic: 1. YaCy handlers are called if request not handled (e.g. proxy) then servlets handle it + ContextHandlerCollection allrequesthandlers = new ContextHandlerCollection(); + allrequesthandlers.addHandler(context); + allrequesthandlers.addHandler(servletContext); + allrequesthandlers.addHandler(new DefaultHandler()); // if not handled by other handler + + // wrap all handlers by security handler + Jetty9YaCySecurityHandler securityHandler = new Jetty9YaCySecurityHandler(); + securityHandler.setLoginService(new YaCyLoginService()); + securityHandler.setRealmName("YaCy Admin Interface"); + securityHandler.setHandler(new CrashProtectionHandler(allrequesthandlers)); + + server.setHandler(securityHandler); + } + + /** + * start http server + */ + @Override + public void startupServer() throws Exception { + server.start(); + } + + /** + * stop http server and wait for it + */ + @Override + public void stop() throws Exception { + server.stop(); + server.join(); + } + + @Override + public void setMaxSessionCount(int maxBusy) { + // TODO: + } + + @Override + public boolean withSSL() { + return false; // TODO: + } + + @Override + public void reconnect(int milsec) { + try { + Thread.sleep(milsec); + } catch (final InterruptedException e) { + ConcurrentLog.logException(e); + } catch (final Exception e) { + ConcurrentLog.logException(e); + } + try { + server.stop(); + server.join(); + server.start(); + } catch (Exception ex) { + ConcurrentLog.logException(ex); + } + } + + @Override + public InetSocketAddress generateSocketAddress(String port) throws SocketException { + return null; // TODO: + } + + @Override + public int getMaxSessionCount() { + return server.getThreadPool().getThreads(); + } + + @Override + public int getJobCount() { + return getMaxSessionCount() - server.getThreadPool().getIdleThreads(); // TODO: + } + + @Override + public String getVersion() { + return "Jetty " + server.getVersion(); + } + +} diff --git a/test/net/yacy/http/Jetty9YaCyDefaultServlet.java b/test/net/yacy/http/Jetty9YaCyDefaultServlet.java new file mode 100644 index 000000000..ab607cbbd --- /dev/null +++ b/test/net/yacy/http/Jetty9YaCyDefaultServlet.java @@ -0,0 +1,924 @@ +// YaCyDefaultServlet
+// Copyright 2013 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// First released 2013 at http://yacy.net
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with this program in the file lgpl21.txt
+// If not, see <http://www.gnu.org/licenses/>.
+//
+package net.yacy.http;
+
+import java.awt.Image;
+import java.awt.image.BufferedImage;
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.lang.ref.SoftReference;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.Enumeration;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import javax.servlet.AsyncContext;
+import javax.servlet.RequestDispatcher;
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.UnavailableException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import net.yacy.cora.date.GenericFormatter;
+import net.yacy.cora.document.analysis.Classification;
+import net.yacy.cora.protocol.HeaderFramework;
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.kelondro.util.FileUtils;
+import net.yacy.kelondro.util.MemoryControl;
+import net.yacy.peers.Seed;
+import net.yacy.peers.graphics.EncodedImage;
+import net.yacy.peers.operation.yacyBuildProperties;
+import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
+import net.yacy.server.http.TemplateEngine;
+import net.yacy.server.serverClassLoader;
+import net.yacy.server.serverCore;
+import net.yacy.server.serverObjects;
+import net.yacy.server.serverSwitch;
+import net.yacy.server.servletProperties;
+import net.yacy.visualization.RasterPlotter;
+import org.apache.commons.fileupload.FileItem;
+import org.apache.commons.fileupload.disk.DiskFileItemFactory;
+import org.apache.commons.fileupload.servlet.ServletFileUpload;
+
+import org.eclipse.jetty.http.HttpContent;
+import org.eclipse.jetty.http.HttpFields;
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpMethod;
+import org.eclipse.jetty.http.MimeTypes;
+import org.eclipse.jetty.io.WriterOutputStream;
+import org.eclipse.jetty.server.HttpOutput;
+import org.eclipse.jetty.server.InclusiveByteRange;
+import org.eclipse.jetty.server.Response;
+import org.eclipse.jetty.util.BufferUtil;
+import org.eclipse.jetty.util.Callback;
+import org.eclipse.jetty.util.MultiPartOutputStream;
+import org.eclipse.jetty.util.QuotedStringTokenizer;
+import org.eclipse.jetty.util.URIUtil;
+import org.eclipse.jetty.util.resource.Resource;
+import org.eclipse.jetty.util.resource.ResourceFactory;
+
+/**
+ * YaCyDefaultServlet base on Jetty DefaultServlet.java
+ * handles static files and the YaCy servlets.
+ */
+
+/**
+ * The default servlet. This servlet, normally mapped to /, provides the
+ * handling for static content, OPTION and TRACE methods for the context. The
+ * following initParameters are supported, these can be set either on the
+ * servlet itself or as ServletContext initParameters :
+ * <PRE>
+ * acceptRanges If true, range requests and responses are
+ * supported
+ *
+ * dirAllowed If true, directory listings are returned if no
+ * welcome file is found. Else 403 Forbidden.
+ *
+ * welcomeFile name of the welcome file (default is "index.html", "welcome.html")
+ *
+ * gzip If set to true, then static content will be served as
+ * gzip content encoded if a matching resource is
+ * found ending with ".gz"
+ *
+ * resourceBase Set to replace the context resource base
+ *
+ * resourceCache If set, this is a context attribute name, which the servlet
+ * will use to look for a shared ResourceCache instance.
+ *
+ * relativeResourceBase
+ * Set with a pathname relative to the base of the
+ * servlet context root. Useful for only serving static content out
+ * of only specific subdirectories.
+ *
+ * pathInfoOnly If true, only the path info will be applied to the resourceBase
+ *
+ *
+ * etags If True, weak etags will be generated and handled.
+ *
+ * </PRE>
+ */
+public class Jetty9YaCyDefaultServlet extends YaCyDefaultServlet implements ResourceFactory {
+
+ private static final long serialVersionUID = 4900000000000001110L;
+
+ private boolean _gzip=true;
+
+ /* ------------------------------------------------------------ */
+ @Override
+ public void init() throws UnavailableException {
+ super.init();
+ _gzip=getInitBoolean("gzip",_gzip);
+ }
+
+ /* ------------------------------------------------------------ */
+ /**
+ * get Resource to serve. Map a path to a resource. The default
+ * implementation calls HttpContext.getResource but derived servlets may
+ * provide their own mapping.
+ *
+ * @param pathInContext The path to find a resource for.
+ * @return The resource to serve.
+ */
+ @Override
+ public Resource getResource(String pathInContext) {
+ Resource r = null;
+ if (_relativeResourceBase != null) {
+ pathInContext = URIUtil.addPaths(_relativeResourceBase, pathInContext);
+ }
+
+ try {
+ if (_resourceBase != null) {
+ r = _resourceBase.addPath(pathInContext);
+ } else {
+ URL u = _servletContext.getResource(pathInContext);
+ r = Resource.newResource(u);
+ }
+
+ if (ConcurrentLog.isFine("YaCyDefaultServlet")) {
+ ConcurrentLog.fine("YaCyDefaultServlet","Resource " + pathInContext + "=" + r);
+ }
+ } catch (IOException e) {
+ // ConcurrentLog.logException(e);
+ }
+
+ return r;
+ }
+
+ /* ------------------------------------------------------------ */
+ @Override
+ protected void doGet(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+ String servletPath = null;
+ String pathInfo = null;
+ Enumeration<String> reqRanges = null;
+ Boolean included = request.getAttribute(RequestDispatcher.INCLUDE_REQUEST_URI) != null;
+ if (included != null && included.booleanValue()) {
+ servletPath = (String) request.getAttribute(RequestDispatcher.INCLUDE_SERVLET_PATH);
+ pathInfo = (String) request.getAttribute(RequestDispatcher.INCLUDE_PATH_INFO);
+ if (servletPath == null) {
+ servletPath = request.getServletPath();
+ pathInfo = request.getPathInfo();
+ }
+ } else {
+ included = Boolean.FALSE;
+ servletPath = _pathInfoOnly ? "/" : request.getServletPath();
+ pathInfo = request.getPathInfo();
+
+ // Is this a Range request?
+ reqRanges = request.getHeaders(HeaderFramework.RANGE);
+ if (!hasDefinedRange(reqRanges)) {
+ reqRanges = null;
+ }
+ }
+
+ if (pathInfo.startsWith("/currentyacypeer/")) pathInfo = pathInfo.substring(16);
+ String pathInContext = URIUtil.addPaths(servletPath, pathInfo);
+ boolean endsWithSlash = (pathInfo == null ? request.getServletPath() : pathInfo).endsWith(URIUtil.SLASH);
+
+ // Find the resource and content
+ Resource resource = null;
+ HttpContent content = null;
+ try {
+ // Look for a class resource
+ boolean hasClass = false;
+ if (reqRanges == null && !endsWithSlash) {
+ final int p = pathInContext.lastIndexOf('.');
+ if (p >= 0) {
+ String pathofClass = pathInContext.substring(0, p) + ".class";
+ resource = getResource(pathofClass);
+ // Does a class resource exist?
+ if (resource != null && resource.exists() && !resource.isDirectory()) {
+ hasClass = true;
+ }
+ }
+ }
+ // is gzip enabled?
+ String pathInContextGz=null;
+ boolean gzip=false;
+ if (!included.booleanValue() && _gzip && reqRanges==null && !endsWithSlash )
+ {
+ // Look for a gzip resource
+ pathInContextGz=pathInContext+".gz";
+ resource=getResource(pathInContextGz);
+ // Does a gzip resource exist?
+ if (resource!=null && resource.exists() && !resource.isDirectory())
+ {
+ // Tell caches that response may vary by accept-encoding
+ response.addHeader(HttpHeader.VARY.asString(),HttpHeader.ACCEPT_ENCODING.asString());
+
+ // Does the client accept gzip?
+ String accept=request.getHeader(HttpHeader.ACCEPT_ENCODING.asString());
+ if (accept!=null && accept.indexOf("gzip")>=0)
+ gzip=true;
+ }
+ }
+
+ // find resource
+ if (!gzip) resource = getResource(pathInContext);
+
+ if (ConcurrentLog.isFine("YaCyDefaultServlet")) {
+ ConcurrentLog.fine("YaCyDefaultServlet","uri=" + request.getRequestURI() + " resource=" + resource + (content != null ? " content" : ""));
+ }
+
+ // Handle resource
+ if (!hasClass && (resource == null || !resource.exists())) {
+ if (included) {
+ throw new FileNotFoundException("!" + pathInContext);
+ }
+ response.sendError(HttpServletResponse.SC_NOT_FOUND);
+ } else if (!resource.isDirectory()) {
+ if (endsWithSlash && pathInContext.length() > 1) {
+ String q = request.getQueryString();
+ pathInContext = pathInContext.substring(0, pathInContext.length() - 1);
+ if (q != null && q.length() != 0) {
+ pathInContext += "?" + q;
+ }
+ response.sendRedirect(response.encodeRedirectURL(URIUtil.addPaths(_servletContext.getContextPath(), pathInContext)));
+ } else {
+ // ensure we have content
+ if (content == null) {
+ content = new HttpContent.ResourceAsHttpContent(resource, _mimeTypes.getMimeByExtension(resource.toString()), response.getBufferSize(), _etags);
+ }
+
+ if (hasClass) { // this is a YaCy servlet, handle the template
+ handleTemplate(pathInfo, request, response);
+ } else {
+ if (included.booleanValue() || passConditionalHeaders(request, response, resource, content)) {
+ //sendData(request, response, included.booleanValue(), resource, content, reqRanges);
+ if (gzip) {
+ response.setHeader(HttpHeader.CONTENT_ENCODING.asString(), "gzip");
+ String mt = _servletContext.getMimeType(pathInContext);
+ if (mt != null) {
+ response.setContentType(mt);
+ }
+ }
+ sendData(request, response, included.booleanValue(), resource, content, reqRanges);
+ }
+ }
+ }
+ } else {
+ if (!endsWithSlash || (pathInContext.length() == 1 && request.getAttribute("org.eclipse.jetty.server.nullPathInfo") != null)) {
+ StringBuffer buf = request.getRequestURL();
+ synchronized (buf) {
+ int param = buf.lastIndexOf(";");
+ if (param < 0) {
+ buf.append('/');
+ } else {
+ buf.insert(param, '/');
+ }
+ String q = request.getQueryString();
+ if (q != null && q.length() != 0) {
+ buf.append('?');
+ buf.append(q);
+ }
+ response.setContentLength(0);
+ response.sendRedirect(response.encodeRedirectURL(buf.toString()));
+ }
+ } else { // look for a welcome file
+ String welcomeFileName = getWelcomeFile (pathInContext);
+ if (welcomeFileName != null) {
+ RequestDispatcher rd = request.getRequestDispatcher(welcomeFileName);
+ rd.forward(request, response);
+ } else { // send directory listing
+ content = new HttpContent.ResourceAsHttpContent(resource, _mimeTypes.getMimeByExtension(resource.toString()), _etags);
+ if (included.booleanValue() || passConditionalHeaders(request, response, resource, content)) {
+ sendDirectory(request, response, resource, pathInContext);
+ }
+ }
+ }
+ }
+ } catch (IllegalArgumentException e) {
+ ConcurrentLog.logException(e);
+ if (!response.isCommitted()) {
+ response.sendError(500, e.getMessage());
+ }
+ } finally {
+ if (content != null) {
+ content.release();
+ } else if (resource != null) {
+ resource.close();
+ }
+ }
+ }
+
+ /* ------------------------------------------------------------ */
+ @Override
+ protected boolean hasDefinedRange(Enumeration<String> reqRanges) {
+ return (reqRanges != null && reqRanges.hasMoreElements());
+ }
+
+ /* ------------------------------------------------------------ */
+ @Override
+ protected void doPost(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+ doGet(request, response);
+ }
+
+ /* ------------------------------------------------------------ */
+ /* Check modification date headers.
+ */
+ @Override
+ protected boolean passConditionalHeaders(HttpServletRequest request, HttpServletResponse response, Resource resource, HttpContent content)
+ throws IOException {
+ try {
+ if (!HttpMethod.HEAD.is(request.getMethod())) {
+ if (_etags) {
+ String ifm = request.getHeader(HttpHeader.IF_MATCH.asString());
+ if (ifm != null) {
+ boolean match = false;
+ if (content.getETag() != null) {
+ QuotedStringTokenizer quoted = new QuotedStringTokenizer(ifm, ", ", false, true);
+ while (!match && quoted.hasMoreTokens()) {
+ String tag = quoted.nextToken();
+ if (content.getETag().toString().equals(tag)) {
+ match = true;
+ }
+ }
+ }
+
+ if (!match) {
+ response.setStatus(HttpServletResponse.SC_PRECONDITION_FAILED);
+ return false;
+ }
+ }
+
+ String if_non_match_etag = request.getHeader(HttpHeader.IF_NONE_MATCH.asString());
+ if (if_non_match_etag != null && content.getETag() != null) {
+ // Look for GzipFiltered version of etag
+ if (content.getETag().toString().equals(request.getAttribute("o.e.j.s.GzipFilter.ETag"))) {
+ response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
+ response.setHeader(HeaderFramework.ETAG, if_non_match_etag);
+ return false;
+ }
+
+ // Handle special case of exact match.
+ if (content.getETag().toString().equals(if_non_match_etag)) {
+ response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
+ response.setHeader(HeaderFramework.ETAG, content.getETag());
+ return false;
+ }
+
+ // Handle list of tags
+ QuotedStringTokenizer quoted = new QuotedStringTokenizer(if_non_match_etag, ", ", false, true);
+ while (quoted.hasMoreTokens()) {
+ String tag = quoted.nextToken();
+ if (content.getETag().toString().equals(tag)) {
+ response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
+ response.setHeader(HeaderFramework.ETAG, content.getETag());
+ return false;
+ }
+ }
+
+ // If etag requires content to be served, then do not check if-modified-since
+ return true;
+ }
+ }
+
+ // Handle if modified since
+ String ifms = request.getHeader(HttpHeader.IF_MODIFIED_SINCE.asString());
+ if (ifms != null) {
+ //Get jetty's Response impl
+ String mdlm = content.getLastModified();
+ if (mdlm != null && ifms.equals(mdlm)) {
+ response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
+ if (_etags) {
+ response.setHeader(HeaderFramework.ETAG, content.getETag());
+ }
+ response.flushBuffer();
+ return false;
+ }
+
+ long ifmsl = request.getDateHeader(HttpHeader.IF_MODIFIED_SINCE.asString());
+ if (ifmsl != -1 && resource.lastModified() / 1000 <= ifmsl / 1000) {
+ response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
+ if (_etags) {
+ response.setHeader(HeaderFramework.ETAG, content.getETag());
+ }
+ response.flushBuffer();
+ return false;
+ }
+ }
+
+ // Parse the if[un]modified dates and compare to resource
+ long date = request.getDateHeader(HttpHeader.IF_UNMODIFIED_SINCE.asString());
+ if (date != -1 && resource.lastModified() / 1000 > date / 1000) {
+ response.sendError(HttpServletResponse.SC_PRECONDITION_FAILED);
+ return false;
+ }
+
+ }
+ } catch (IllegalArgumentException iae) {
+ if (!response.isCommitted()) {
+ response.sendError(400, iae.getMessage());
+ }
+ throw iae;
+ }
+ return true;
+ }
+
+
+ /* ------------------------------------------------------------------- */
+ @Override
+ protected void sendDirectory(HttpServletRequest request,
+ HttpServletResponse response,
+ Resource resource,
+ String pathInContext)
+ throws IOException {
+ if (!_dirAllowed) {
+ response.sendError(HttpServletResponse.SC_FORBIDDEN);
+ return;
+ }
+
+ byte[] data = null;
+ String base = URIUtil.addPaths(request.getRequestURI(), URIUtil.SLASH);
+
+ String dir = resource.getListHTML(base, pathInContext.length() > 1);
+ if (dir == null) {
+ response.sendError(HttpServletResponse.SC_FORBIDDEN,
+ "No directory");
+ return;
+ }
+
+ data = dir.getBytes("UTF-8");
+ response.setContentType("text/html; charset=UTF-8");
+ response.setContentLength(data.length);
+ response.getOutputStream().write(data);
+ }
+
+ /* ------------------------------------------------------------ */
+ @Override
+ protected void sendData(HttpServletRequest request,
+ HttpServletResponse response,
+ boolean include,
+ Resource resource,
+ HttpContent content,
+ Enumeration<String> reqRanges)
+ throws IOException {
+ final long content_length = (content == null) ? resource.length() : content.getContentLength();
+
+ // Get the output stream (or writer)
+ OutputStream out = null;
+ boolean written;
+ try {
+ out = response.getOutputStream();
+
+ // has a filter already written to the response?
+ written = out instanceof HttpOutput
+ ? ((HttpOutput) out).isWritten()
+ : true;
+ } catch (IllegalStateException e) {
+ out = new WriterOutputStream(response.getWriter());
+ written = true; // there may be data in writer buffer, so assume written
+ }
+
+ if (reqRanges == null || !reqRanges.hasMoreElements() || content_length < 0) {
+ // if there were no ranges, send entire entity
+ if (include) {
+ resource.writeTo(out, 0, content_length);
+ } // else if we can't do a bypass write because of wrapping
+ else if (content == null || written || !(out instanceof HttpOutput)) {
+ // write normally
+ writeHeaders(response, content, written ? -1 : content_length);
+ ByteBuffer buffer = (content == null) ? null : content.getIndirectBuffer();
+ if (buffer != null) {
+ BufferUtil.writeTo(buffer, out);
+ } else {
+ resource.writeTo(out, 0, content_length);
+ }
+ } // else do a bypass write
+ else {
+ // write the headers
+ if (response instanceof Response) {
+ Response r = (Response) response;
+ writeOptionHeaders(r.getHttpFields());
+ r.setHeaders(content);
+ } else {
+ writeHeaders(response, content, content_length);
+ }
+
+ // write the content asynchronously if supported
+ if (request.isAsyncSupported()) {
+ final AsyncContext context = request.startAsync();
+
+ ((HttpOutput) out).sendContent(content, new Callback() {
+ @Override
+ public void succeeded() {
+ context.complete();
+ }
+
+ @Override
+ public void failed(Throwable x) {
+ ConcurrentLog.logException(x);
+ context.complete();
+ }
+ });
+ } // otherwise write content blocking
+ else {
+ ((HttpOutput) out).sendContent(content);
+ }
+ }
+ } else {
+ // Parse the satisfiable ranges
+ List<InclusiveByteRange> ranges = InclusiveByteRange.satisfiableRanges(reqRanges, content_length);
+
+ // if there are no satisfiable ranges, send 416 response
+ if (ranges == null || ranges.size() == 0) {
+ writeHeaders(response, content, content_length);
+ response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
+ response.setHeader(HeaderFramework.CONTENT_RANGE,
+ InclusiveByteRange.to416HeaderRangeString(content_length));
+ resource.writeTo(out, 0, content_length);
+ return;
+ }
+
+ // if there is only a single valid range (must be satisfiable
+ // since were here now), send that range with a 216 response
+ if (ranges.size() == 1) {
+ InclusiveByteRange singleSatisfiableRange = ranges.get(0);
+ long singleLength = singleSatisfiableRange.getSize(content_length);
+ writeHeaders(response, content, singleLength);
+ response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
+ response.setHeader(HeaderFramework.CONTENT_RANGE,
+ singleSatisfiableRange.toHeaderRangeString(content_length));
+ resource.writeTo(out, singleSatisfiableRange.getFirst(content_length), singleLength);
+ return;
+ }
+
+ // multiple non-overlapping valid ranges cause a multipart
+ // 216 response which does not require an overall
+ // content-length header
+ //
+ writeHeaders(response, content, -1);
+ String mimetype = (content == null || content.getContentType() == null ? null : content.getContentType().toString());
+ if (mimetype == null) {
+ ConcurrentLog.warn("YaCyDefaultServlet", "Unknown mimetype for " + request.getRequestURI());
+ }
+ MultiPartOutputStream multi = new MultiPartOutputStream(out);
+ response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
+
+ // If the request has a "Request-Range" header then we need to
+ // send an old style multipart/x-byteranges Content-Type. This
+ // keeps Netscape and acrobat happy. This is what Apache does.
+ String ctp;
+ if (request.getHeader(HttpHeader.REQUEST_RANGE.asString()) != null) {
+ ctp = "multipart/x-byteranges; boundary=";
+ } else {
+ ctp = "multipart/byteranges; boundary=";
+ }
+ response.setContentType(ctp + multi.getBoundary());
+
+ InputStream in = resource.getInputStream();
+ long pos = 0;
+
+ // calculate the content-length
+ int length = 0;
+ String[] header = new String[ranges.size()];
+ for (int i = 0; i < ranges.size(); i++) {
+ InclusiveByteRange ibr = ranges.get(i);
+ header[i] = ibr.toHeaderRangeString(content_length);
+ length +=
+ ((i > 0) ? 2 : 0)
+ + 2 + multi.getBoundary().length() + 2
+ + (mimetype == null ? 0 : HeaderFramework.CONTENT_TYPE.length() + 2 + mimetype.length()) + 2
+ + HeaderFramework.CONTENT_RANGE.length() + 2 + header[i].length() + 2
+ + 2
+ + (ibr.getLast(content_length) - ibr.getFirst(content_length)) + 1;
+ }
+ length += 2 + 2 + multi.getBoundary().length() + 2 + 2;
+ response.setContentLength(length);
+
+ for (int i = 0; i < ranges.size(); i++) {
+ InclusiveByteRange ibr = ranges.get(i);
+ multi.startPart(mimetype, new String[]{HeaderFramework.CONTENT_RANGE + ": " + header[i]});
+
+ long start = ibr.getFirst(content_length);
+ long size = ibr.getSize(content_length);
+ if (in != null) {
+ // Handle non cached resource
+ if (start < pos) {
+ in.close();
+ in = resource.getInputStream();
+ pos = 0;
+ }
+ if (pos < start) {
+ in.skip(start - pos);
+ pos = start;
+ }
+
+ FileUtils.copy(in, multi, size);
+ pos += size;
+ } else // Handle cached resource
+ {
+ (resource).writeTo(multi, start, size);
+ }
+
+ }
+ if (in != null) {
+ in.close();
+ }
+ multi.close();
+ }
+ }
+
+ /* ------------------------------------------------------------ */
+ @Override
+ protected void writeHeaders(HttpServletResponse response, HttpContent content, long count) {
+ if (content.getContentType() != null && response.getContentType() == null) {
+ response.setContentType(content.getContentType().toString());
+ }
+
+ if (response instanceof Response) {
+ Response r = (Response) response;
+ HttpFields fields = r.getHttpFields();
+
+ if (content.getLastModified() != null) {
+ fields.put(HeaderFramework.LAST_MODIFIED, content.getLastModified());
+ } else if (content.getResource() != null) {
+ long lml = content.getResource().lastModified();
+ if (lml != -1) {
+ fields.putDateField(HeaderFramework.LAST_MODIFIED, lml);
+ }
+ }
+
+ if (count != -1) {
+ r.setLongContentLength(count);
+ }
+
+ writeOptionHeaders(fields);
+
+ if (_etags) {
+ fields.put(HeaderFramework.ETAG, content.getETag());
+ }
+ } else {
+ long lml = content.getResource().lastModified();
+ if (lml >= 0) {
+ response.setDateHeader(HeaderFramework.LAST_MODIFIED, lml);
+ }
+
+ if (count != -1) {
+ if (count < Integer.MAX_VALUE) {
+ response.setContentLength((int) count);
+ } else {
+ response.setHeader(HeaderFramework.CONTENT_LENGTH, Long.toString(count));
+ }
+ }
+
+ writeOptionHeaders(response);
+
+ if (_etags) {
+ response.setHeader(HeaderFramework.ETAG, content.getETag().toString());
+ }
+ }
+ }
+
+ @Override
+ public void handleTemplate(String target, HttpServletRequest request,
+ HttpServletResponse response) throws IOException, ServletException {
+ Switchboard sb = Switchboard.getSwitchboard();
+
+ String localeSelection = Switchboard.getSwitchboard().getConfig("locale.language", "default");
+ File targetFile = getLocalizedFile(target, localeSelection);
+ File targetClass = rewriteClassFile(_resourceBase.addPath(target).getFile());
+ String targetExt = target.substring(target.lastIndexOf('.') + 1, target.length());
+
+ if ((targetClass != null)) {
+ serverObjects args = new serverObjects();
+ @SuppressWarnings("unchecked")
+ Enumeration<String> argNames = request.getParameterNames();
+ while (argNames.hasMoreElements()) {
+ String argName = argNames.nextElement();
+ args.put(argName, request.getParameter(argName));
+ }
+ //TODO: for SSI request, local parameters are added as attributes, put them back as parameter for the legacy request
+ // likely this should be implemented via httpservletrequestwrapper to supply complete parameters
+ @SuppressWarnings("unchecked")
+ Enumeration<String> attNames = request.getAttributeNames();
+ while (attNames.hasMoreElements()) {
+ String argName = attNames.nextElement();
+ args.put(argName, request.getAttribute(argName).toString());
+ }
+
+ // add multipart-form fields to parameter
+ if (request.getContentType() != null && request.getContentType().startsWith("multipart/form-data")) {
+ parseMultipart(request, args);
+ }
+ // eof modification to read attribute
+ RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt);
+
+ Object tmp;
+ try {
+ tmp = invokeServlet(targetClass, legacyRequestHeader, args);
+ } catch (InvocationTargetException e) {
+ ConcurrentLog.logException(e);
+ throw new ServletException();
+ } catch (IllegalArgumentException e) {
+ ConcurrentLog.logException(e);
+ throw new ServletException();
+ } catch (IllegalAccessException e) {
+ ConcurrentLog.logException(e);
+ throw new ServletException();
+ }
+
+ if (tmp instanceof RasterPlotter || tmp instanceof EncodedImage || tmp instanceof Image) {
+
+ net.yacy.cora.util.ByteBuffer result = null;
+
+ if (tmp instanceof RasterPlotter) {
+ final RasterPlotter yp = (RasterPlotter) tmp;
+ // send an image to client
+ result = RasterPlotter.exportImage(yp.getImage(), "png");
+ }
+ if (tmp instanceof EncodedImage) {
+ final EncodedImage yp = (EncodedImage) tmp;
+ result = yp.getImage();
+ }
+
+ if (tmp instanceof Image) {
+ final Image i = (Image) tmp;
+
+ // generate an byte array from the generated image
+ int width = i.getWidth(null);
+ if (width < 0) {
+ width = 96; // bad hack
+ }
+ int height = i.getHeight(null);
+ if (height < 0) {
+ height = 96; // bad hack
+ }
+ final BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
+ bi.createGraphics().drawImage(i, 0, 0, width, height, null);
+ result = RasterPlotter.exportImage(bi, targetExt);
+ }
+
+ final String mimeType = Classification.ext2mime(targetExt, "text/html");
+ response.setContentType(mimeType);
+ response.setContentLength(result.length());
+ response.setStatus(HttpServletResponse.SC_OK);
+
+ result.writeTo(response.getOutputStream());
+
+ return;
+ }
+
+ servletProperties templatePatterns = null;
+ if (tmp == null) {
+ // if no args given, then tp will be an empty Hashtable object (not null)
+ templatePatterns = new servletProperties();
+ } else if (tmp instanceof servletProperties) {
+ templatePatterns = (servletProperties) tmp;
+ } else {
+ templatePatterns = new servletProperties((serverObjects) tmp);
+ }
+ // add the application version, the uptime and the client name to every rewrite table
+ templatePatterns.put(servletProperties.PEER_STAT_VERSION, yacyBuildProperties.getVersion());
+ templatePatterns.put(servletProperties.PEER_STAT_UPTIME, ((System.currentTimeMillis() - serverCore.startupTime) / 1000) / 60); // uptime in minutes
+ templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTNAME, sb.peers.mySeed().getName());
+ templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTID, sb.peers.myID());
+ templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format());
+ Seed myPeer = sb.peers.mySeed();
+ templatePatterns.put("newpeer", myPeer.getAge() >= 1 ? 0 : 1);
+ templatePatterns.putHTML("newpeer_peerhash", myPeer.hash);
+ templatePatterns.put("p2p", sb.getConfigBool(SwitchboardConstants.DHT_ENABLED, true) || !sb.isRobinsonMode() ? 1 : 0);
+
+ if (targetFile.exists() && targetFile.isFile() && targetFile.canRead()) {
+ String mimeType = Classification.ext2mime(targetExt, "text/html");
+
+ InputStream fis = null;
+ long fileSize = targetFile.length();
+
+ if (fileSize <= Math.min(4 * 1024 * 1204, MemoryControl.available() / 100)) {
+ // read file completely into ram, avoid that too many files are open at the same time
+ fis = new ByteArrayInputStream(FileUtils.read(targetFile));
+ } else {
+ fis = new BufferedInputStream(new FileInputStream(targetFile));
+ }
+
+ // set response header
+ response.setContentType(mimeType);
+ response.setStatus(HttpServletResponse.SC_OK);
+ ByteArrayOutputStream bas = new ByteArrayOutputStream(4096);
+ // apply templates
+ TemplateEngine.writeTemplate(fis, bas, templatePatterns, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
+ fis.close();
+ // handle SSI
+ doContentMod (bas.toByteArray(),request,response);
+ }
+ }
+ }
+
+ protected void doContentMod(final byte[] in, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
+ net.yacy.cora.util.ByteBuffer buffer = new net.yacy.cora.util.ByteBuffer(in);
+ OutputStream out = response.getOutputStream();
+
+
+ // remove virtual host "currentyacypeer"
+ int off = 0; // starting offset
+ int x = buffer.indexOf("/currentyacypeer/".getBytes(), off);
+ while (x >= 0) {
+ for (int i = 0; i < 16; i++) {
+ in[x + i] = 32;
+ }
+ off = x + 16;
+ x = buffer.indexOf("/currentyacypeer/".getBytes(), off);
+ }
+
+ // check and handle SSI (ServerSideIncludes)
+ off = 0;
+ int p = buffer.indexOf("<!--#".getBytes(), off);
+ int q;
+ while (p >= 0) {
+ q = buffer.indexOf("-->".getBytes(), p + 10);
+
+ out.write(in, off, p - off);
+ out.flush();
+ parseSSI(buffer, p, request, response);
+ off = q + 3;
+ p = buffer.indexOf("<!--#".getBytes(), off);
+ }
+ out.write(in, off, in.length - off);
+ out.flush();
+ }
+
+ // parse SSI line and include resource
+ @Override
+ protected void parseSSI(final net.yacy.cora.util.ByteBuffer in, final int off, HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
+ if (in.startsWith("<!--#include virtual=\"".getBytes(), off)) {
+ final int q = in.indexOf("\"".getBytes(), off + 22);
+ if (q > 0) {
+ final String path = in.toString(off + 22, q - off - 22);
+ try {
+ RequestDispatcher dispatcher = request.getRequestDispatcher("/" + path);
+ dispatcher.include(request, response);
+ response.flushBuffer();
+ } catch (Exception e) {
+ ConcurrentLog.logException(e);
+ throw new ServletException();
+ }
+ }
+ }
+ }
+ /**
+ * TODO: add same functionality & checks as in HTTPDemon.parseMultipart
+ *
+ * parse multi-part form data for formfields (only), see also original
+ * implementation in HTTPDemon.parseMultipart
+ *
+ * @param request
+ * @param args found fields/values are added to the map
+ */
+ public void parseMultipart(HttpServletRequest request, serverObjects args) {
+ DiskFileItemFactory factory = new DiskFileItemFactory();
+ // maximum size that will be stored in memory
+ factory.setSizeThreshold(4096 * 16);
+ // Location to save data that is larger than maxMemSize.
+ // factory.setRepository(new File("."));
+ // Create a new file upload handler
+ ServletFileUpload upload = new ServletFileUpload(factory);
+ upload.setSizeMax(4096 * 16);
+ try {
+ // Parse the request to get form field items
+ @SuppressWarnings("unchecked")
+ List<FileItem> fileItems = upload.parseRequest(request);
+ // Process the uploaded file items
+ Iterator<FileItem> i = fileItems.iterator();
+ while (i.hasNext()) {
+ FileItem fi = i.next();
+ if (fi.isFormField()) {
+ args.put(fi.getFieldName(), fi.getString());
+ }
+ }
+ } catch (Exception ex) {
+ ConcurrentLog.logException(ex);
+ }
+ }
+}
\ No newline at end of file diff --git a/test/net/yacy/http/Jetty9YaCySecurityHandler.java b/test/net/yacy/http/Jetty9YaCySecurityHandler.java new file mode 100644 index 000000000..f40250c7f --- /dev/null +++ b/test/net/yacy/http/Jetty9YaCySecurityHandler.java @@ -0,0 +1,174 @@ +// +// YaCySecurityHandler +// Copyright 2011 by Florian Richter +// First released 16.04.2011 at http://yacy.net +// +// $LastChangedDate$ +// $LastChangedRevision$ +// $LastChangedBy$ +// +// This library is free software; you can redistribute it and/or +// modify it under the terms of the GNU Lesser General Public +// License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// +// This library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with this program in the file lgpl21.txt +// If not, see <http://www.gnu.org/licenses/>. +// + +package net.yacy.http; + +import java.io.IOException; +import java.net.MalformedURLException; +import net.yacy.cora.document.id.MultiProtocolURL; + +import net.yacy.cora.protocol.Domains; +import net.yacy.search.Switchboard; +import org.eclipse.jetty.http.HttpStatus; +import org.eclipse.jetty.security.RoleInfo; + +import org.eclipse.jetty.security.SecurityHandler; +import org.eclipse.jetty.security.UserDataConstraint; +import org.eclipse.jetty.server.HttpChannel; +import org.eclipse.jetty.server.HttpConfiguration; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.Response; +import org.eclipse.jetty.server.UserIdentity; + +/** + * jetty security handler + * demands authentication for pages with _p. inside + * and updates AccessTracker + */ +public class Jetty9YaCySecurityHandler extends SecurityHandler { + + @Override + protected boolean checkUserDataPermissions(String pathInContext, Request request, Response response, RoleInfo roleInfo) throws IOException + // check the SecurityHandler code, denying here does not provide authentication + // - identical with ConstraintSecurityHandler.checkUserDataPermissions implementation of Jetty source distribution + { + if (roleInfo == null) { + return true; + } + + if (roleInfo.isForbidden()) { + return false; + } + + UserDataConstraint dataConstraint = roleInfo.getUserDataConstraint(); + if (dataConstraint == null || dataConstraint == UserDataConstraint.None) { + return true; + } + + HttpConfiguration httpConfig = HttpChannel.getCurrentHttpChannel().getHttpConfiguration(); + + if (dataConstraint == UserDataConstraint.Confidential || dataConstraint == UserDataConstraint.Integral) { + if (request.isSecure()) { + return true; + } + + if (httpConfig.getSecurePort() > 0) { + String scheme = httpConfig.getSecureScheme(); + int port = httpConfig.getSecurePort(); + String url = ("https".equalsIgnoreCase(scheme) && port == 443) + ? "https://" + request.getServerName() + request.getRequestURI() + : scheme + "://" + request.getServerName() + ":" + port + request.getRequestURI(); + if (request.getQueryString() != null) { + url += "?" + request.getQueryString(); + } + response.setContentLength(0); + response.sendRedirect(url); + } else { + response.sendError(HttpStatus.FORBIDDEN_403, "!Secure"); + } + + request.setHandled(true); + return false; + } else { + throw new IllegalArgumentException("Invalid dataConstraint value: " + dataConstraint); + } + } + + @Override + protected boolean checkWebResourcePermissions(String pathInContext, Request request, + Response response, Object constraintInfo, UserIdentity userIdentity) throws IOException { + // deny and request for authentication, if necessary + // - identical with ConstraintSecurityHandler.checkWebResourcePermissions implementation of Jetty source distribution + if (constraintInfo == null) { + return true; + } + RoleInfo roleInfo = (RoleInfo) constraintInfo; + + if (!roleInfo.isChecked()) { + return true; + } + + if (roleInfo.isAnyRole() && request.getAuthType() != null) { + return true; + } + + for (String role : roleInfo.getRoles()) { + if (userIdentity.isUserInRole(role, null)) { + return true; + } + } + return false; + } + + @Override + protected boolean isAuthMandatory(Request baseRequest, Response base_response, Object constraintInfo) { + // identical with ConstraintSecurityHandler.isAuthMandatory implementation of Jetty source distribution + return constraintInfo != null && ((RoleInfo) constraintInfo).isChecked(); + } + + /** + * create the constraint for the given path + * for urls containing *_p. (like info_p.html) admin access is required, + * on localhost = admin setting no constraint is set + * @param pathInContext + * @param request + * @return RoleInfo with + * isChecked=true if any security contraint applies (compare reference implementation org.eclipse.jetty.security.ConstraintSecurityHandler) + * role = "admin" for resource name containint _p. + */ + @Override + protected RoleInfo prepareConstraintInfo(String pathInContext, Request request) { + final Switchboard sb = Switchboard.getSwitchboard(); + final boolean adminAccountForLocalhost = sb.getConfigBool("adminAccountForLocalhost", false); + //final String adminAccountBase64MD5 = sb.getConfig(YaCyLegacyCredential.ADMIN_ACCOUNT_B64MD5, ""); + + String refererHost; + // update AccessTracker + refererHost = request.getRemoteAddr(); + sb.track(refererHost, pathInContext); + + try { + refererHost = new MultiProtocolURL(request.getHeader("Referer")).getHost(); + } catch (MalformedURLException e) { + refererHost = null; + } + final boolean accessFromLocalhost = Domains.isLocalhost(request.getRemoteHost()) && (refererHost == null || refererHost.length() == 0 || Domains.isLocalhost(refererHost)); + // ! note : accessFromLocalhost compares localhost ip pattern ( ! currently also any intranet host is a local host) + final boolean grantedForLocalhost = adminAccountForLocalhost && accessFromLocalhost; + final boolean protectedPage = pathInContext.indexOf("_p.") > 0; + //final boolean accountEmpty = adminAccountBase64MD5.length() == 0; + //final boolean yacyBot = request.getHeader("User-Agent").startsWith("yacybot"); + + if (protectedPage) { // TODO: none public site + if (!grantedForLocalhost) { + RoleInfo roleinfo = new RoleInfo(); + roleinfo.setChecked(true); // RoleInfo.setChecked() : in Jetty this means - marked to have any security constraint + roleinfo.addRole("admin"); //YaCyLoginService assigns "admin" role to admin user + return roleinfo; + } // can omit else, as if grantedForLocalhost==true no constraint applies + // TODO: is this correct or adminAccountBase64MD5 not empty check neccessary ? + } + return null; + } +} diff --git a/test/net/yacy/http/Jetty9_Implementation_Test.java b/test/net/yacy/http/Jetty9_Implementation_Test.java new file mode 100644 index 000000000..ea58fae2d --- /dev/null +++ b/test/net/yacy/http/Jetty9_Implementation_Test.java @@ -0,0 +1,34 @@ +package net.yacy.http;
+
+
+import static org.junit.Assert.*;
+import org.junit.Test;
+
+/**
+ *
+ * @author bubu
+ */
+public class Jetty9_Implementation_Test {
+
+ public Jetty9_Implementation_Test() {
+ }
+
+ /**
+ * Test of main method, of class yacy.
+ */
+ @Test
+ public void testYaCyMain() {
+ System.out.println("* * * * * * * * * * * * * * * * * * * * *");
+ System.err.println("* This is a Implementation test *");
+ System.out.println("* for Jetty 9 (not a normal test case) *");
+ System.out.println("* Shutdown YaCy to continue *");
+ System.out.println("* * * * * * * * * * * * * * * * * * * * *");
+
+ String[] args = new String[]{};
+ YacyMain.main(args);
+
+ assertTrue(true);
+
+ }
+
+}
\ No newline at end of file diff --git a/test/net/yacy/http/YacyMain.java b/test/net/yacy/http/YacyMain.java new file mode 100644 index 000000000..56659b28d --- /dev/null +++ b/test/net/yacy/http/YacyMain.java @@ -0,0 +1,702 @@ +// yacy.java
+// -----------------------
+// (C) by Michael Peter Christen; mc@yacy.net
+// first published on http://www.yacy.net
+// Frankfurt, Germany, 2004, 2005
+//
+// $LastChangedDate$
+// $LastChangedRevision$
+// $LastChangedBy$
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package net.yacy.http;
+
+import net.yacy.*;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.io.RandomAccessFile;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.util.Properties;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.ExecutionException;
+
+import net.yacy.cora.date.GenericFormatter;
+import net.yacy.cora.lod.JenaTripleStore;
+import net.yacy.cora.protocol.ClientIdentification;
+import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.protocol.TimeoutRequest;
+import net.yacy.cora.protocol.http.HTTPClient;
+import net.yacy.cora.sorting.Array;
+import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.data.Translator;
+import net.yacy.gui.YaCyApp;
+import net.yacy.gui.framework.Browser;
+import net.yacy.http.YaCyHttpServer;
+import net.yacy.kelondro.util.FileUtils;
+import net.yacy.kelondro.util.Formatter;
+import net.yacy.kelondro.util.MemoryControl;
+import net.yacy.kelondro.util.OS;
+import net.yacy.peers.operation.yacyBuildProperties;
+import net.yacy.peers.operation.yacyRelease;
+import net.yacy.peers.operation.yacyVersion;
+import net.yacy.search.Switchboard;
+import net.yacy.search.SwitchboardConstants;
+
+import com.google.common.io.Files;
+import net.yacy.http.Jetty9HttpServerImpl;
+import net.yacy.migration;
+
+import net.yacy.server.serverCore;
+
+
+
+/**
+* This is the main class of YaCy. Several threads are started from here:
+* <ul>
+* <li>one single instance of the plasmaSwitchboard is generated, which itself
+* starts a thread with a plasmaHTMLCache object. This object simply counts
+* files sizes in the cache and terminates them. It also generates a
+* plasmaCrawlerLoader object, which may itself start some more httpc-calling
+* threads to load web pages. They terminate automatically when a page has
+* loaded.
+* <li>one serverCore - thread is started, which implements a multi-threaded
+* server. The process may start itself many more processes that handle
+* connections.lo
+* <li>finally, all idle-dependent processes are written in a queue in
+* plasmaSwitchboard which are worked off inside an idle-sensitive loop of the
+* main process. (here)
+* </ul>
+*
+* On termination, the following must be done:
+* <ul>
+* <li>stop feeding of the crawling process because it otherwise fills the
+* indexing queue.
+* <li>say goodbye to connected peers and disable new connections. Don't wait for
+* success.
+* <li>first terminate the serverCore thread. This prevents that new cache
+* objects are queued.
+* <li>wait that the plasmaHTMLCache terminates (it should be normal that this
+* process already has terminated).
+* <li>then wait for termination of all loader process of the
+* plasmaCrawlerLoader.
+* <li>work off the indexing and cache storage queue. These values are inside a
+* RAM cache and would be lost otherwise.
+* <li>write all settings.
+* <li>terminate.
+* </ul>
+*/
+
+public final class YacyMain {
+
+ // static objects
+ public static final String vString = yacyBuildProperties.getVersion();
+ public static float version = 0.1f;
+
+ public static final String vDATE = yacyBuildProperties.getBuildDate();
+ public static final String copyright = "[ YaCy v" + vString + ", build " + vDATE + " by Michael Christen / www.yacy.net ]";
+ public static final String hline = "-------------------------------------------------------------------------------";
+ public static final Semaphore shutdownSemaphore = new Semaphore(0);
+
+ /**
+ * a reference to the {@link Switchboard} created by the
+ * {@link yacy#startup(String, long, long)} method.
+ */
+ private static Switchboard sb = null;
+ public static String homedir;
+ public static File dataHome_g;
+
+ /**
+ * Starts up the whole application. Sets up all datastructures and starts
+ * the main threads.
+ *
+ * @param homePath Root-path where all information is to be found.
+ * @param startupFree free memory at startup time, to be used later for statistics
+ */
+ private static void startup(final File dataHome, final File appHome, final long startupMemFree, final long startupMemTotal, final boolean gui) {
+ try {
+ // start up
+ System.out.println(copyright);
+ System.out.println(hline);
+
+ // check java version
+ try {
+ "a".isEmpty(); // needs at least Java 1.6
+ } catch (final NoSuchMethodError e) {
+ System.err.println("STARTUP: Java Version too low. You need at least Java 1.6 to run YaCy");
+ System.exit(-1);
+ }
+
+ // ensure that there is a DATA directory, if not, create one and if that fails warn and die
+ mkdirsIfNeseccary(dataHome);
+ dataHome_g = dataHome;
+ mkdirsIfNeseccary(appHome);
+ File f = new File(dataHome, "DATA/");
+ mkdirsIfNeseccary(f);
+ if (!(f.exists())) {
+ System.err.println("Error creating DATA-directory in " + dataHome.toString() + " . Please check your write-permission for this folder. YaCy will now terminate.");
+ System.exit(-1);
+ }
+
+ homedir = appHome.toString();
+
+ // setting up logging
+ f = new File(dataHome, "DATA/LOG/");
+ mkdirsIfNeseccary(f);
+ f = new File(dataHome, "DATA/LOG/yacy.logging");
+ final File f0 = new File(appHome, "defaults/yacy.logging");
+ if (!f.exists() || f0.lastModified() > f.lastModified()) try {
+ Files.copy(f0, f);
+ } catch (final IOException e){
+ System.out.println("could not copy yacy.logging");
+ }
+ try{
+ ConcurrentLog.configureLogging(dataHome, appHome, new File(dataHome, "DATA/LOG/yacy.logging"));
+ } catch (final IOException e) {
+ System.out.println("could not find logging properties in homePath=" + dataHome);
+ ConcurrentLog.logException(e);
+ }
+ ConcurrentLog.config("STARTUP", "YaCy version: " + yacyBuildProperties.getVersion() + "/" + yacyBuildProperties.getSVNRevision());
+ ConcurrentLog.config("STARTUP", "Java version: " + System.getProperty("java.version", "no-java-version"));
+ ConcurrentLog.config("STARTUP", "Operation system: " + System.getProperty("os.name","unknown"));
+ ConcurrentLog.config("STARTUP", "Application root-path: " + appHome);
+ ConcurrentLog.config("STARTUP", "Data root-path: " + dataHome);
+ ConcurrentLog.config("STARTUP", "Time zone: UTC" + GenericFormatter.UTCDiffString() + "; UTC+0000 is " + System.currentTimeMillis());
+ ConcurrentLog.config("STARTUP", "Maximum file system path length: " + OS.maxPathLength);
+
+ f = new File(dataHome, "DATA/yacy.running");
+ final String conf = "DATA/SETTINGS/yacy.conf".replace("/", File.separator);
+ if (f.exists()) { // another instance running? VM crash? User will have to care about this
+ ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " exists, this usually means that a YaCy instance is still running. If you want to restart YaCy, try first ./stopYACY.sh, then ./startYACY.sh. If ./stopYACY.sh fails, try ./killYACY.sh");
+
+ // If YaCy is actually running, then we check if the server port is open.
+ // If yes, then we consider that a restart is a user mistake and then we just respond
+ // as the user expects and tell the browser to open the start page.
+ // That will especially happen if Windows Users double-Click the YaCy Icon on the desktop to simply
+ // open the web interface. (They don't think of 'servers' they just want to get to the search page).
+ // We need to parse the configuration file for that to get the host port
+ File dataFile = new File(dataHome, conf);
+ if (dataFile.exists()) {
+ Properties p = new Properties();
+ p.load(new FileInputStream(dataFile));
+ int port = Integer.parseInt(p.getProperty("port", "8090"));
+ try {
+ if (TimeoutRequest.ping("127.0.0.1", port, 1000)) {
+ Browser.openBrowser("http://localhost:" + port + "/" + p.getProperty(SwitchboardConstants.BROWSER_POP_UP_PAGE, "index.html"));
+ // Thats it; YaCy was running, the user is happy, we can stop now.
+ ConcurrentLog.severe("STARTUP", "WARNING: YaCy instance was still running; just opening the browser and exit.");
+ System.exit(0);
+ }
+ } catch (final ExecutionException ex) {
+ ConcurrentLog.info("STARTUP", "INFO: delete old yacy.running file; likely previous YaCy session was not orderly shutdown!");
+ }
+ }
+
+ // YaCy is not running; thus delete the file an go on as nothing was wrong.
+ delete(f);
+ }
+ if (!f.createNewFile()) ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be created!");
+ try { new FileOutputStream(f).write(Integer.toString(OS.getPID()).getBytes()); } catch (final Exception e) { } // write PID
+ f.deleteOnExit();
+ FileChannel channel = null;
+ FileLock lock = null;
+ try {
+ channel = new RandomAccessFile(f,"rw").getChannel();
+ lock = channel.tryLock(); // lock yacy.running
+ } catch (final Exception e) { }
+
+ try {
+ sb = new Switchboard(dataHome, appHome, "defaults/yacy.init".replace("/", File.separator), conf);
+ } catch (final RuntimeException e) {
+ ConcurrentLog.severe("STARTUP", "YaCy cannot start: " + e.getMessage(), e);
+ System.exit(-1);
+ }
+ //sbSync.V(); // signal that the sb reference was set
+
+ // switch the memory strategy
+ MemoryControl.setStandardStrategy(sb.getConfigBool("memory.standardStrategy", true));
+
+ // save information about available memory at startup time
+ sb.setConfig("memoryFreeAfterStartup", startupMemFree);
+ sb.setConfig("memoryTotalAfterStartup", startupMemTotal);
+
+ // start gui if wanted
+ if (gui) YaCyApp.start("localhost", (int) sb.getConfigLong("port", 8090));
+
+ // hardcoded, forced, temporary value-migration
+ sb.setConfig("htTemplatePath", "htroot/env/templates");
+
+ int oldRev;
+ try {
+ oldRev = Integer.parseInt(sb.getConfig("svnRevision", "0"));
+ } catch (final NumberFormatException e) {
+ oldRev = 0;
+ }
+ final int newRev = Integer.parseInt(yacyBuildProperties.getSVNRevision());
+ sb.setConfig("svnRevision", yacyBuildProperties.getSVNRevision());
+ sb.setConfig("applicationRoot", appHome.toString());
+ sb.setConfig("dataRoot", dataHome.toString());
+ yacyVersion.latestRelease = version;
+
+ // read environment
+ final int timeout = Math.max(5000, Integer.parseInt(sb.getConfig("httpdTimeout", "5000")));
+
+ // create some directories
+ final File htRootPath = new File(appHome, sb.getConfig("htRootPath", "htroot"));
+ final File htDocsPath = sb.getDataPath(SwitchboardConstants.HTDOCS_PATH, SwitchboardConstants.HTDOCS_PATH_DEFAULT);
+ mkdirIfNeseccary(htDocsPath);
+ //final File htTemplatePath = new File(homePath, sb.getConfig("htTemplatePath","htdocs"));
+
+ // create default notifier picture
+ //TODO: Use templates instead of copying images ...
+ if (!((new File(htDocsPath, "notifier.gif")).exists())) try {
+ Files.copy(new File(htRootPath, "env/grafics/empty.gif"),
+ new File(htDocsPath, "notifier.gif"));
+ } catch (final IOException e) {}
+
+ final File htdocsReadme = new File(htDocsPath, "readme.txt");
+ if (!(htdocsReadme.exists())) try {FileUtils.copy((
+ "This is your root directory for individual Web Content\r\n" +
+ "\r\n" +
+ "Please place your html files into the www subdirectory.\r\n" +
+ "The URL of that path is either\r\n" +
+ "http://www.<your-peer-name>.yacy or\r\n" +
+ "http://<your-ip>:<your-port>/www\r\n" +
+ "\r\n" +
+ "Other subdirectories may be created; they map to corresponding sub-domains.\r\n" +
+ "This directory shares it's content with the applications htroot path, so you\r\n" +
+ "may access your yacy search page with\r\n" +
+ "http://<your-peer-name>.yacy/\r\n" +
+ "\r\n").getBytes(), htdocsReadme);} catch (final IOException e) {
+ System.out.println("Error creating htdocs readme: " + e.getMessage());
+ }
+
+ final File wwwDefaultPath = new File(htDocsPath, "www");
+ mkdirIfNeseccary(wwwDefaultPath);
+
+
+ final File shareDefaultPath = new File(htDocsPath, "share");
+ mkdirIfNeseccary(shareDefaultPath);
+
+ migration.migrate(sb, oldRev, newRev);
+
+ // delete old release files
+ final int deleteOldDownloadsAfterDays = (int) sb.getConfigLong("update.deleteOld", 30);
+ yacyRelease.deleteOldDownloads(sb.releasePath, deleteOldDownloadsAfterDays );
+
+ // set user-agent
+ HTTPClient.setDefaultUserAgent(ClientIdentification.yacyInternetCrawlerAgent.userAgent);
+
+ // initial fill of the triplestore
+ File triplestore = new File(sb.getConfig("triplestore", new File(dataHome, "DATA/TRIPLESTORE").getAbsolutePath()));
+ mkdirIfNeseccary(triplestore);
+ for (String s: triplestore.list()) {
+ if ((s.endsWith(".rdf") || s.endsWith(".nt")) && !s.equals("local.rdf") && !s.endsWith("_triplestore.rdf") && !s.startsWith("private_store_")) {
+ try {
+ JenaTripleStore.load(new File(triplestore, s).getAbsolutePath());
+ } catch (final IOException e) {
+ ConcurrentLog.logException(e);
+ }
+ }
+ }
+ if (sb.getConfigBool("triplestore.persistent", false)) {
+ File local = new File(triplestore, "local.rdf");
+ if (local.exists()) {
+ try {
+ JenaTripleStore.load(local.getAbsolutePath());
+ } catch (final IOException e) {
+ ConcurrentLog.logException(e);
+ }
+ }
+ }
+
+ // start main threads
+ final int port = sb.getConfigInt("port", 8090);
+ try {
+
+ // start jetty http server
+ YaCyHttpServer httpServer = new Jetty9HttpServerImpl(port);
+ httpServer.startupServer();
+ sb.setHttpServer(httpServer);
+ ConcurrentLog.info("STARTUP",httpServer.getVersion());
+ //final HTTPDemon protocolHandler = new HTTPDemon(sb);
+ //final serverCore server = new serverCore(
+ // timeout /*control socket timeout in milliseconds*/,
+ // true /* block attacks (wrong protocol) */,
+ // protocolHandler /*command class*/,
+ // sb,
+ // 30000 /*command max length incl. GET args*/);
+ //server.setName("httpd:"+port);
+ //server.setPriority(Thread.MAX_PRIORITY);
+ //server.setObeyIntermission(false);
+
+ // start the server
+ //sb.deployThread("10_httpd", "HTTPD Server/Proxy", "the HTTPD, used as web server and proxy", null, server, 0, 0, 0, 0);
+ //server.start();
+
+ // open the browser window
+ final boolean browserPopUpTrigger = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_TRIGGER, "true").equals("true");
+ if (browserPopUpTrigger) try {
+ final String browserPopUpPage = sb.getConfig(SwitchboardConstants.BROWSER_POP_UP_PAGE, "ConfigBasic.html");
+ //boolean properPW = (sb.getConfig("adminAccount", "").isEmpty()) && (sb.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0);
+ //if (!properPW) browserPopUpPage = "ConfigBasic.html";
+ Browser.openBrowser((false?"https":"http") + "://localhost:" + port + "/" + browserPopUpPage);
+ // Browser.openBrowser((server.withSSL()?"https":"http") + "://localhost:" + serverCore.getPortNr(port) + "/" + browserPopUpPage);
+ } catch (final Throwable e) {
+ // cannot open browser. This may be normal in headless environments
+ //Log.logException(e);
+ }
+
+ // enable browser popup, http server is ready now
+ sb.tray.setReady();
+
+ //regenerate Locales from Translationlist, if needed
+ final File locale_source = sb.getAppPath("locale.source", "locales");
+ final String lang = sb.getConfig("locale.language", "");
+ if (!lang.equals("") && !lang.equals("default")) { //locale is used
+ String currentRev = "";
+ try{
+ final BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang+"/version" ))));
+ currentRev = br.readLine();
+ br.close();
+ }catch(final IOException e){
+ //Error
+ }
+
+ if (!currentRev.equals(sb.getConfig("svnRevision", ""))) try { //is this another version?!
+ final File sourceDir = new File(sb.getConfig("htRootPath", "htroot"));
+ final File destDir = new File(sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot"), lang);
+ if (Translator.translateFilesRecursive(sourceDir, destDir, new File(locale_source, lang + ".lng"), "html,template,inc", "locale")){ //translate it
+ //write the new Versionnumber
+ final BufferedWriter bw = new BufferedWriter(new PrintWriter(new FileWriter(new File(destDir, "version"))));
+ bw.write(sb.getConfig("svnRevision", "Error getting Version"));
+ bw.close();
+ }
+ } catch (final IOException e) {}
+ }
+ // initialize number formatter with this locale
+ Formatter.setLocale(lang);
+
+ // registering shutdown hook
+ ConcurrentLog.config("STARTUP", "Registering Shutdown Hook");
+ final Runtime run = Runtime.getRuntime();
+ run.addShutdownHook(new shutdownHookThread(Thread.currentThread(), sb));
+
+ // save information about available memory after all initializations
+ //try {
+ sb.setConfig("memoryFreeAfterInitBGC", MemoryControl.free());
+ sb.setConfig("memoryTotalAfterInitBGC", MemoryControl.total());
+ System.gc();
+ sb.setConfig("memoryFreeAfterInitAGC", MemoryControl.free());
+ sb.setConfig("memoryTotalAfterInitAGC", MemoryControl.total());
+ //} catch (final ConcurrentModificationException e) {}
+
+ // wait for server shutdown
+ try {
+ sb.waitForShutdown();
+ } catch (final Exception e) {
+ ConcurrentLog.severe("MAIN CONTROL LOOP", "PANIC: " + e.getMessage(),e);
+ }
+ // shut down
+ Array.terminate();
+ ConcurrentLog.config("SHUTDOWN", "caught termination signal");
+ httpServer.stop();
+
+ ConcurrentLog.config("SHUTDOWN", "server has terminated");
+ sb.close();
+ } catch (final Exception e) {
+ ConcurrentLog.severe("STARTUP", "Unexpected Error: " + e.getClass().getName(),e);
+ //System.exit(1);
+ }
+ if(lock != null) lock.release();
+ if(channel != null) channel.close();
+ } catch (final Exception ee) {
+ ConcurrentLog.severe("STARTUP", "FATAL ERROR: " + ee.getMessage(),ee);
+ } finally {
+ }
+
+ // save the triple store
+ if (sb.getConfigBool("triplestore.persistent", false)) {
+ JenaTripleStore.saveAll();
+ }
+
+ ConcurrentLog.config("SHUTDOWN", "goodbye. (this is the last line)");
+ ConcurrentLog.shutdown();
+ shutdownSemaphore.release(1000);
+ try {
+ System.exit(0);
+ } catch (final Exception e) {} // was once stopped by de.anomic.net.ftpc$sm.checkExit(ftpc.java:1790)
+ }
+
+ /**
+ * @param f
+ */
+ private static void delete(final File f) {
+ if(!f.delete())
+ ConcurrentLog.severe("STARTUP", "WARNING: the file " + f + " can not be deleted!");
+ }
+
+ /**
+ * @see File#mkdir()
+ * @param path
+ */
+ private static void mkdirIfNeseccary(final File path) {
+ if (!(path.exists()))
+ if(!path.mkdir())
+ ConcurrentLog.warn("STARTUP", "could not create directory "+ path.toString());
+ }
+
+ /**
+ * @see File#mkdirs()
+ * @param path
+ */
+ public static void mkdirsIfNeseccary(final File path) {
+ if (!(path.exists()))
+ if(!path.mkdirs())
+ ConcurrentLog.warn("STARTUP", "could not create directories "+ path.toString());
+ }
+
+ /**
+ * Loads the configuration from the data-folder.
+ * FIXME: Why is this called over and over again from every method, instead
+ * of setting the configurationdata once for this class in main?
+ *
+ * @param mes Where are we called from, so that the errormessages can be
+ * more descriptive.
+ * @param homePath Root-path where all the information is to be found.
+ * @return Properties read from the configurationfile.
+ */
+ private static Properties configuration(final String mes, final File homePath) {
+ ConcurrentLog.config(mes, "Application Root Path: " + homePath.toString());
+
+ // read data folder
+ final File dataFolder = new File(homePath, "DATA");
+ if (!(dataFolder.exists())) {
+ ConcurrentLog.severe(mes, "Application was never started or root path wrong.");
+ System.exit(-1);
+ }
+
+ final Properties config = new Properties();
+ FileInputStream fis = null;
+ try {
+ fis = new FileInputStream(new File(homePath, "DATA/SETTINGS/yacy.conf"));
+ config.load(fis);
+ } catch (final FileNotFoundException e) {
+ ConcurrentLog.severe(mes, "could not find configuration file.");
+ System.exit(-1);
+ } catch (final IOException e) {
+ ConcurrentLog.severe(mes, "could not read configuration file.");
+ System.exit(-1);
+ } finally {
+ if(fis != null) {
+ try {
+ fis.close();
+ } catch (final IOException e) {
+ ConcurrentLog.logException(e);
+ }
+ }
+ }
+
+ return config;
+ }
+
+ /**
+ * Call the shutdown-page of YaCy to tell it to shut down. This method is
+ * called if you start yacy with the argument -shutdown.
+ *
+ * @param homePath Root-path where all the information is to be found.
+ */
+ public static void shutdown(final File homePath) {
+ // start up
+ System.out.println(copyright);
+ System.out.println(hline);
+ submitURL(homePath, "Steering.html?shutdown=", "Terminate YaCy");
+ }
+
+ public static void update(final File homePath) {
+ // start up
+ System.out.println(copyright);
+ System.out.println(hline);
+ submitURL(homePath, "ConfigUpdate_p.html?autoUpdate=", "Update YaCy to most recent version");
+ }
+
+ private static void submitURL(final File homePath, final String path, final String processdescription) {
+ final Properties config = configuration("COMMAND-STEERING", homePath);
+
+ // read port
+ final int port = serverCore.getPortNr(config.getProperty("port", "8090"));
+
+ // read password
+ String encodedPassword = (String) config.get(SwitchboardConstants.ADMIN_ACCOUNT_B64MD5);
+ if (encodedPassword == null) encodedPassword = ""; // not defined
+
+ // send 'wget' to web interface
+ final RequestHeader requestHeader = new RequestHeader();
+ requestHeader.put(RequestHeader.AUTHORIZATION, "realm=" + encodedPassword); // for http-authentify
+// final Client con = new Client(10000, requestHeader);
+ final HTTPClient con = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent);
+ con.setHeader(requestHeader.entrySet());
+// ResponseContainer res = null;
+ try {
+// res = con.GET("http://localhost:"+ port +"/" + path);
+ con.GETbytes("http://localhost:"+ port +"/" + path);
+
+ // read response
+// if (res.getStatusLine().startsWith("2")) {
+ if (con.getStatusCode() > 199 && con.getStatusCode() < 300) {
+ ConcurrentLog.config("COMMAND-STEERING", "YACY accepted steering command: " + processdescription);
+// final ByteArrayOutputStream bos = new ByteArrayOutputStream(); //This is stream is not used???
+// try {
+// FileUtils.copyToStream(new BufferedInputStream(res.getDataAsStream()), new BufferedOutputStream(bos));
+// } finally {
+// res.closeStream();
+// }
+ } else {
+// Log.logSevere("COMMAND-STEERING", "error response from YACY socket: " + res.getStatusLine());
+ ConcurrentLog.severe("COMMAND-STEERING", "error response from YACY socket: " + con.getHttpResponse().getStatusLine());
+ System.exit(-1);
+ }
+ } catch (final IOException e) {
+ ConcurrentLog.severe("COMMAND-STEERING", "could not establish connection to YACY socket: " + e.getMessage());
+ System.exit(-1);
+// } finally {
+// // release connection
+// if(res != null) {
+// res.closeStream();
+// }
+ }
+
+ try {
+ HTTPClient.closeConnectionManager();
+ } catch (final InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ // finished
+ ConcurrentLog.config("COMMAND-STEERING", "SUCCESSFULLY FINISHED COMMAND: " + processdescription);
+ }
+
+ /**
+ * Main-method which is started by java. Checks for special arguments or
+ * starts up the application.
+ *
+ * @param args
+ * Given arguments from the command line.
+ */
+ public static void main(String args[]) {
+
+ try {
+
+ // check assertion status
+ //ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true);
+ boolean assertionenabled = false;
+ assert (assertionenabled = true) == true; // compare to true to remove warning: "Possible accidental assignement"
+ if (assertionenabled) System.out.println("Asserts are enabled");
+
+ // check memory amount
+ System.gc();
+ final long startupMemFree = MemoryControl.free();
+ final long startupMemTotal = MemoryControl.total();
+
+ // maybe go into headless awt mode: we have three cases depending on OS and one exception:
+ // windows : better do not go into headless mode
+ // mac : go into headless mode because an application is shown in gui which may not be wanted
+ // linux : go into headless mode because this does not need any head operation
+ // exception : if the -gui option is used then do not go into headless mode since that uses a gui
+ boolean headless = true;
+ if (OS.isWindows) headless = false;
+ if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) headless = false;
+ System.setProperty("java.awt.headless", headless ? "true" : "false");
+ // System.setProperty("java.net.preferIPv4Stack", "true"); // DO NOT PREFER IPv6, i.e. freifunk uses ipv6 only and host resolving does not work
+
+ String s = ""; for (final String a: args) s += a + " ";
+ yacyRelease.startParameter = s.trim();
+
+ File applicationRoot = new File(System.getProperty("user.dir").replace('\\', '/'));
+ File dataRoot = applicationRoot;
+ //System.out.println("args.length=" + args.length);
+ //System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]");
+ if ((args.length >= 1) && (args[0].toLowerCase().equals("-startup") || args[0].equals("-start"))) {
+ // normal start-up of yacy
+ if (args.length > 1) dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
+ startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
+ } else if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) {
+ // start-up of yacy with gui
+ if (args.length > 1) dataRoot = new File(System.getProperty("user.home").replace('\\', '/'), args[1]);
+ startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, true);
+ } else if ((args.length >= 1) && ((args[0].toLowerCase().equals("-shutdown")) || (args[0].equals("-stop")))) {
+ // normal shutdown of yacy
+ if (args.length == 2) applicationRoot= new File(args[1]);
+ shutdown(applicationRoot);
+ } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-update"))) {
+ // aut-update yacy
+ if (args.length == 2) applicationRoot= new File(args[1]);
+ update(applicationRoot);
+ } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-version"))) {
+ // show yacy version
+ System.out.println(copyright);
+ } else {
+ if (args.length == 1) applicationRoot= new File(args[0]);
+ startup(dataRoot, applicationRoot, startupMemFree, startupMemTotal, false);
+ }
+ } finally {
+ ConcurrentLog.shutdown();
+ }
+ }
+}
+/**
+* This class is a helper class whose instance is started, when the java virtual
+* machine shuts down. Signals the plasmaSwitchboard to shut down.
+*/
+class shutdownHookThread extends Thread {
+ private final Switchboard sb;
+ private final Thread mainThread;
+
+ public shutdownHookThread(final Thread mainThread, final Switchboard sb) {
+ super();
+ this.sb = sb;
+ this.mainThread = mainThread;
+ }
+
+ @Override
+ public void run() {
+ try {
+ if (!this.sb.isTerminated()) {
+ ConcurrentLog.config("SHUTDOWN","Shutdown via shutdown hook.");
+
+ // sending the yacy main thread a shutdown signal
+ ConcurrentLog.fine("SHUTDOWN","Signaling shutdown to the switchboard.");
+ this.sb.terminate("shutdown hook");
+
+ // waiting for the yacy thread to finish execution
+ ConcurrentLog.fine("SHUTDOWN","Waiting for main thread to finish.");
+ if (this.mainThread.isAlive() && !this.sb.isTerminated()) {
+ this.mainThread.join();
+ }
+ }
+ } catch (final Exception e) {
+ ConcurrentLog.severe("SHUTDOWN","Unexpected error. " + e.getClass().getName(),e);
+ }
+ }
+}
|