move testing SolrServlet.main to test, making include of jetty*.jar in distribution and classpath obsolete

- move jetty*.jar to test library 
- move SolrServlet.main as is to test, add also a junit test simulating main 
  - add build.xml cleanup for EmbeddedSolrConnectorTest created test/DATA
- adjust some test compile errors

12 files changed, 200 insertions, 89 deletions

diff --git a/addon/YaCy.app/Contents/Info.plist b/addon/YaCy.app/Contents/Info.plist
index 159b876e9..e7d3ba5e3 100644
--- a/addon/YaCy.app/Contents/Info.plist
+++ b/addon/YaCy.app/Contents/Info.plist
@@ -68,8 +68,6 @@
 			<string>$JAVAROOT/lib/jcl-over-slf4j-1.7.2.jar</string>
 			<string>$JAVAROOT/lib/jempbox-1.7.1.jar</string>
 			<string>$JAVAROOT/lib/jena-2.6.4.jar</string>
-			<string>$JAVAROOT/lib/jetty-6.1.26-patched-JETTY-1340.jar</string>
-			<string>$JAVAROOT/lib/jetty-util-6.1.26-patched-JETTY-1340.jar</string>
 			<string>$JAVAROOT/lib/jsch-0.1.42.jar</string>
 			<string>$JAVAROOT/lib/json-simple-1.1.jar</string>
 			<string>$JAVAROOT/lib/log4j-1.2.17.jar</string>
diff --git a/build.xml b/build.xml
index ce7df8ccf..fbab83bb0 100644
--- a/build.xml
+++ b/build.xml
@@ -187,8 +187,6 @@
     	<pathelement location="${lib}/jcl-over-slf4j-1.7.2.jar" />
     	<pathelement location="${lib}/jempbox-1.7.1" />
     	<pathelement location="${lib}/jena-2.6.4.jar" />
-    	<pathelement location="${lib}/jetty-6.1.26-patched-JETTY-1340.jar" />
-    	<pathelement location="${lib}/jetty-util-6.1.26-patched-JETTY-1340.jar" />
     	<pathelement location="${lib}/jsch-0.1.42.jar" />
     	<pathelement location="${lib}/json-simple-1.1.jar" />
     	<pathelement location="${lib}/jsoup-1.6.3.jar" />
@@ -433,7 +431,7 @@
       <classpath>
         <pathelement location="${build}"/>
         <pathelement location="${htroot}"/>
-	<pathelement location="${libt}/junit-4.7.jar" />
+        <fileset dir="${libt}" includes="**/*.jar" />
         <fileset dir="${lib}" includes="**/*.jar" />
       </classpath>
       <compilerarg value="-Xlint"/>
@@ -441,7 +439,7 @@
   </target>
 
   <target name="test" depends="compileTest" description="run some unittests">
-    <junit printsummary="yes" haltonfailure="no">
+    <junit printsummary="yes" haltonfailure="no" fork="true" dir="${yacyroot}">
       <formatter type="plain"/>
       <batchtest>
         <fileset dir="${test}">
@@ -452,7 +450,7 @@
         <pathelement location="${test}"/>
         <pathelement location="${build}"/>
         <pathelement location="${htroot}"/>
-	<pathelement location="${libt}/junit-4.7.jar" />
+	<fileset dir="${libt}" includes="**/*.jar" />
         <fileset dir="${lib}" includes="**/*.jar" />
       </classpath>
     </junit>
@@ -508,7 +506,7 @@
     <delete file="${lib}/yacy-cora.jar" failonerror="false"/>
     <delete file="${lib}/svnRevNr.jar" failonerror="false"/>
     <delete file="${libbuild}/svnRevNr.jar" failonerror="false"/>
-    <delete file="${libbuild}/GitRevTask.jar" failonerror="false"/>
+    <delete file="${libbuild}/GitRevTask.jar" failonerror="false"/>    
     <delete failonerror="false">
       <fileset dir="${src}" includes="**/*.class" />
       <fileset dir="${build}" includes="**/*.class" />
@@ -518,6 +516,7 @@
       <fileset dir="${libbuild}/GitRevTask" includes="**/*.class" />
       <fileset dir="." includes="TEST-*" />
     </delete>
+    <delete dir="test/DATA" failonerror="false"/>
   </target>
 
   <target name="installonlinux">
diff --git a/lib/jetty-6.1.26-patched-JETTY-1340.jar b/libt/jetty-6.1.26-patched-JETTY-1340.jar
index 6be492c92..6be492c92 100644
--- a/lib/jetty-6.1.26-patched-JETTY-1340.jar
+++ b/libt/jetty-6.1.26-patched-JETTY-1340.jar
diff --git a/lib/jetty-LICENSE-ASL.txt b/libt/jetty-LICENSE-ASL.txt
index d64569567..d64569567 100644
--- a/lib/jetty-LICENSE-ASL.txt
+++ b/libt/jetty-LICENSE-ASL.txt
diff --git a/lib/jetty-util-6.1.26-patched-JETTY-1340.jar b/libt/jetty-util-6.1.26-patched-JETTY-1340.jar
index 1a9ace88d..1a9ace88d 100644
--- a/lib/jetty-util-6.1.26-patched-JETTY-1340.jar
+++ b/libt/jetty-util-6.1.26-patched-JETTY-1340.jar
diff --git a/lib/jetty-util-LICENSE-ASL.txt b/libt/jetty-util-LICENSE-ASL.txt
index d64569567..d64569567 100644
--- a/lib/jetty-util-LICENSE-ASL.txt
+++ b/libt/jetty-util-LICENSE-ASL.txt
diff --git a/source/net/yacy/cora/federate/solr/SolrServlet.java b/source/net/yacy/cora/federate/solr/SolrServlet.java
index f6f661bfd..2bd75ad66 100644
--- a/source/net/yacy/cora/federate/solr/SolrServlet.java
+++ b/source/net/yacy/cora/federate/solr/SolrServlet.java
@@ -58,10 +58,6 @@ import org.apache.solr.search.SolrIndexSearcher;
 import org.apache.solr.servlet.cache.HttpCacheHeaderUtil;
 import org.apache.solr.servlet.cache.Method;
 import org.apache.solr.util.FastWriter;
-import org.mortbay.jetty.Handler;
-import org.mortbay.jetty.Server;
-import org.mortbay.jetty.servlet.Context;
-import org.mortbay.jetty.servlet.FilterHolder;
 
 
 public class SolrServlet implements Filter {
@@ -194,38 +190,6 @@ public class SolrServlet implements Filter {
         hresponse.sendError((code < 100) ? 500 : code, ex.getMessage() + "\n\n" + sw.toString());
     }
 
-    /**
-     * from org.apache.solr.client.solrj.embedded.JettySolrRunner
-     */
-    public static Server startServer(String context, int port, EmbeddedSolrConnector c) {
-        //this.context = context;
-        Server server = new Server(port);
-        /*
-            SocketConnector connector = new SocketConnector();
-            connector.setPort(port);
-            connector.setReuseAddress(true);
-            this.server.setConnectors(new Connector[] { connector });
-            this.server.setSessionIdManager(new HashSessionIdManager(new Random()));
-        */
-        server.setStopAtShutdown(true);
-        Context root = new Context(server, context, Context.SESSIONS);
-        root.addServlet(Servlet404.class, "/*");
-
-        // attach org.apache.solr.response.XMLWriter to search requests
-        SolrServlet.initCore(c);
-        FilterHolder dispatchFilter = root.addFilter(SolrServlet.class, "*", Handler.REQUEST);
-
-        if (!server.isRunning()) {
-            try {
-                server.start();
-                waitForSolr(context, port);
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
-        }
-        return server;
-    }
-
     public static void waitForSolr(String context, int port) throws Exception {
         // A raw term query type doesn't check the schema
         URL url = new URL("http://127.0.0.1:" + port + context + "/select?q={!raw+f=test_query}ping");
diff --git a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
index 6a204f4c5..fd3b65535 100644
--- a/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
+++ b/source/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnector.java
@@ -26,18 +26,13 @@ import java.io.IOException;
 
 import javax.xml.parsers.ParserConfigurationException;
 
-import net.yacy.cora.federate.solr.SolrServlet;
-import net.yacy.cora.federate.solr.YaCySchema;
 import net.yacy.kelondro.logging.Log;
 import net.yacy.kelondro.util.MemoryControl;
 
 import org.apache.solr.client.solrj.SolrServerException;
 import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer;
 import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
 import org.apache.solr.common.SolrException;
-import org.apache.solr.common.SolrInputDocument;
 import org.apache.solr.common.params.ModifiableSolrParams;
 import org.apache.solr.common.params.SolrParams;
 import org.apache.solr.common.util.NamedList;
@@ -207,37 +202,4 @@ public class EmbeddedSolrConnector extends SolrServerConnector implements SolrCo
         }
     }
 
-    public static void main(String[] args) {
-        File solr_config = new File("defaults/solr");
-        File storage = new File("DATA/INDEX/webportal/SEGMENTS/text/solr/");
-        storage.mkdirs();
-        try {
-            EmbeddedSolrConnector solr = new EmbeddedSolrConnector(storage, solr_config);
-            SolrInputDocument doc = new SolrInputDocument();
-            doc.addField(YaCySchema.id.name(), "ABCD0000abcd");
-            doc.addField(YaCySchema.title.name(), "Lorem ipsum");
-            doc.addField(YaCySchema.host_s.name(), "yacy.net");
-            doc.addField(YaCySchema.text_t.name(), "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.");
-            solr.add(doc);
-            
-            // start a server
-            SolrServlet.startServer("/solr", 8091, solr); // try http://localhost:8091/solr/select?q=*:*
-
-            // do a normal query
-            SolrDocumentList select = solr.query(YaCySchema.text_t.name() + ":tempor", 0, 10);
-            for (SolrDocument d : select) System.out.println("***TEST SELECT*** " + d.toString());
-
-            // do a facet query
-            select = solr.query(YaCySchema.text_t.name() + ":tempor", 0, 10);
-            for (SolrDocument d : select) System.out.println("***TEST SELECT*** " + d.toString());
-            
-            
-            // try http://127.0.0.1:8091/solr/select?q=ping
-            try {Thread.sleep(1000 * 1000);} catch (InterruptedException e) {}
-            solr.close();
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-
-    }
 }
diff --git a/test/de/anomic/document/ParserTest.java b/test/de/anomic/document/ParserTest.java
index b26529b53..7316c07e2 100644
--- a/test/de/anomic/document/ParserTest.java
+++ b/test/de/anomic/document/ParserTest.java
@@ -41,9 +41,9 @@ public class ParserTest {
 			final String mimetype = testFile[1];
 			final DigestURI url = new DigestURI("http://localhost/"+filename);
 
-			final Document[] docs = TextParser.parseSource(url, mimetype, null, file.length(), new FileInputStream(file), true);
+			final Document[] docs = TextParser.parseSource(url, mimetype, null, file.length(), new FileInputStream(file));
 			for (final Document doc: docs) {
-    			final Reader content = new InputStreamReader(doc.getText(), doc.getCharset());
+    			final Reader content = new InputStreamReader(doc.getTextStream(), doc.getCharset());
     			final StringBuilder str = new StringBuilder();
     			int c;
     			while( (c = content.read()) != -1 )
diff --git a/test/de/anomic/yacy/yacyURLTest.java b/test/de/anomic/yacy/yacyURLTest.java
index 66d3995c4..69c924b08 100644
--- a/test/de/anomic/yacy/yacyURLTest.java
+++ b/test/de/anomic/yacy/yacyURLTest.java
@@ -23,15 +23,19 @@ public class yacyURLTest extends TestCase {
 				new String[]{"/test/test2/../hallo/","/test/hallo/"},
 				new String[]{"/home/..test/../hallo/../","/home/"}
 		};		
-		
+		String testhost = "http://localhost";
 		for (int i=0; i < testStrings.length; i++) {
 			// desired conversion result
 			System.out.print("testResolveBackpath: " + testStrings[i][0]);
-			String shouldBe = testStrings[i][1];
+			String shouldBe = testhost + testStrings[i][1];
 			
 			// conversion result
-			String resolvedURL = MultiProtocolURI.resolveBackpath(testStrings[i][0]);
-			
+			String resolvedURL="";
+                        try {
+                            resolvedURL = (new MultiProtocolURI(testhost + testStrings[i][0])).toString();
+                        } catch (MalformedURLException ex) {
+                            fail ("malformed URL");
+                        }
 			// test if equal
 			assertEquals(shouldBe,resolvedURL);
 			System.out.println(" -> " + resolvedURL);
diff --git a/test/net/yacy/cora/document/MultiProtocolURITest.java b/test/net/yacy/cora/document/MultiProtocolURITest.java
index 3f18c5309..f69098727 100644
--- a/test/net/yacy/cora/document/MultiProtocolURITest.java
+++ b/test/net/yacy/cora/document/MultiProtocolURITest.java
@@ -28,7 +28,7 @@ public class MultiProtocolURITest {
 		for (int i=0; i<testURIs.length; i++) {
 			MultiProtocolURI uri = new MultiProtocolURI(testURIs[i][0]);
     
-   			assertEquals(uri.toNormalform(true, true, false, true), testURIs[i][1]);
+   			assertEquals(uri.toNormalform(true, true), testURIs[i][1]);
 		}
 	}
 }
diff --git a/test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java b/test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java
new file mode 100644
index 000000000..e85e4f94e
--- /dev/null
+++ b/test/net/yacy/cora/federate/solr/connector/EmbeddedSolrConnectorTest.java
@@ -0,0 +1,184 @@
+package net.yacy.cora.federate.solr.connector;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import net.yacy.cora.federate.solr.SolrServlet;
+import net.yacy.cora.federate.solr.YaCySchema;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.mortbay.jetty.Handler;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.servlet.Context;
+import org.mortbay.jetty.servlet.FilterHolder;
+
+public class EmbeddedSolrConnectorTest {
+
+    Server jetty; // Jetty server
+    EmbeddedSolrConnector solr;
+
+    public EmbeddedSolrConnectorTest() {
+    }
+
+    @Before
+    public void setUp() {
+        File solr_config = new File("defaults/solr");
+        File storage = new File("test/DATA/INDEX/webportal/SEGMENTS/text/solr/");
+        storage.mkdirs();
+        System.out.println("setup EmeddedSolrConnector using config dir: " + solr_config.getAbsolutePath());
+        try {
+            solr = new EmbeddedSolrConnector(storage, solr_config);
+
+            // start a server
+            jetty = startServer("/solr", 8091, solr); // try http://localhost:8091/solr/select?q=*:*
+
+        } catch (IOException ex) {
+            fail("IOException starting Jetty");
+        }
+    }
+
+    @After
+    public void tearDown() {
+        if (jetty != null) {
+            try {
+                jetty.stop();
+            } catch (Exception ex) {
+                fail("Exception stopping Jetty");
+            }
+        }
+        solr.close();
+    }
+
+    /**
+     * Test of query solr via jetty
+     */
+    @Test
+    public void testQuery() {
+        System.out.println("adding test document to solr");
+        SolrInputDocument doc = new SolrInputDocument();
+        doc.addField(YaCySchema.id.name(), "ABCD0000abcd");
+        doc.addField(YaCySchema.title.name(), "Lorem ipsum");
+        doc.addField(YaCySchema.host_s.name(), "yacy.net");
+        doc.addField(YaCySchema.text_t.name(), "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.");
+        try {
+            solr.add(doc);
+        } catch (IOException ex) {
+            fail("IOException adding test document to Solr");
+        } catch (SolrException ex) {
+            fail("SolrExceptin adding test document to Solr");
+        }
+        solr.commit(true);
+
+        System.out.println("query solr");
+        long expResult = 1;
+        SolrDocumentList result;
+        try {
+            result = solr.query(YaCySchema.text_t.name() + ":tempor", 0, 10);
+            assertEquals(expResult, result.getNumFound());
+        } catch (IOException ex) {
+            fail("Solr query no result");
+        }
+    }
+
+    public static void waitForSolr(String context, int port) throws Exception {
+        // A raw term query type doesn't check the schema
+        URL url = new URL("http://127.0.0.1:" + port + context + "/select?q={!raw+f=test_query}ping");
+
+        Exception ex = null;
+        // Wait for a total of 20 seconds: 100 tries, 200 milliseconds each
+        for (int i = 0; i < 600; i++) {
+            try {
+                InputStream stream = url.openStream();
+                stream.close();
+            } catch (IOException e) {
+                ex = e;
+                Thread.sleep(200);
+                continue;
+            }
+            return;
+        }
+        throw new RuntimeException("Jetty/Solr unresponsive", ex);
+    }
+
+    /**
+     * from org.apache.solr.client.solrj.embedded.JettySolrRunner
+     */
+    public static Server startServer(String context, int port, EmbeddedSolrConnector c) {
+        //this.context = context;
+        Server server = new Server(port);
+        /*
+         SocketConnector connector = new SocketConnector();
+         connector.setPort(port);
+         connector.setReuseAddress(true);
+         this.server.setConnectors(new Connector[] { connector });
+         this.server.setSessionIdManager(new HashSessionIdManager(new Random()));
+         */
+        server.setStopAtShutdown(true);
+        Context root = new Context(server, context, Context.SESSIONS);
+        root.addServlet(SolrServlet.Servlet404.class, "/*");
+
+        // attach org.apache.solr.response.XMLWriter to search requests
+        SolrServlet.initCore(c);
+        FilterHolder dispatchFilter = root.addFilter(SolrServlet.class, "*", Handler.REQUEST);
+
+        if (!server.isRunning()) {
+            try {
+                server.start();
+                waitForSolr(context, port);
+            } catch (Exception e) {
+                e.printStackTrace();
+            }
+        }
+        return server;
+    }
+
+    public static void main(String[] args) {
+        File solr_config = new File("defaults/solr");
+        File storage = new File("DATA/INDEX/webportal/SEGMENTS/text/solr/");
+        storage.mkdirs();
+        try {
+            EmbeddedSolrConnector solr = new EmbeddedSolrConnector(storage, solr_config);
+            SolrInputDocument doc = new SolrInputDocument();
+            doc.addField(YaCySchema.id.name(), "ABCD0000abcd");
+            doc.addField(YaCySchema.title.name(), "Lorem ipsum");
+            doc.addField(YaCySchema.host_s.name(), "yacy.net");
+            doc.addField(YaCySchema.text_t.name(), "Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.");
+            solr.add(doc);
+
+            // start a server
+            startServer("/solr", 8091, solr); // try http://localhost:8091/solr/select?q=*:*
+
+            // do a normal query
+            SolrDocumentList select = solr.query(YaCySchema.text_t.name() + ":tempor", 0, 10);
+            for (SolrDocument d : select) {
+                System.out.println("***TEST SELECT*** " + d.toString());
+            }
+
+            // do a facet query
+            select = solr.query(YaCySchema.text_t.name() + ":tempor", 0, 10);
+            for (SolrDocument d : select) {
+                System.out.println("***TEST SELECT*** " + d.toString());
+            }
+
+
+            // try http://127.0.0.1:8091/solr/select?q=ping
+            try {
+                Thread.sleep(1000 * 1000);
+            } catch (InterruptedException e) {
+            }
+            solr.close();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+
+    }
+}