Merge branch 'master' of ssh://git@gitorious.org/yacy/rc1.git

8 files changed, 206 insertions, 67 deletions

diff --git a/htroot/Connections_p.html b/htroot/Connections_p.html
index 79d07a4c0..847266c65 100644
--- a/htroot/Connections_p.html
+++ b/htroot/Connections_p.html
@@ -11,26 +11,22 @@
     <script type="text/javascript" src="js/sorttable.js"></script>

     <h2>Server Connection Tracking</h2>

     <h3>Incoming Connections</h3>

-    <p>Showing #[numActiveRunning]# active, #[numActivePending]# pending connections from a max. of #[numMax]# allowed incoming connections.</p>

+    <p>Showing #[numActiveRunning]# active connections from a max. of #[numMax]# allowed incoming connections.</p>

     <table class="sortable" border="0">

       <tr class="TableHeader" valign="bottom">

         <td>Protocol</td>

         <td>Duration</td>

         <td>Source IP[:Port]</td>

-        <td>Dest. IP[:Port]</td>

         <td>Command</td>

-        <td>Used</td>

-        <td>Close</td>

+        <td>ID</td>

       </tr>

       #{list}#

       <tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#" title="#[sessionName]#">

         <td>#[proto]#</td>

-        <td>#(ms)##[duration]#::#[duration]# ms#(/ms)#</td>

+        <td>#[duration]#</td>

         <td>#[source]#</td>

-        <td>#[dest]#</td>

-        <td>#(running)#Waiting for new request nr. # #[reqNr]#::#[command]##(/running)#</td>

-        <td>#[used]#</td>

-        <td><a href="Connections_p.html?closeServerSession=#[serverSessionID]#">[Close]</a></td>

+        <td>#[command]#</td>

+        <td>#[id]#</td>

       </tr>

       #{/list}#

     </table>

diff --git a/htroot/Connections_p.java b/htroot/Connections_p.java
index be76f0a7c..5d28da486 100644
--- a/htroot/Connections_p.java
+++ b/htroot/Connections_p.java
@@ -30,52 +30,55 @@
 import java.util.Collections;

 import java.util.LinkedList;

 import java.util.List;

-import java.util.Set;

 

 import net.yacy.cora.protocol.ConnectionInfo;

 import net.yacy.cora.protocol.RequestHeader;

-import net.yacy.http.YaCyHttpServer;

-import net.yacy.search.Switchboard;

 import net.yacy.server.serverObjects;

 import net.yacy.server.serverSwitch;

 

 public final class Connections_p {

 

-    public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, final serverSwitch env) {

+    public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {

         // return variable that accumulates replacements

-        final Switchboard sb = (Switchboard) env;

         final serverObjects prop = new serverObjects();

 

         // server sessions

-        // get the serverCore thread

-        final YaCyHttpServer httpd = sb.getHttpServer();

-

-        // waiting for all threads to finish

-        int idx = 0, numActivePending = 0;

-        prop.put("list", idx);

+        List<ConnectionInfo> allConnectionsSorted = new LinkedList<ConnectionInfo>(ConnectionInfo.getServerConnections());

+        Collections.sort(allConnectionsSorted);

+        Collections.reverse(allConnectionsSorted); // toggle ascending/descending

 

-        prop.putNum("numMax", httpd.getMaxSessionCount());

-        prop.putNum("numActiveRunning", httpd.getJobCount());

-        prop.putNum("numActivePending", numActivePending);

+        int c = 0;

+        synchronized (allConnectionsSorted) {

+	        for (final ConnectionInfo conInfo: allConnectionsSorted) {

+	            prop.put("list_" + c + "_proto", conInfo.getProtocol());

+	            prop.putNum("list_" + c + "_duration", conInfo.getLifetime());

+	            prop.put("list_" + c + "_source", conInfo.getTargetHost());

+	            prop.putHTML("list_" + c + "_command", conInfo.getCommand());

+	            prop.put("list_" + c + "_id", conInfo.getID());

+	            c++;

+	        }

+        }

+        prop.put("list", c);

+        prop.putNum("numMax", ConnectionInfo.getServerMaxcount());

+        prop.putNum("numActiveRunning", c);

 

         // client sessions

-        final Set<ConnectionInfo> allConnections = ConnectionInfo.getAllConnections();

         // sorting: sort by initTime, decending

-        List<ConnectionInfo> allConnectionsSorted = new LinkedList<ConnectionInfo>(allConnections);

+        allConnectionsSorted = new LinkedList<ConnectionInfo>(ConnectionInfo.getAllConnections());

         Collections.sort(allConnectionsSorted);

         Collections.reverse(allConnectionsSorted); // toggle ascending/descending

 

-        int c = 0;

+        c = 0;

         synchronized (allConnectionsSorted) {

-        for (final ConnectionInfo conInfo: allConnectionsSorted) {

-            prop.put("clientList_" + c + "_clientProtocol", conInfo.getProtocol());

-            prop.putNum("clientList_" + c + "_clientLifetime", conInfo.getLifetime());

-            prop.putNum("clientList_" + c + "_clientUpbytes", conInfo.getUpbytes());

-            prop.put("clientList_" + c + "_clientTargetHost", conInfo.getTargetHost());

-            prop.putHTML("clientList_" + c + "_clientCommand", conInfo.getCommand());

-            prop.put("clientList_" + c + "_clientID", conInfo.getID());

-            c++;

-        }

+	        for (final ConnectionInfo conInfo: allConnectionsSorted) {

+	            prop.put("clientList_" + c + "_clientProtocol", conInfo.getProtocol());

+	            prop.putNum("clientList_" + c + "_clientLifetime", conInfo.getLifetime());

+	            prop.putNum("clientList_" + c + "_clientUpbytes", conInfo.getUpbytes());

+	            prop.put("clientList_" + c + "_clientTargetHost", conInfo.getTargetHost());

+	            prop.putHTML("clientList_" + c + "_clientCommand", conInfo.getCommand());

+	            prop.put("clientList_" + c + "_clientID", conInfo.getID());

+	            c++;

+	        }

         }

         prop.put("clientList", c);

         prop.put("clientActive", ConnectionInfo.getCount());

diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java
index 982d97b04..197e1441a 100644
--- a/htroot/PerformanceQueues_p.java
+++ b/htroot/PerformanceQueues_p.java
@@ -28,9 +28,9 @@ import java.io.File;
 import java.util.Iterator;
 import java.util.Map;
 
+import net.yacy.cora.protocol.ConnectionInfo;
 import net.yacy.cora.protocol.HeaderFramework;
 import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.http.YaCyHttpServer;
 import net.yacy.kelondro.data.word.WordReference;
 import net.yacy.kelondro.rwi.IndexCell;
 import net.yacy.kelondro.util.FileUtils;
@@ -241,14 +241,13 @@ public class PerformanceQueues_p {
             /*
              * configuring the http pool
              */
-            final YaCyHttpServer httpd = sb.getHttpServer();
             try {
                 maxBusy = post.getInt("httpd Session Pool_maxActive", 8);
             } catch (final NumberFormatException e) {
                 maxBusy = 8;
             }
 
-            (httpd).setMaxSessionCount(maxBusy);
+            ConnectionInfo.setServerMaxcount(maxBusy);
 
             // storing the new values into configfile
             sb.setConfig("httpdMaxBusySessions",maxBusy);
@@ -281,10 +280,9 @@ public class PerformanceQueues_p {
         prop.put("pool_0_maxActive", sb.getConfigLong(SwitchboardConstants.CRAWLER_THREADS_ACTIVE_MAX, 0));
         prop.put("pool_0_numActive", sb.crawlQueues.activeWorkerEntries().size());
 
-        final YaCyHttpServer httpd = sb.getHttpServer(); 
         prop.put("pool_1_name", "httpd Session Pool");
-        prop.put("pool_1_maxActive", httpd.getMaxSessionCount());
-        prop.put("pool_1_numActive", httpd.getJobCount());
+        prop.put("pool_1_maxActive", ConnectionInfo.getServerMaxcount());
+        prop.put("pool_1_numActive", ConnectionInfo.getServerCount());
 
         prop.put("pool", "2");
 
diff --git a/htroot/Status.java b/htroot/Status.java
index 87ff658c3..3d43644df 100644
--- a/htroot/Status.java
+++ b/htroot/Status.java
@@ -29,10 +29,11 @@
 

 import java.net.InetAddress;

 import java.util.Date;

+

+import net.yacy.cora.protocol.ConnectionInfo;

 import net.yacy.cora.protocol.Domains;

 import net.yacy.cora.protocol.RequestHeader;

 import net.yacy.cora.util.Memory;

-import net.yacy.http.YaCyHttpServer;

 import net.yacy.kelondro.io.ByteCount;

 import net.yacy.kelondro.util.Formatter;

 import net.yacy.kelondro.util.MemoryControl;

@@ -327,10 +328,8 @@ public class Status
         prop.put("trafficCrawler", Formatter.bytesToString(ByteCount.getAccountCount(ByteCount.CRAWLER)));

 

         // connection information

-        final YaCyHttpServer httpd =  sb.getHttpServer();

-

-        prop.putNum("connectionsActive", httpd.getJobCount());

-        prop.putNum("connectionsMax", httpd.getMaxSessionCount());

+        prop.putNum("connectionsActive", ConnectionInfo.getServerCount());

+        prop.putNum("connectionsMax", ConnectionInfo.getServerMaxcount());

 

         // Queue information

         final int loaderJobCount = sb.crawlQueues.activeWorkerEntries().size();

diff --git a/source/net/yacy/cora/protocol/ConnectionInfo.java b/source/net/yacy/cora/protocol/ConnectionInfo.java
index 0845c9699..0487b632c 100644
--- a/source/net/yacy/cora/protocol/ConnectionInfo.java
+++ b/source/net/yacy/cora/protocol/ConnectionInfo.java
@@ -43,10 +43,13 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
      */
     private final static Set<ConnectionInfo> allConnections = Collections
             .synchronizedSet(new HashSet<ConnectionInfo>());
+    private final static Set<ConnectionInfo> serverConnections = Collections
+            .synchronizedSet(new HashSet<ConnectionInfo>());
     // this is only for statistics, so it can be bigger to see lost connectionInfos
     private final static int staleAfterMillis = 30 * 60000; // 30 minutes
     
     private static int maxcount = 20;
+    private static int serverMaxCount = 20;
 
     private final String protocol;
     private final String targetHost;
@@ -126,6 +129,17 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     public static Set<ConnectionInfo> getAllConnections() {
         return allConnections;
     }
+
+    /**
+     * gets a {@link Set} of all collected server ConnectionInfos
+     * 
+     * Important: iterations must be synchronized!
+     * 
+     * @return the allConnections
+     */
+    public static Set<ConnectionInfo> getServerConnections() {
+        return serverConnections;
+    }
     
     /**
      * gets the number of active client connections
@@ -137,6 +151,15 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     }
     
     /**
+     * gets the number of active server connections
+     * 
+     * @return count of active connections
+     */
+    public static int getServerCount() {
+    	return getServerConnections().size();
+    }
+    
+    /**
      * gets the usage of the Client connection manager by active connections
      * 
      * @return load in percent
@@ -146,6 +169,13 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     }
     
     /**
+     * @return wether the server max connection-count is reached
+     */
+    public static boolean isServerCountReached() {
+    	return getServerCount() >= getServerMaxcount();
+    }
+    
+    /**
      * @return how many bytes queued up
      */
     public static long getActiveUpbytes() {
@@ -179,6 +209,26 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     public static void setMaxcount(final int max) {
     	if (max > 0) maxcount = max;
     }
+    
+    /**
+     * gets the max connection count of the Server connection manager
+     * 
+     * @return max connections
+     */
+    public static int getServerMaxcount() {
+    	return serverMaxCount;
+    }
+    
+    /**
+     * gets the max connection count of the Sever connection manager
+     * to be used in statistics
+     * 
+     * @param max connections
+     * @TODO Is it correct to only set if max > 0? What if maxcount is > 0 and max = 0 ?
+     */
+    public static void setServerMaxcount(final int max) {
+    	if (max > 0) serverMaxCount = max;
+    }
 
     /**
      * add a connection to the list of all current connections
@@ -190,6 +240,15 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     }
 
     /**
+     * add a Server connection to the list of all current connections
+     * 
+     * @param conInfo
+     */
+    public static void addServerConnection(final ConnectionInfo conInfo) {
+    	getServerConnections().add(conInfo);
+    }
+
+    /**
      * remove a connection from the list of all current connections
      * 
      * @param conInfo
@@ -199,6 +258,15 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     }
 
     /**
+     * remove a Server connection from the list of all current connections
+     * 
+     * @param conInfo
+     */
+    public static void removeServerConnection(final ConnectionInfo conInfo) {
+    	getServerConnections().remove(conInfo);
+    }
+
+    /**
      * connections with same id {@link equals()} another
      * 
      * @param id
@@ -206,13 +274,26 @@ public class ConnectionInfo implements Comparable<ConnectionInfo> {
     public static void removeConnection(final int id) {
         removeConnection(new ConnectionInfo(null, null, null, id, 0, 0));
     }
+
+    /**
+     * connections with same id {@link equals()} another
+     * 
+     * @param id
+     */
+    public static void removeServerConnection(final int id) {
+        removeServerConnection(new ConnectionInfo(null, null, null, id, 0, 0));
+    }
     
     /**
      * removes stale connections
      */
     public static void cleanUp() {
-        Iterator<ConnectionInfo> iter = getAllConnections().iterator();
-        synchronized (iter) { 
+    	cleanup(getAllConnections().iterator());
+    	cleanup(getServerConnections().iterator());
+    }
+    
+    private static void cleanup(final Iterator<ConnectionInfo> iter) {
+    	synchronized (iter) { 
             while (iter.hasNext()) {
                 ConnectionInfo con = iter.next();
                 if(con.getLifetime() > staleAfterMillis) {
diff --git a/source/net/yacy/http/Jetty9HttpServerImpl.java b/source/net/yacy/http/Jetty9HttpServerImpl.java
index 7fda0ed0e..42342f860 100644
--- a/source/net/yacy/http/Jetty9HttpServerImpl.java
+++ b/source/net/yacy/http/Jetty9HttpServerImpl.java
@@ -167,7 +167,7 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
         // define list of YaCy specific general handlers
         HandlerList handlers = new HandlerList();
         handlers.setHandlers(new Handler[] 
-           {domainHandler, new ProxyCacheHandler(), new ProxyHandler()}); 
+           {new MonitorHandler(), domainHandler, new ProxyCacheHandler(), new ProxyHandler()}); 
 
         // context handler for dispatcher and security (hint: dispatcher requires a context)
         ContextHandler context = new ContextHandler();
@@ -241,11 +241,6 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
         server.join();
     }
 
-    @Override
-    public void setMaxSessionCount(int maxBusy) {
-        // TODO:
-    }
-
     /**
      * @return true if ssl/https connector is available
      */
@@ -408,16 +403,6 @@ public class Jetty9HttpServerImpl implements YaCyHttpServer {
     }
 
     @Override
-    public int getMaxSessionCount() {
-        return server.getThreadPool().getThreads();
-    }
-
-    @Override
-    public int getJobCount() {
-        return getMaxSessionCount() - server.getThreadPool().getIdleThreads(); // TODO:
-    }
-
-    @Override
     public String getVersion() {
         return "Jetty " + Server.getVersion();
     }
diff --git a/source/net/yacy/http/MonitorHandler.java b/source/net/yacy/http/MonitorHandler.java
new file mode 100644
index 000000000..43e44c3e1
--- /dev/null
+++ b/source/net/yacy/http/MonitorHandler.java
@@ -0,0 +1,80 @@
+/**
+ *  MonitorHandler
+ *  Copyright 2014 by Sebastian Gaebel
+ *  First released 15.05.2014 at http://yacy.net
+ *
+ *  $LastChangedDate$
+ *  $LastChangedRevision$
+ *  $LastChangedBy$
+ *
+ *  This library is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU Lesser General Public
+ *  License as published by the Free Software Foundation; either
+ *  version 2.1 of the License, or (at your option) any later version.
+ *
+ *  This library is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ *  Lesser General Public License for more details.
+ *
+ *  You should have received a copy of the GNU Lesser General Public License
+ *  along with this program in the file lgpl21.txt
+ *  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package net.yacy.http;
+
+import java.io.IOException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import net.yacy.cora.protocol.ConnectionInfo;
+import net.yacy.cora.protocol.Domains;
+
+import org.eclipse.jetty.io.Connection;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.handler.AbstractHandler;
+
+public class MonitorHandler extends AbstractHandler {
+	
+	private final Connection.Listener remover = new Connection.Listener() {
+
+		@Override
+		public void onClosed(Connection c) {
+			ConnectionInfo.removeServerConnection(c.hashCode());
+		}
+
+		@Override
+		public void onOpened(Connection c) {
+		}
+	};
+
+	@Override
+	public void handle(String target, Request baseRequest, HttpServletRequest request,
+            HttpServletResponse response) throws IOException, ServletException {
+		
+		final Connection connection = baseRequest.getHttpChannel().getEndPoint().getConnection();
+		final ConnectionInfo info = new ConnectionInfo(
+				baseRequest.getProtocol(),
+				baseRequest.getRemoteAddr() + ":" + baseRequest.getRemotePort(),
+				baseRequest.getMethod() + " " + baseRequest.getUri().getPathAndParam(),
+				connection.hashCode(),
+				baseRequest.getTimeStamp(),
+				-1);
+		
+		if (ConnectionInfo.getServerConnections().contains(info)) {
+			ConnectionInfo.removeServerConnection(info);
+		} else {
+			connection.addListener(remover);
+		}
+		ConnectionInfo.addServerConnection(info);
+		
+		if (ConnectionInfo.isServerCountReached()) {
+			if (Domains.isLocal(baseRequest.getRemoteAddr(), baseRequest.getRemoteInetSocketAddress().getAddress())) return;
+			response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE,"max. server connections reached (see System Administration -> Performance Settings of Busy Queues -> Thread Pool Settings).");
+            baseRequest.setHandled(true);
+		}
+	}
+}
diff --git a/source/net/yacy/http/YaCyHttpServer.java b/source/net/yacy/http/YaCyHttpServer.java
index eb39e997a..2d8528cdf 100644
--- a/source/net/yacy/http/YaCyHttpServer.java
+++ b/source/net/yacy/http/YaCyHttpServer.java
@@ -19,10 +19,7 @@ public interface YaCyHttpServer {
     

     abstract void startupServer() throws Exception;

     abstract void stop() throws Exception;

-    abstract void setMaxSessionCount(int cnt);

     abstract InetSocketAddress generateSocketAddress(String port) throws SocketException;

-    abstract int getMaxSessionCount();

-    abstract int getJobCount();

     abstract int getSslPort();

     abstract boolean withSSL();

     abstract void reconnect(int milsec);