- fixed charset problem in ConfigProfil_p.html (use accept-charset="UTF-8" in forms)

- fixed wrong XML output if no peers are known in Network.xml
- simplified parsing of table properties in wikiCode and ZTableToken
- reimplemented GC heuristics. They are needed to constantly ensure that an amount of free memory is available which is higher than Java's max. limit for performing a Full GC (please use serverMemory.request(long, boolean) rather than serverMemory.available(long, boolean) to provide data for averaging over the last GCs)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@3793 6c8d7289-2bf4-0310-a012-ef5d649a1542

6 files changed, 175 insertions, 148 deletions

diff --git a/htroot/ConfigProfile_p.html b/htroot/ConfigProfile_p.html
index 40cb39380..bba9ecaff 100644
--- a/htroot/ConfigProfile_p.html
+++ b/htroot/ConfigProfile_p.html
@@ -14,7 +14,7 @@
     <p>

       You do not need to provide any personal data here, but if you want to distribute your contact information, you can do that here.

     </p>

-    <form action="ConfigProfile_p.html" accept-charset="ascii">

+    <form action="ConfigProfile_p.html" accept-charset="UTF-8">

       <fieldset>

         <dl>

           <dt><label for="name">Name</label></dt>

diff --git a/htroot/Network.xml b/htroot/Network.xml
index eb244c310..c5c961665 100644
--- a/htroot/Network.xml
+++ b/htroot/Network.xml
@@ -1,7 +1,6 @@
 <?xml version="1.0"?>

-#(table)#

-::

 <peers>

+#(table)#::

 #{list}#

   <peer>

     <hash>#[hash]#</hash>

@@ -33,9 +32,7 @@
     #(/complete)#    

   </peer>

 #{/list}#

-</peers>

 ::

-<peers>

   <active>

     <count>#[active-count]#</count>

     <links>#[active-links]#</links>

@@ -57,30 +54,31 @@
     <words>#[all-words]#</words>

   </all>

   <your>

-        <name>#[my-name]#</name>

-        <hash>#[my-hash]#</hash>

-        <type>#(my-info)#virgin::junior::senior::principal#(/my-info)#</type>

-        <version>#[my-version]#</version>

-        <utc>#[my-utc]#</utc>

-        <uptime>#[my-uptime]#</uptime>

-        <links>#[my-links]#</links>

-        <words>#[my-words]#</words> 

-        <acceptcrawl>#[my-acceptcrawl]#</acceptcrawl>

-        <acceptindex>#[my-dhtreceive]#</acceptindex>

-        <sentwords>#[my-sI]#</sentwords>

-        <senturls>#[my-sU]#</senturls>

-        <receivedwords>#[my-rI]#</receivedwords>

-        <receivedurls>#[my-rU]#</receivedurls>

-        <ppm>#[my-ppm]#</ppm>

-		<qph>#[my-qph]#</qph>

-        <seeds>#[my-seeds]#</seeds>

-        <connects>#[my-connects]#</connects>

-    	<location>#[my-location]#</location>

-    	<seedurl>#[my-url]#</seedurl>

+    <name>#[my-name]#</name>

+    <hash>#[my-hash]#</hash>

+    <type>#(my-info)#virgin::junior::senior::principal#(/my-info)#</type>

+    <version>#[my-version]#</version>

+    <utc>#[my-utc]#</utc>

+    <uptime>#[my-uptime]#</uptime>

+    <links>#[my-links]#</links>

+    <words>#[my-words]#</words> 

+    <acceptcrawl>#[my-acceptcrawl]#</acceptcrawl>

+    <acceptindex>#[my-dhtreceive]#</acceptindex>

+    <acceptranking>#[my-rankingreceive]#</acceptranking>

+    <sentwords>#[my-sI]#</sentwords>

+    <senturls>#[my-sU]#</senturls>

+    <receivedwords>#[my-rI]#</receivedwords>

+    <receivedurls>#[my-rU]#</receivedurls>

+    <ppm>#[my-ppm]#</ppm>

+    <qph>#[my-qph]#</qph>

+    <seeds>#[my-seeds]#</seeds>

+    <connects>#[my-connects]#</connects>

+    <location>#[my-location]#</location>

+    <seedurl>#[my-url]#</seedurl>

   </your>

   <cluster>

-  	<ppm>#[gppm]#</ppm>

-	<qph>#[gqph]#</qph>

+    <ppm>#[gppm]#</ppm>

+    <qph>#[gqph]#</qph>

   </cluster>

   <status>#[comment]#</status>

   <!--

@@ -88,5 +86,5 @@
   1 - online mode, but no Internet available

   2 - not in online mode

   -->

-</peers>

 #(/table)#

+</peers>

diff --git a/source/de/anomic/data/wiki/tokens/TableToken.java b/source/de/anomic/data/wiki/tokens/TableToken.java
index a9aee689d..2fd8e13b1 100644
--- a/source/de/anomic/data/wiki/tokens/TableToken.java
+++ b/source/de/anomic/data/wiki/tokens/TableToken.java
@@ -47,6 +47,7 @@
 
 package de.anomic.data.wiki.tokens;
 
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.regex.Pattern;
 
@@ -92,10 +93,16 @@ public class TableToken extends AbstractToken {
 	private static final String[] tps = { "rowspan", "colspan", "vspace", "hspace", "cellspacing", "cellpadding", "border" };
     private static final HashMap/* <String,String[]> */ ps = new HashMap();
     static {
-    	ps.put("frame", 	new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" });
-    	ps.put("rules", 	new String[] { "none", "groups", "rows", "cols", "all" });
-    	ps.put("valign", 	new String[] { "top", "middle", "bottom", "baseline" });
-    	ps.put("align", 	new String[] { "left", "right", "center" });
+        Arrays.sort(tps);
+        String[] array;
+        Arrays.sort(array = new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" });
+        ps.put("frame", array);
+        Arrays.sort(array = new String[] { "none", "groups", "rows", "cols", "all" });
+    	ps.put("rules", array);
+        Arrays.sort(array = new String[] { "top", "middle", "bottom", "baseline" });
+    	ps.put("valign", array);
+        Arrays.sort(array = new String[] { "left", "right", "center" });
+    	ps.put("align", array);
     }
     
 	// contributed by [MN]
@@ -106,7 +113,7 @@ public class TableToken extends AbstractToken {
       * @param properties A string that may contain several table properties and/or junk.
       * @return A string that only contains table properties.
       */
-    private static StringBuffer parseTableProperties(final String properties){
+    private static StringBuffer parseTableProperties(final String properties) {
         String[] values = properties.replaceAll("&quot;", "").split("[= ]");     //splitting the string at = and blanks
         StringBuffer sb = new StringBuffer(properties.length());
         String key, value;
@@ -114,21 +121,20 @@ public class TableToken extends AbstractToken {
         int numberofvalues = values.length;
         for (int i=0; i<numberofvalues; i++) {
         	key = values[i].trim();
-        	if (i + 1 < numberofvalues) {
+            if (key.equals("nowrap")) {
+                addPair("nowrap", "nowrap", sb);
+            } else if (i + 1 < numberofvalues) {
         		value = values[++i].trim();
         		if (
         				(key.equals("summary")) ||
         				(key.equals("bgcolor") && value.matches("#{0,1}[0-9a-fA-F]{1,6}|[a-zA-Z]{3,}")) ||
         				((key.equals("width") || key.equals("height")) && value.matches("\\d+%{0,1}")) ||
-                        ((posVals = (String[])ps.get(key)) != null && isInArray(posVals, value)) ||
-        				(isInArray(tps, key) && value.matches("\\d+"))
+                        ((posVals = (String[])ps.get(key)) != null && Arrays.binarySearch(posVals, value) >= 0) ||
+        				(Arrays.binarySearch(tps, key) >= 0 && value.matches("\\d+"))
         		) {
                 	addPair(key, value, sb);
-                	continue;
         		}
         	}
-            if (key.equals("nowrap"))
-                addPair("nowrap", "nowrap", sb);
         }
         return sb;
     }
@@ -136,12 +142,6 @@ public class TableToken extends AbstractToken {
     private static StringBuffer addPair(String key, String value, StringBuffer sb) {
     	return sb.append(" ").append(key).append("=\"").append(value).append("\"");
     }
-    
-    private static boolean isInArray(Object[] array, Object find) {
-    	for (int i=array.length-1; i>-1; i--)
-    		if (array[i].equals(find)) return true;
-    	return false;
-    }
 	
 	public Pattern[] getRegex() { return pattern; }
 	public String[] getBlockElementNames() { return blockElementNames; }
diff --git a/source/de/anomic/data/wikiCode.java b/source/de/anomic/data/wikiCode.java
index 878d54cc2..6031ff075 100644
--- a/source/de/anomic/data/wikiCode.java
+++ b/source/de/anomic/data/wikiCode.java
@@ -49,6 +49,8 @@ package de.anomic.data;
 import java.io.BufferedReader;

 import java.io.IOException;

 import java.util.ArrayList;

+import java.util.Arrays;

+import java.util.HashMap;

 

 import de.anomic.data.wiki.abstractWikiParser;

 import de.anomic.data.wiki.wikiParser;

@@ -60,6 +62,23 @@ import de.anomic.yacy.yacyCore;
   * parts of YaCy that use this class, like the blog or the profile.

   */

 public class wikiCode extends abstractWikiParser implements wikiParser {

+    

+    /* Table properties */

+    private static final String[] tps = { "rowspan", "colspan", "vspace", "hspace", "cellspacing", "cellpadding", "border" };

+    private static final HashMap/* <String,String[]> */ ps = new HashMap();

+    static {

+        Arrays.sort(tps);

+        String[] array;

+        Arrays.sort(array = new String[] { "void", "above", "below", "hsides", "lhs", "rhs", "vsides", "box", "border" });

+        ps.put("frame", array);

+        Arrays.sort(array = new String[] { "none", "groups", "rows", "cols", "all" });

+        ps.put("rules", array);

+        Arrays.sort(array = new String[] { "top", "middle", "bottom", "baseline" });

+        ps.put("valign", array);

+        Arrays.sort(array = new String[] { "left", "right", "center" });

+        ps.put("align", array);

+    }

+    

     private String numListLevel="";

     private String ListLevel="";

     private String defListLevel="";

@@ -119,7 +138,7 @@ public class wikiCode extends abstractWikiParser implements wikiParser {
             newrowstart=true;

             line="<table";

             if (result.trim().length()>lenTableStart) {

-                line+=parseTableProperties(result.substring(lenTableStart).trim());

+                line+=parseTableProperties(result.substring(lenTableStart).trim()).toString();

             }

             line+=">";

             result=line;

@@ -157,7 +176,7 @@ public class wikiCode extends abstractWikiParser implements wikiParser {
                 propEnd=lenCellDivider;

             }

             else {

-                line+=parseTableProperties(result.substring(lenCellDivider,propEnd-lenAttribDivider).trim());

+                line+=parseTableProperties(result.substring(lenCellDivider,propEnd-lenAttribDivider).trim()).toString();

             }

             // quick&dirty fix for http://www.yacy-forum.de/viewtopic.php?t=2825 [MN]

             if(propEnd > cellEnd){

@@ -178,112 +197,44 @@ public class wikiCode extends abstractWikiParser implements wikiParser {
         }

         return result;

     }

-

-    //contributed by [MN]

+    

+    // contributed by [MN], changes by [FB]

     /** This method takes possible table properties and tests if they are valid.

       * Valid in this case means if they are a property for the table, tr or td

       * tag as stated in the HTML Pocket Reference by Jennifer Niederst (1st edition)

       * The method is important to avoid XSS attacks on the wiki via table properties.

-      * @param str A string that may contain several table properties and/or junk.

+      * @param properties A string that may contain several table properties and/or junk.

       * @return A string that only contains table properties.

       */

-    private String parseTableProperties(String str){

-        str = str.replaceAll("&quot;", "");      //killing all quotationmarks

-        String[] values = str.split("[= ]");     //splitting the string at = and blanks

-        str="";                                  //recycling... ;-)

-        int numberofvalues = values.length;

-        for(int i=0;i<numberofvalues;i++){

-            if((values[i].equals("rowspan")) ||

-               (values[i].equals("colspan")) ||

-               (values[i].equals("vspace")) ||

-               (values[i].equals("hspace")) ||

-               (values[i].equals("cellspacing")) ||

-               (values[i].equals("cellpadding")) ||

-               (values[i].equals("border"))){

-                if(i+1<numberofvalues){

-                    if(values[i+1].matches("\\d+")){

-                        str = str + " "+values[i]+"=\""+values[i+1]+"\"";

-                        i++;

-                    }

-                }

-            }

-            else if((values[i].equals("width"))||(values[i].equals("height"))){

-                if(i+1<numberofvalues){

-                    if(values[i+1].matches("\\d+%{0,1}")){

-                        str = str + " "+values[i]+"=\""+values[i+1]+"\"";

-                        i++;

-                    }

-                }

-            }

-            else if(values[i].equals("align")){

-                if(i+1<numberofvalues) {

-                    if((values[i+1].equals("left")) ||

-                       (values[i+1].equals("right")) ||

-                       (values[i+1].equals("center"))) {

-                        str = str + " "+values[i]+"=\""+values[i+1]+"\"";

-                        i++;

-                    }

-                }

-            }

-            else if(values[i].equals("valign")){

-                if(i+1<numberofvalues) {

-                    if((values[i+1].equals("top")) ||

-                       (values[i+1].equals("middle")) ||

-                       (values[i+1].equals("bottom")) ||

-                       (values[i+1].equals("baseline"))) {

-                        str = str + " "+values[i]+"=\""+values[i+1]+"\"";

-                        i++;

-                    }

+    private static StringBuffer parseTableProperties(final String properties) {

+        final String[] values = properties.replaceAll("&quot;", "").split("[= ]");     //splitting the string at = and blanks

+        final StringBuffer sb = new StringBuffer(properties.length());

+        String key, value;

+        String[] posVals;

+        final int numberofvalues = values.length;

+        for (int i=0; i<numberofvalues; i++) {

+            key = values[i].trim();

+            if (key.equals("nowrap")) {

+                addPair("nowrap", "nowrap", sb);

+            } else if (i + 1 < numberofvalues) {

+                value = values[++i].trim();

+                if (

+                        (key.equals("summary")) ||

+                        (key.equals("bgcolor") && value.matches("#{0,1}[0-9a-fA-F]{1,6}|[a-zA-Z]{3,}")) ||

+                        ((key.equals("width") || key.equals("height")) && value.matches("\\d+%{0,1}")) ||

+                        ((posVals = (String[])ps.get(key)) != null && Arrays.binarySearch(posVals, value) >= 0) ||

+                        (Arrays.binarySearch(tps, key) >= 0 && value.matches("\\d+"))

+                ) {

+                    addPair(key, value, sb);

                 }

             }

-            else if(values[i].equals("bgcolor")){

-                if(i+1<numberofvalues){

-                    if(values[i+1].matches("#{0,1}[0-9a-fA-F]{1,6}|[a-zA-Z]{3,}")){

-                        str = str + " "+values[i]+"=\""+values[i+1]+"\"";

-                        i++;

-                    }

-                }

-            }

-            else if(values[i].equals("rules")){

-                if(i+1<numberofvalues) {

-                    if((values[i+1].equals("none")) ||

-                       (values[i+1].equals("groups")) ||

-                       (values[i+1].equals("rows")) ||

-                       (values[i+1].equals("cols")) ||

-                       (values[i+1].equals("all"))) {

-                        str = str + " "+values[i]+"=\""+values[i+1]+"\"";

-                        i++;

-                    }

-                }

-            }

-            else if(values[i].equals("frame")){

-                if(i+1<numberofvalues) {

-                    if((values[i+1].equals("void")) ||

-                       (values[i+1].equals("above")) ||

-                       (values[i+1].equals("below")) ||

-                       (values[i+1].equals("hsides")) ||

-                       (values[i+1].equals("lhs")) ||

-                       (values[i+1].equals("rhs")) ||

-                       (values[i+1].equals("vsides")) ||

-                       (values[i+1].equals("box")) ||

-                       (values[i+1].equals("border"))) {

-                        str = str + " "+values[i]+"=\""+values[i+1]+"\"";

-                        i++;

-                    }

-                }

-            }

-            else if(values[i].equals("summary")){

-                if(i+1<numberofvalues){

-                    str = str + " "+values[i]+"=\""+values[i+1]+"\"";

-                    i++;

-                }

-            }

-            else if(values[i].equals("nowrap")){

-                str = str + " nowrap";

-            }

         }

-        return str;

-    } //end contrib [MN]

+        return sb;

+    }

+    

+    private static StringBuffer addPair(String key, String value, StringBuffer sb) {

+        return sb.append(" ").append(key).append("=\"").append(value).append("\"");

+    }

 

     /** This method processes ordered lists.

       */

diff --git a/source/de/anomic/server/serverAbstractThread.java b/source/de/anomic/server/serverAbstractThread.java
index 1ca6d07a8..efaceba45 100644
--- a/source/de/anomic/server/serverAbstractThread.java
+++ b/source/de/anomic/server/serverAbstractThread.java
@@ -250,7 +250,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
                 logSystem("thread '" + this.getName() + "' deployed, " + ((this.busyPause < 0) ? "starting job." : "starting loop."));

         }

         long timestamp;

-        long memstamp0, memstamp1, memnow;

+        long memstamp0, memstamp1;

         boolean isBusy;

         //Runtime rt = Runtime.getRuntime();

         

@@ -273,7 +273,8 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
                 timestamp = System.currentTimeMillis();

                 ratz(this.idlePause);

                 idletime += System.currentTimeMillis() - timestamp;

-            } else if ((memnow = serverMemory.available()) > memprereq) try {

+            //} else if ((memnow = serverMemory.available()) > memprereq) try {

+            } else if (serverMemory.request(memprereq, false)) try {

                 // do job

                 timestamp = System.currentTimeMillis();

                 memstamp0 = serverMemory.used();

@@ -307,7 +308,7 @@ public abstract class serverAbstractThread extends Thread implements serverThrea
                 busyCycles++;

             } else {

                 log.logWarning("Thread '" + this.getName() + "' runs short memory cycle. Free mem: " +

-                        (memnow / 1024) + " KB, needed: " + (memprereq / 1024) + " KB");

+                        (serverMemory.available() / 1024) + " KB, needed: " + (memprereq / 1024) + " KB");

                 // omit job, not enough memory

                 // process scheduled pause

                 timestamp = System.currentTimeMillis();

diff --git a/source/de/anomic/server/serverMemory.java b/source/de/anomic/server/serverMemory.java
index d5e711895..dd8f7c350 100644
--- a/source/de/anomic/server/serverMemory.java
+++ b/source/de/anomic/server/serverMemory.java
@@ -28,17 +28,60 @@ package de.anomic.server;
 

 import java.text.DecimalFormat;

 

+import de.anomic.server.logging.serverLog;

+

 public class serverMemory {

 

     public static boolean vm14 = System.getProperty("java.vm.version").startsWith("1.4");

     public static final long max = (vm14) ? computedMaxMemory() : Runtime.getRuntime().maxMemory() ; // patch for maxMemory bug in Java 1.4.2

     private static final Runtime runtime = Runtime.getRuntime();

     

+    private static final long[] gcs = new long[5];

+    private static int gcs_pos = 0;

+    

+    /** @return the amount of freed bytes by a forced GC this method performes */

+    private static long runGC() {

+        long memnow = available();

+        System.gc();

+        if (++gcs_pos >= gcs.length) gcs_pos = 0;

+        return (gcs[gcs_pos] = available() - memnow);

+    }

+    

+    /**

+     * This method calculates the average amount of bytes freed by the last GCs forced by this class

+     * @return the average amount of freed bytes of the last forced GCs or <code>0</code> if no

+     * GC has been run yet

+     */

+    public static long getAverageGCFree() {

+        long x = 0;

+        int y = 0;

+        for (int i=0; i<gcs.length; i++)

+            if (gcs[i] != 0) {

+                x += gcs[i];

+                y++;

+            }

+        return (y == 0) ? 0 : x / y;

+    }

+

     public static long free() {

         // memory that is free without increasing of total memory taken from os

         return runtime.freeMemory();

     }

-   

+    

+    /**

+     * Tries to free a specified amount of bytes. If the currently available memory is enough, the

+     * method returns <code>true</code> without performing additional steps. Otherwise - if

+     * <code>gciffail</code> is set - a Full GC is run, if <code>gciffail</code> is set to

+     * <code>false</code> and not enough memory is available, this method returns <code>false</code>.

+     * 

+     * @see serverMemory#request(long, boolean) for another implementation

+     * @param memory amount of bytes to be assured to be free

+     * @param gciffail if not enough memory is available, this parameter specifies whether to perform

+     * a Full GC to free enough RAM

+     * @return whether enough RAM is available

+     * @deprecated use {@link serverMemory#request(long, boolean)} instead to enable the collection of

+     * heuristics about explicit usage of Full GCs.

+     */

     public static boolean available(long memory, boolean gciffail) {

         if (available() >= memory) return true;

         if (!gciffail) return false;

@@ -51,6 +94,40 @@ public class serverMemory {
         return max - runtime.totalMemory() + runtime.freeMemory();

     }

     

+    /**

+     * <p>Tries to free a specified amount of bytes.</p>

+     * <p>

+     *   If the currently available memory is enough, the method returns <code>true</code> without

+     *   performing additional steps. If not, the behaviour depends on the parameter <code>force</code>.

+     *   If <code>false</code>, a Full GC is only performed if former GCs indicated that a GC should

+     *   provide enough free memory. If former GCs didn't but <code>force</code> is set to <code>true</code>

+     *   a Full GC is performed nevertheless.

+     * </p>

+     * <p>

+     *   Setting the <code>force</code> parameter to false doesn't necessarily mean, that no GC may be

+     *   performed by this method, if the currently available memory doesn't suffice!

+     * </p>

+     * <p><em>Be careful with this method as GCs should always be the last measure to take</em></p>

+     * 

+     * @param size the requested amount of free memory in bytes

+     * @param specifies whether a GC should be run even in case former GCs didn't provide enough memory

+     * @return whether enough memory could be freed (or is free) or not

+     */

+    public static boolean request(long size, boolean force) {

+        long avail = available();

+        if (avail >= size) return true;

+        long avg;

+        if (force || ((avg = getAverageGCFree()) == 0 || avg + avail >= size)) {

+            long freed = runGC();

+            avail = available();

+            serverLog.logInfo("MEMORY", "performed explicit GC, freed " + (freed / 1024) + " KB (requested/available: " + size + " / " + avail + " KB)");

+            return avail >= size;

+        } else {

+            serverLog.logInfo("MEMORY", "couldn't free enough memory (requested/available " + size + "/" + avail + " KB)");

+            return false;

+        }

+    }

+    

     public static long used() {

         // memory that is currently bound in objects

         return runtime.totalMemory() - runtime.freeMemory();