- fixed possible bugs in Stack (may affect Crawler reset) and RandomAccess handling

- increased default memory size to 180MB
- fixed possible bug in http client reset (there was a deadlock)
- bug in BOBHeap marked, but not solved, cause is still unknown.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@5912 6c8d7289-2bf4-0310-a012-ef5d649a1542

9 files changed, 51 insertions, 19 deletions

diff --git a/defaults/yacy.init b/defaults/yacy.init
index fde9dd867..40713818c 100644
--- a/defaults/yacy.init
+++ b/defaults/yacy.init
@@ -619,8 +619,8 @@ cleanup.deletionPublishedNews = true
 # -Xmx<size> and -Xms<size> maximum/init Java heap size

 # both values should be equal,

 # othervise the YaCy-internal memory supervision does not work

-javastart_Xmx=Xmx120m

-javastart_Xms=Xms120m

+javastart_Xmx=Xmx180m

+javastart_Xms=Xms180m

 

 # priority of the yacy-process

 # is valid in unix/shell and windows environments but

diff --git a/htroot/Threaddump_p.java b/htroot/Threaddump_p.java
index 553fc509e..eb7fcb067 100644
--- a/htroot/Threaddump_p.java
+++ b/htroot/Threaddump_p.java
@@ -40,6 +40,7 @@ import java.util.HashMap;
 import de.anomic.htmlFilter.htmlFilterCharacterCoding;
 import de.anomic.http.httpRequestHeader;
 import de.anomic.kelondro.util.FileUtils;
+import de.anomic.kelondro.util.MemoryControl;
 import de.anomic.plasma.plasmaSwitchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
@@ -78,6 +79,7 @@ public class Threaddump_p {
             final ArrayList<Map<Thread,StackTraceElement[]>> traces = new ArrayList<Map<Thread,StackTraceElement[]>>();
             for (int i = 0; i < multipleCount; i++) {
                 traces.add(Thread.getAllStackTraces());
+                if (MemoryControl.available() < 20 * 1024 * 1024) break;
             }
             appendStackTraceStats(sb.getRootPath(), buffer, traces, plain, Thread.State.BLOCKED);
             appendStackTraceStats(sb.getRootPath(), buffer, traces, plain, Thread.State.RUNNABLE);
diff --git a/source/de/anomic/crawler/IndexingStack.java b/source/de/anomic/crawler/IndexingStack.java
index aaaa12726..0b4411d6f 100644
--- a/source/de/anomic/crawler/IndexingStack.java
+++ b/source/de/anomic/crawler/IndexingStack.java
@@ -95,16 +95,33 @@ public class IndexingStack {
     }

 

     public synchronized QueueEntry pop() throws IOException {

-        if (sbQueueStack.size() == 0) {

-        	Log.logInfo("IndexingStack", "sbQueueStack.size() == 0");

-        	return null;

-        }

-        final Row.Entry b = sbQueueStack.pot();

-        if (b == null) {

-        	Log.logInfo("IndexingStack", "sbQueueStack.pot() == null");

-        	return null;

-        }

-        return new QueueEntry(b);

+    	synchronized (sbQueueStack) {

+    		int sizeBefore;

+	    	while ((sizeBefore = sbQueueStack.size()) > 0) {

+		        Row.Entry b = sbQueueStack.pot();

+		        if (b == null) {

+		        	Log.logInfo("IndexingStack", "sbQueueStack.pot() == null");

+		        	if (sbQueueStack.size() < sizeBefore) continue;

+		        	Log.logSevere("IndexingStack", "sbQueueStack does not shrink after pot() == null; trying pop()");

+		        }

+		        if (sbQueueStack.size() < sizeBefore) {

+		        	return new QueueEntry(b);

+		        } else {

+		        	Log.logSevere("IndexingStack", "sbQueueStack does not shrink after pot() != null; trying pop()");

+		        }

+		        sizeBefore = sbQueueStack.size();

+		        b = sbQueueStack.pop();

+		        if (b == null) {

+		        	Log.logInfo("IndexingStack", "sbQueueStack.pop() == null");

+		        	if (sbQueueStack.size() < sizeBefore) continue;

+		        	Log.logSevere("IndexingStack", "sbQueueStack does not shrink after pop() == null; failed");

+		        	return null;

+		        }

+		        return new QueueEntry(b);

+	    	}

+	    	Log.logInfo("IndexingStack", "sbQueueStack.size() == 0");

+	        return null;

+    	}

     }

 

     public synchronized QueueEntry remove(final String urlHash) {

diff --git a/source/de/anomic/http/httpClient.java b/source/de/anomic/http/httpClient.java
index a869c393f..f787fe660 100644
--- a/source/de/anomic/http/httpClient.java
+++ b/source/de/anomic/http/httpClient.java
@@ -90,6 +90,7 @@ public class httpClient {
         /**
          * set options for client
          */
+        MultiThreadedHttpConnectionManager.shutdownAll();
         initConnectionManager();
         
         // accept self-signed or untrusted certificates
@@ -112,7 +113,11 @@ public class httpClient {
     }
 
     public static void initConnectionManager() {
-        MultiThreadedHttpConnectionManager.shutdownAll();
+        if (conManager != null) {
+        	conManager.closeIdleConnections(0);
+        	conManager.deleteClosedConnections();
+        	conManager.shutdown();
+        }
         conManager = new MultiThreadedHttpConnectionManager();
         apacheHttpClient = new HttpClient(conManager);
         
diff --git a/source/de/anomic/kelondro/blob/BLOBHeap.java b/source/de/anomic/kelondro/blob/BLOBHeap.java
index 8faa25a92..e896c2d76 100755
--- a/source/de/anomic/kelondro/blob/BLOBHeap.java
+++ b/source/de/anomic/kelondro/blob/BLOBHeap.java
@@ -175,7 +175,7 @@ public final class BLOBHeap extends BLOBHeapModifier implements BLOB {
             assert posBuffer + 4 + key.length <= ba.length : "posBuffer = " + posBuffer + ", key.length = " + key.length + ", ba.length = " + ba.length;
             System.arraycopy(key, 0, ba, posBuffer + 4, key.length);
             assert posBuffer + 4 + key.length + blob.length <= ba.length : "posBuffer = " + posBuffer + ", key.length = " + key.length + ", blob.length = " + blob.length + ", ba.length = " + ba.length;
-            System.arraycopy(blob, 0, ba, posBuffer + 4 + key.length, blob.length);
+            System.arraycopy(blob, 0, ba, posBuffer + 4 + key.length, blob.length); //java.lang.ArrayIndexOutOfBoundsException here
             posFile += 4 + key.length + blob.length;
             posBuffer += 4 + key.length + blob.length;
         }
diff --git a/source/de/anomic/kelondro/io/CachedRandomAccess.java b/source/de/anomic/kelondro/io/CachedRandomAccess.java
index 16f641f63..d287d665f 100644
--- a/source/de/anomic/kelondro/io/CachedRandomAccess.java
+++ b/source/de/anomic/kelondro/io/CachedRandomAccess.java
@@ -72,8 +72,8 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra
             return;

         }

         // we fill the cache here

-        int available = (int) (this.RAFile.length() - seek);

-        if (available < len) throw new IOException("EOF, available = " + available + ", requested = " + len);

+        long available = this.RAFile.length() - seek;

+        if (available < (long) len) throw new IOException("EOF, available = " + available + ", requested = " + len);

         if (cachestart + cachelen == seek && cache.length - cachelen >= len) {

             RAFile.readFully(cache, cachelen, len);

             //System.out.println("*** DEBUG FileRA " + this.file.getName() + ": append fill " + len + " bytes");

@@ -81,7 +81,7 @@ public final class CachedRandomAccess extends AbstractRandomAccess implements Ra
             cachelen += len;

         } else {

             // fill the cache as much as possible

-            int m = Math.min(available, cache.length);

+            int m = (int) Math.min(available, (long) cache.length);

             RAFile.readFully(cache, 0, m);

             cachestart = seek;

             cachelen = m;

diff --git a/source/de/anomic/kelondro/text/IODispatcher.java b/source/de/anomic/kelondro/text/IODispatcher.java
index 9b8bc8c8f..a4ba373f1 100644
--- a/source/de/anomic/kelondro/text/IODispatcher.java
+++ b/source/de/anomic/kelondro/text/IODispatcher.java
@@ -209,6 +209,14 @@ public class IODispatcher <ReferenceType extends Reference> extends Thread {
         }

 

         public File merge() {

+        	if (!f1.exists()) {

+        		Log.logWarning("IODispatcher", "merge of file (1) " + f1.getName() + " failed: file does not exists");

+        		return null;

+        	}

+        	if (!f2.exists()) {

+        		Log.logWarning("IODispatcher", "merge of file (2) " + f2.getName() + " failed: file does not exists");

+        		return null;

+        	}

             try {

                 return array.mergeMount(f1, f2, factory, payloadrow, newFile);

             } catch (IOException e) {

diff --git a/startYACY.bat b/startYACY.bat
index c4eb3f32f..d20f90a31 100644
--- a/startYACY.bat
+++ b/startYACY.bat
@@ -15,7 +15,7 @@ For %%X in (libx/*.jar) Do Call %0 CPGEN libx\%%X
 REM Please change the "javastart" settings in the web-interface "Basic Configuration" -> "Advanced" 

 set jmx=

 set jms=

-set javacmd=-Xmx120m -Xms120m

+set javacmd=-Xmx180m -Xms180m

 set priolvl=10

 set priority=/BELOWNORMAL

 if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX

diff --git a/startYACY.sh b/startYACY.sh
index 87ff4e2ef..2f0415559 100755
--- a/startYACY.sh
+++ b/startYACY.sh
@@ -155,7 +155,7 @@ then
 #		JAVA_ARGS="-$i $JAVA_ARGS";
 #	done
 else
-    JAVA_ARGS="-Xmx120m -Xms120m $JAVA_ARGS";
+    JAVA_ARGS="-Xmx180m -Xms180m $JAVA_ARGS";
     PORT="8080"
 fi