changed way of storage for search requests:

- the search request cache can now get as large as 1000 entries
- if more entries arrive, unused are deleted
- the elements may stay in the cache up to 10 minutes and longer if they are used
- the elements are deleted earlier that 10 minutes if the memory gets low
This commit was mainly done for metager-feeding peers that have a query load of 50000 queries each day. Also added:
- a monitor for cache hit/cache miss in PerformanceMemory_p.html (see at bottom of page)

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7093 6c8d7289-2bf4-0310-a012-ef5d649a1542

3 files changed, 76 insertions, 24 deletions

diff --git a/htroot/PerformanceMemory_p.html b/htroot/PerformanceMemory_p.html
index e0fac875b..bc7b2cfb6 100644
--- a/htroot/PerformanceMemory_p.html
+++ b/htroot/PerformanceMemory_p.html
@@ -184,19 +184,51 @@
     <table border="0" cellpadding="2" cellspacing="1">

       <tr class="TableHeader" valign="bottom">

         <td>Type</td>

-        <td>Amount</td>

+        <td>Size</td>

+        <td>Hit</td>

+        <td>Miss</td>

+        <td>Insert</td>

+        <td>Delete</td>

       </tr>

       <tr class="TableCellLight">

         <td>DNSCache</td>

-        <td>#[namecache.hit]#</td>

+        <td>#[namecacheHit.size]#</td>

+        <td></td>

+        <td></td>

+        <td></td>

+        <td></td>

+      </tr>

+      <tr class="TableCellLight">

+        <td>DNSCache</td>

+        <td>#[namecacheMiss.size]#</td>

+        <td></td>

+        <td></td>

+        <td></td>

+        <td></td>

       </tr>

       <tr class="TableCellDark">

         <td>DNSNoCache</td>

+        <td></td>

         <td>#[namecache.noCache]#</td>

+        <td></td>

+        <td></td>

+        <td></td>

       </tr>

       <tr class="TableCellLight">

         <td>HashBlacklistedCache</td>

         <td>#[blacklistcache.size]#</td>

+        <td></td>

+        <td></td>

+        <td></td>

+        <td></td>

+      </tr>

+      <tr class="TableCellLight">

+        <td>Search Event Cache</td>

+        <td>#[searchevent.size]#</td>

+        <td>#[searchevent.hit]#</td>

+        <td>#[searchevent.miss]#</td>

+        <td>#[searchevent.insert]#</td>

+        <td>#[searchevent.delete]#</td>

       </tr>

     </table>

     

diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java
index 0b874e7a7..49c01b95a 100644
--- a/htroot/PerformanceMemory_p.java
+++ b/htroot/PerformanceMemory_p.java
@@ -38,6 +38,7 @@ import net.yacy.kelondro.util.FileUtils;
 import net.yacy.kelondro.util.Formatter;

 import net.yacy.kelondro.util.MemoryControl;

 

+import de.anomic.search.SearchEventCache;

 import de.anomic.search.Switchboard;

 import de.anomic.server.serverObjects;

 import de.anomic.server.serverSwitch;

@@ -187,12 +188,15 @@ public class PerformanceMemory_p {
         prop.putNum("objectMissCacheTotalMem", totalmissmem / (1024 * 1024d));

         

         // other caching structures

-        long amount = Domains.nameCacheHitSize();

-        prop.putNum("namecache.hit", amount);

-        amount = Domains.nameCacheNoCachingListSize();

-        prop.putNum("namecache.noCache", amount);

-        amount = Switchboard.urlBlacklist.blacklistCacheSize();

-        prop.putNum("blacklistcache.size", amount);

+        prop.putNum("namecacheHit.size", Domains.nameCacheHitSize());

+        prop.putNum("namecacheMiss.size", Domains.nameCacheMissSize());

+        prop.putNum("namecache.noCache", Domains.nameCacheNoCachingListSize());

+        prop.putNum("blacklistcache.size", Switchboard.urlBlacklist.blacklistCacheSize());

+        prop.putNum("searchevent.size", SearchEventCache.size());

+        prop.putNum("searchevent.hit", SearchEventCache.cacheHit);

+        prop.putNum("searchevent.miss", SearchEventCache.cacheMiss);

+        prop.putNum("searchevent.insert", SearchEventCache.cacheInsert);

+        prop.putNum("searchevent.delete", SearchEventCache.cacheDelete);

         // return rewrite values for templates

         return prop;

     }

diff --git a/source/de/anomic/search/SearchEventCache.java b/source/de/anomic/search/SearchEventCache.java
index dbd51edda..3effceecb 100644
--- a/source/de/anomic/search/SearchEventCache.java
+++ b/source/de/anomic/search/SearchEventCache.java
@@ -26,10 +26,14 @@
 

 package de.anomic.search;

 

-import java.util.Iterator;

+import java.util.ArrayList;

+import java.util.List;

+import java.util.Map;

 import java.util.TreeMap;

-import java.util.concurrent.ConcurrentHashMap;

 

+import net.yacy.cora.storage.ARC;

+import net.yacy.cora.storage.ConcurrentARC;

+import net.yacy.kelondro.util.MemoryControl;

 import net.yacy.repository.LoaderDispatcher;

 

 import de.anomic.crawler.ResultURLs;

@@ -37,33 +41,43 @@ import de.anomic.yacy.yacySeedDB;
 

 public class SearchEventCache {

 

-    private static ConcurrentHashMap<String, SearchEvent> lastEvents = new ConcurrentHashMap<String, SearchEvent>(); // a cache for objects from this class: re-use old search requests

-    public static final long eventLifetime = 60000; // the time an event will stay in the cache, 1 Minute

-

+    private static ARC<String, SearchEvent> lastEvents = new ConcurrentARC<String, SearchEvent>(1000, Runtime.getRuntime().availableProcessors() * 4); // a cache for objects from this class: re-use old search requests

+    public static final long eventLifetime = 600000; // the time an event will stay in the cache, 10 Minutes

+    public static final long memlimit = 100 * 1024 * 1024; // 100 MB

     public static String lastEventID = "";

+    public static long cacheInsert = 0, cacheHit = 0, cacheMiss = 0, cacheDelete = 0;

+    

+    public static int size() {

+        return lastEvents.size();

+    }

     

     public static void put(String eventID, SearchEvent event) {

         lastEventID = eventID;

-        lastEvents.put(eventID, event);

+        SearchEvent oldEvent = lastEvents.put(eventID, event);

+        if (oldEvent == null) cacheInsert++;

     }

     

     public static void cleanupEvents(final boolean all) {

         // remove old events in the event cache

-        final Iterator<SearchEvent> i = lastEvents.values().iterator();

-        SearchEvent event;

-        while (i.hasNext()) {

-            event = i.next();

-            if (all || event.getEventTime() + eventLifetime < System.currentTimeMillis()) {

-                event.cleanup();

-                

-                // remove the event

-                i.remove();

+        List<String> delete = new ArrayList<String>();

+        // the less memory is there, the less time is acceptable for elements in the cache

+        long memx = MemoryControl.available();

+        long acceptTime = memx > memlimit ? eventLifetime : memx  * eventLifetime / memlimit;

+        for (Map.Entry<String, SearchEvent> event: lastEvents) {

+            if (all || event.getValue().getEventTime() + acceptTime < System.currentTimeMillis()) {

+                event.getValue().cleanup();

+                delete.add(event.getKey());

             }

         }

+        // remove the events

+        cacheDelete += delete.size();

+        for (String k: delete) lastEvents.remove(k);

     }

 

     public static SearchEvent getEvent(final String eventID) {

-        return lastEvents.get(eventID);

+        SearchEvent event = lastEvents.get(eventID);

+        if (event == null) cacheMiss++; else cacheHit++;

+        return event;

     }

     

     public static SearchEvent getEvent(

@@ -76,11 +90,13 @@ public class SearchEventCache {
         

         String id = query.id(false);

         SearchEvent event = SearchEventCache.lastEvents.get(id);

+        if (event == null) cacheMiss++; else cacheHit++;

         if (Switchboard.getSwitchboard() != null && !Switchboard.getSwitchboard().crawlQueues.noticeURL.isEmpty() && event != null && System.currentTimeMillis() - event.getEventTime() > 60000) {

             // if a local crawl is ongoing, don't use the result from the cache to use possibly more results that come from the current crawl

             // to prevent that this happens during a person switches between the different result pages, a re-search happens no more than

             // once a minute

             SearchEventCache.lastEvents.remove(id);

+            cacheDelete++;

             event = null;

         } else {

             if (event != null) {