this is a re-implementation of the YaCy Block Rank feature

This time it works like this:
- each peer provides its ranking information using the yacy/idx.json servlet
- peers with more than 1 GB ram will load this information from all other peers, combine that into one ranking table and store it locally. This happens during the start-up of the peer concurrently. The new generated file with the ranking information is at DATA/INDEX/<network>/QUEUES/hostIndex.blob
- this index is then computed to generate a new fresh ranking table. Peers which can calculate their own ranking table will do that every start-up to get latest feature updates until the feature is stable
- I computed new ranking tables as part of the distribition and commit it here also
- the YBR feature must be enabled manually by setting the YBR value in the ranking servlet to level 15. A default configuration for that is also in the commit but it does not affect your current installation only fresh peers
- a recursive block rank refinement is implemented but disabled at this point. it needs more testing

Please play around with the ranking settings and see if this helped to make search results better.

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7729 6c8d7289-2bf4-0310-a012-ef5d649a1542

3 files changed, 12 insertions, 8 deletions

diff --git a/htroot/IndexControlRWIs_p.html b/htroot/IndexControlRWIs_p.html
index 367bd7942..8f80f77fb 100644
--- a/htroot/IndexControlRWIs_p.html
+++ b/htroot/IndexControlRWIs_p.html
@@ -178,7 +178,7 @@
       <table border="0" cellpadding="2" cellspacing="1">

       <tr class="TableHeader">

         <td colspan="3">Resource</td>

-        <td colspan="8">Negative Ranking Factors</td>

+        <td colspan="9">Negative Ranking Factors</td>

         <td colspan="8">Positive Ranking Factors</td>

         <td rowspan="2">props</td>

         <td rowspan="2">Reverse Normalized Weighted Ranking Sum</td>

@@ -195,6 +195,7 @@
         <td>pos of phrase</td>

         <td>pos in phrase</td>

         <td>word distance</td>

+        <td>YBR</td>

         <td>term frequency</td>

         <td>authority</td>

         <td>date</td>

@@ -224,6 +225,7 @@
         <td class="TableCellDark">#[phrase]#</td>

         <td class="TableCellDark">#[posinphrase]#</td>

         <td class="TableCellDark">#[worddistance]#</td>

+        <td class="TableCellDark">#[ybr]#</td>

         <td>#[tf]#</td>

         <td>#[authority]#</td>

         <td>#[date]#</td>

diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java
index 08b15ee4d..25c534b7c 100644
--- a/htroot/IndexControlRWIs_p.java
+++ b/htroot/IndexControlRWIs_p.java
@@ -59,6 +59,7 @@ import de.anomic.crawler.ResultURLs;
 import de.anomic.data.ListManager;
 import de.anomic.data.WorkTables;
 import de.anomic.http.client.Cache;
+import de.anomic.search.BlockRank;
 import de.anomic.search.QueryParams;
 import de.anomic.search.RankingProcess;
 import de.anomic.search.ReferenceOrder;
@@ -279,7 +280,7 @@ public class IndexControlRWIs_p {
                 }
                 
                 // make an indexContainerCache
-                ReferenceContainerCache<WordReference> icc = new ReferenceContainerCache<WordReference>(Segment.wordReferenceFactory, index.row(), Segment.wordOrder);
+                ReferenceContainerCache<WordReference> icc = new ReferenceContainerCache<WordReference>(Segment.wordReferenceFactory, Segment.wordOrder, Word.commonHashLength);
                 try {
                     icc.add(index);
                 } catch (RowSpaceExceededException e) {
@@ -442,7 +443,7 @@ public class IndexControlRWIs_p {
                 prop.put("genUrlList_urlList_"+i+"_urlExists_urlStringShort", (us.length() > 40) ? (us.substring(0, 20) + "<br>" + us.substring(20,  40) + "...") : ((us.length() > 30) ? (us.substring(0, 20) + "<br>" + us.substring(20)) : us));
                 prop.putNum("genUrlList_urlList_"+i+"_urlExists_ranking", (entry.ranking() - rn));
                 prop.putNum("genUrlList_urlList_"+i+"_urlExists_domlength", DigestURI.domLengthEstimation(entry.hash()));
-                prop.putNum("genUrlList_urlList_"+i+"_urlExists_ybr", RankingProcess.ybr(entry.hash()));
+                prop.putNum("genUrlList_urlList_"+i+"_urlExists_ybr", BlockRank.ranking(entry.hash()));
                 prop.putNum("genUrlList_urlList_"+i+"_urlExists_tf", 1000.0 * entry.word().termFrequency());
                 prop.putNum("genUrlList_urlList_"+i+"_urlExists_authority", (ranked.getOrder() == null) ? -1 : ranked.getOrder().authority(entry.hash()));
                 prop.put("genUrlList_urlList_"+i+"_urlExists_date", GenericFormatter.SHORT_DAY_FORMATTER.format(new Date(entry.word().lastModified())));
@@ -453,6 +454,7 @@ public class IndexControlRWIs_p {
                 prop.putNum("genUrlList_urlList_"+i+"_urlExists_lother", entry.word().lother());
                 prop.putNum("genUrlList_urlList_"+i+"_urlExists_hitcount", entry.word().hitcount());
                 prop.putNum("genUrlList_urlList_"+i+"_urlExists_worddistance", 0);
+                prop.putNum("genUrlList_urlList_"+i+"_urlExists_ybr", BlockRank.ranking(entry.hash()));
                 prop.putNum("genUrlList_urlList_"+i+"_urlExists_pos", entry.word().minposition());
                 prop.putNum("genUrlList_urlList_"+i+"_urlExists_phrase", entry.word().posofphrase());
                 prop.putNum("genUrlList_urlList_"+i+"_urlExists_posinphrase", entry.word().posinphrase());
diff --git a/htroot/yacy/idx.java b/htroot/yacy/idx.java
index 09a4a4cce..42813617f 100644
--- a/htroot/yacy/idx.java
+++ b/htroot/yacy/idx.java
@@ -41,15 +41,15 @@ public final class idx {
     // http://localhost:8090/yacy/idx.json?object=host
     
     public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
-        if (post == null || env == null) { return null; }
-
-        // return variable that accumulates replacements
-        final Switchboard sb = (Switchboard) env;
         final serverObjects prop = new serverObjects();
 
         prop.put("list", 0);
         prop.put("rowdef","");
         prop.put("name","");
+        if (post == null || env == null) return prop;
+
+        // return variable that accumulates replacements
+        final Switchboard sb = (Switchboard) env;
         
         if (sb.adminAuthenticated(header) < 2 && !yacyNetwork.authentifyRequest(post, env)) {
             return prop;
@@ -58,7 +58,7 @@ public final class idx {
         if (post.get("object", "").equals("host")) {
             prop.put("name","host");
             ReferenceContainerCache<HostReference> idx = sb.webStructure.incomingReferences();
-            prop.put("rowdef", WebStructureGraph.hostReferenceFacory.getRow().toString());
+            prop.put("rowdef", WebStructureGraph.hostReferenceFactory.getRow().toString());
             int count = 0;
             for (ReferenceContainer<HostReference> references: idx) {
                 prop.put("list_" + count + "_term", UTF8.String(references.getTermHash()));