migrated the index export methods from the old metadata to solr. Now

exports are done using solr queries. removed superfluous methods and
servlets.

7 files changed, 19 insertions, 130 deletions

diff --git a/htroot/CrawlResults.java b/htroot/CrawlResults.java
index c0ba43b4c..51020747f 100644
--- a/htroot/CrawlResults.java
+++ b/htroot/CrawlResults.java
@@ -124,10 +124,9 @@ public class CrawlResults {
 

             if (post.containsKey("deletedomain")) {

                 final String domain = post.get("domain", null);

-                final String hashpart = domain == null ? null : DigestURI.hosthash6(domain);

-                if (hashpart != null) {

-                    sb.index.fulltext().deleteDomain(hashpart, null, false);

-                    ResultURLs.deleteDomain(tabletype, domain, hashpart);

+                if (domain != null) {

+                    sb.index.fulltext().deleteDomainHostname(domain, null, false);

+                    ResultURLs.deleteDomain(tabletype, domain);

                 }

             }

 

diff --git a/htroot/Crawler_p.java b/htroot/Crawler_p.java
index 389860108..f5f40a8e8 100644
--- a/htroot/Crawler_p.java
+++ b/htroot/Crawler_p.java
@@ -294,7 +294,7 @@ public class Crawler_p {
                         siteFilter = CrawlProfile.siteFilter(rootURLs);
                         if (deleteold) {
                             for (DigestURI u: rootURLs) {
-                                int count = sb.index.fulltext().deleteDomain(u.hosthash(), deleteageDate, rootURLs.size() > 1);
+                                int count = sb.index.fulltext().deleteDomainHashpart(u.hosthash(), deleteageDate, rootURLs.size() > 1);
                                 if (count > 0) Log.logInfo("Crawler_p", "deleted " + count + " documents for host " + u.getHost());
                             }
                         }
diff --git a/htroot/IndexControlURLs_p.html b/htroot/IndexControlURLs_p.html
index 3f7db9c6f..23b79b53a 100644
--- a/htroot/IndexControlURLs_p.html
+++ b/htroot/IndexControlURLs_p.html
@@ -77,7 +77,6 @@ function updatepage(str) {
         <dt class="TableCellDark">Retrieve by URL-Hash:</dt>

         <dd><input type="text" name="urlhash" value="#[urlhash]#" size="40" maxlength="12" />

             <input type="submit" name="urlhashsearch" value="Show Details for URL-Hash" class="submitready" style="width:240px;"/>

-            <input type="submit" name="urlhashsimilar" value="Generate List" class="submitready" style="width:240px;"/>

         </dd>

       </dl>

     </fieldset>

@@ -132,7 +131,7 @@ function updatepage(str) {
         <td>

           <form action="IndexControlURLs_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">

             <div>

-              <input type="hidden" name="hashpart" value="#[hashpart]#" />

+              <input type="hidden" name="domain" value="#[domain]#" />

               <input type="hidden" name="lines" value="#[lines]#" />

               <input type="submit" name="deletedomain" value="delete all" class="submitready" style="width:240px;"/>

             </div>

@@ -206,13 +205,6 @@ function updatepage(str) {
     <div class="commit">Stored a solr dump to file #[dumpfile]#</div>::

     #(/indexdump)#

     

-    #(urlhashsimilar)#::<p>Sequential List of URL-Hashes:<br />

-    #{rows}# 

-    #{cols}#<a href="/IndexControlURLs_p.html?urlhash=#[urlHash]#&amp;urlhashsearch=1" class="tt">#[urlHash]#</a> #{/cols}#<br />

-    #{/rows}#

-    </p>

-    #(/urlhashsimilar)#

-    

     #(genUrlProfile)#

     ::No entry found for URL-hash #[urlhash]#

     ::<iframe src="/api/yacydoc.html?urlhash=#[urlhash]#" width="100%" height="420" frameborder="0" scrolling="no"></iframe><br />

diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java
index 59ac827e1..bbbb2b480 100644
--- a/htroot/IndexControlURLs_p.java
+++ b/htroot/IndexControlURLs_p.java
@@ -30,13 +30,15 @@ import java.io.IOException;
 import java.net.MalformedURLException;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 
 import net.yacy.cora.date.GenericFormatter;
 import net.yacy.cora.document.ASCII;
+import net.yacy.cora.federate.solr.YaCySchema;
 import net.yacy.cora.federate.yacy.CacheStrategy;
 import net.yacy.cora.lod.JenaTripleStore;
-import net.yacy.cora.order.Base64Order;
 import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.sorting.ReversibleScoreMap;
 import net.yacy.crawler.data.Cache;
 import net.yacy.crawler.data.ResultURLs;
 import net.yacy.data.WorkTables;
@@ -44,7 +46,6 @@ import net.yacy.kelondro.data.meta.DigestURI;
 import net.yacy.kelondro.data.meta.URIMetadataNode;
 import net.yacy.kelondro.data.word.Word;
 import net.yacy.kelondro.logging.Log;
-import net.yacy.kelondro.util.RotateIterator;
 import net.yacy.search.Switchboard;
 import net.yacy.search.index.Fulltext;
 import net.yacy.search.index.Segment;
@@ -236,30 +237,6 @@ public class IndexControlURLs_p {
             }
         }
 
-        // generate list
-        if (post.containsKey("urlhashsimilar")) {
-            final Iterator<DigestURI> entryIt = new RotateIterator<DigestURI>(segment.fulltext().urls(), ASCII.String(Base64Order.zero((urlhash == null ? 0 : urlhash.length()))), (int) segment.RWICount());
-			final StringBuilder result = new StringBuilder("Sequential List of URL-Hashes:<br />");
-			DigestURI entry;
-			int i = 0, rows = 0, cols = 0;
-			prop.put("urlhashsimilar", "1");
-			while (entryIt.hasNext() && i < 256) {
-			    entry = entryIt.next();
-			    if (entry == null) break;
-			    prop.put("urlhashsimilar_rows_"+rows+"_cols_"+cols+"_urlHash", ASCII.String(entry.hash()));
-			    cols++;
-			    if (cols==8) {
-			        prop.put("urlhashsimilar_rows_"+rows+"_cols", cols);
-			        cols = 0;
-			        rows++;
-			    }
-			    i++;
-			}
-			prop.put("statistics", 0);
-			prop.put("urlhashsimilar_rows", rows);
-			prop.put("result", result.toString());
-        }
-
         if (post.containsKey("lurlexport")) {
             // parse format
             int format = 0;
@@ -279,7 +256,7 @@ public class IndexControlURLs_p {
             final File f = new File(s);
             f.getParentFile().mkdirs();
             final String filter = post.get("exportfilter", ".*");
-            final Fulltext.Export running = segment.fulltext().export(f, filter, null, format, dom);
+            final Fulltext.Export running = segment.fulltext().export(f, filter, format, dom);
 
             prop.put("lurlexport_exportfile", s);
             prop.put("lurlexport_urlcount", running.count());
@@ -301,29 +278,29 @@ public class IndexControlURLs_p {
         }
 
         if (post.containsKey("deletedomain")) {
-            final String hp = post.get("hashpart");
-            segment.fulltext().deleteDomain(hp, null, false);
+            final String domain = post.get("domain");
+            segment.fulltext().deleteDomainHostname(domain, null, false);
             // trigger the loading of the table
             post.put("statistics", "");
         }
 
         if (post.containsKey("statistics")) {
             final int count = post.getInt("lines", 100);
-            Iterator<Fulltext.HostStat> statsiter;
             prop.put("statistics_lines", count);
             int cnt = 0;
             try {
                 final Fulltext metadata = segment.fulltext();
-                statsiter = metadata.statistics(count, metadata.urlSampleScores(metadata.domainSampleCollector()));
+                Map<String, ReversibleScoreMap<String>> scores = metadata.getSolr().getFacets(YaCySchema.httpstatus_i.getSolrFieldName() + ":200", count, YaCySchema.host_s.getSolrFieldName());
+                ReversibleScoreMap<String> stats = scores.get(YaCySchema.host_s.getSolrFieldName());
+                Iterator<String> statsiter = stats.keys(false);
                 boolean dark = true;
-                Fulltext.HostStat hs;
+                String hostname;
+                prop.put("statisticslines_domains_" + cnt + "lines", count);
                 while (statsiter.hasNext() && cnt < count) {
-                    hs = statsiter.next();
+                    hostname = statsiter.next();
                     prop.put("statisticslines_domains_" + cnt + "_dark", (dark) ? "1" : "0");
-                    prop.put("statisticslines_domains_" + cnt + "_domain", hs.hostname + ((hs.port == 80) ? "" : ":" + hs.port));
-                    prop.put("statisticslines_domains_" + cnt + "lines", count);
-                    prop.put("statisticslines_domains_" + cnt + "_hashpart", hs.hosthash);
-                    prop.put("statisticslines_domains_" + cnt + "_count", hs.count);
+                    prop.put("statisticslines_domains_" + cnt + "_domain", hostname);
+                    prop.put("statisticslines_domains_" + cnt + "_count", stats.get(hostname));
                     dark = !dark;
                     cnt++;
                 }
diff --git a/htroot/IndexControlURLs_p.xml b/htroot/IndexControlURLs_p.xml
index 79ce730bc..0a9b0a63c 100644
--- a/htroot/IndexControlURLs_p.xml
+++ b/htroot/IndexControlURLs_p.xml
@@ -13,13 +13,4 @@
     #(indexdump)#::
     <dumpfile>#[dumpfile]#</dumpfile>::
     #(/indexdump)#
-    #(urlhashsimilar)#::
-    <urls>
-    #{rows}# 
-    #{cols}#
-    <urlhash>#[urlHash]#</urlhash>
-    #{/cols}#
-    #{/rows}#
-    </urls>
-    #(/urlhashsimilar)#
 </data>
\ No newline at end of file
diff --git a/htroot/YBRFetch_p.html b/htroot/YBRFetch_p.html
deleted file mode 100644
index e69de29bb..000000000
--- a/htroot/YBRFetch_p.html
+++ /dev/null
diff --git a/htroot/YBRFetch_p.java b/htroot/YBRFetch_p.java
deleted file mode 100644
index 9ff84558f..000000000
--- a/htroot/YBRFetch_p.java
+++ /dev/null
@@ -1,70 +0,0 @@
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import net.yacy.cora.protocol.RequestHeader;
-import net.yacy.kelondro.logging.Log;
-import net.yacy.kelondro.rwi.ReferenceContainerCache;
-import net.yacy.kelondro.util.MemoryControl;
-import net.yacy.peers.graphics.WebStructureGraph.HostReference;
-import net.yacy.search.Switchboard;
-import net.yacy.search.index.Fulltext;
-import net.yacy.search.index.Fulltext.HostStat;
-import net.yacy.search.index.Segment;
-import net.yacy.search.ranking.BlockRank;
-import net.yacy.server.serverObjects;
-import net.yacy.server.serverSwitch;
-import net.yacy.server.servletProperties;
-
-public class YBRFetch_p
-{
-
-    public static servletProperties respond(
-        @SuppressWarnings("unused") final RequestHeader requestHeader,
-        final serverObjects post,
-        final serverSwitch env) {
-        final servletProperties prop = new servletProperties();
-        final Switchboard sb = (Switchboard) env;
-
-        if ( post == null || !post.containsKey("ghrt4") || MemoryControl.available() < 1024L * 1024L * 1024L ) {
-            return prop;
-        }
-        final File hostIndexFile = new File(sb.queuesRoot, "hostIndex.blob");
-
-        ReferenceContainerCache<HostReference> hostIndex; // this will get large, more than 0.5 million entries by now
-        if ( !hostIndexFile.exists() ) {
-            hostIndex = BlockRank.collect(sb.peers, sb.webStructure, Integer.MAX_VALUE);
-            BlockRank.saveHostIndex(hostIndex, hostIndexFile);
-        } else {
-            hostIndex = BlockRank.loadHostIndex(hostIndexFile);
-        }
-
-        // use an index segment to find hosts for given host hashes
-        final Segment segment = sb.index;
-        final Fulltext metadata = segment.fulltext();
-        Map<String, HostStat> hostHashResolver;
-        try {
-            hostHashResolver = metadata.domainHashResolver(metadata.domainSampleCollector());
-        } catch ( final IOException e ) {
-            hostHashResolver = new HashMap<String, HostStat>();
-        }
-
-        // recursively compute a new ranking table
-        Log.logInfo("BLOCK RANK", "computing new ranking tables...");
-        BlockRank.ybrTables = BlockRank.evaluate(hostIndex, hostHashResolver, null, 0);
-        hostIndex = null; // we don't need that here any more, so free the memory
-
-        // use the web structure and the hostHash resolver to analyse the ranking table
-        Log.logInfo("BLOCK RANK", "analysis of " + BlockRank.ybrTables.length + " tables...");
-        BlockRank.analyse(sb.webStructure, hostHashResolver);
-        // store the new table
-        Log.logInfo("BLOCK RANK", "storing fresh table...");
-        final File rankingPath = new File(sb.appPath, "ranking/YBR".replace('/', File.separatorChar));
-        BlockRank.storeBlockRankTable(rankingPath);
-        BlockRank.loadBlockRankTable(rankingPath, 16);
-
-        return prop;
-    }
-
-}