replaced url and date retrieval by only url retrieval

This should prevent that the search index is used for freshnes of the
index entry.

4 files changed, 152 insertions, 149 deletions

diff --git a/htroot/api/citation.java b/htroot/api/citation.java
index 8479b55f3..29a13c9de 100644
--- a/htroot/api/citation.java
+++ b/htroot/api/citation.java
@@ -90,11 +90,12 @@ public class citation {
         }
         if (uri == null && hash.length() > 0) {
             try {
-                uri = sb.getURL(ASCII.getBytes(hash));
-                if (uri == null) {
+                String u = sb.getURL(ASCII.getBytes(hash));
+                if (u == null) {
                     connector.commit(true); // try again, that url can be fresh
-                    uri = sb.getURL(ASCII.getBytes(hash));
+                    u = sb.getURL(ASCII.getBytes(hash));
                 }
+                if (u != null) uri = new DigestURL(u);
             } catch (IOException e) {
                 ConcurrentLog.logException(e);
             }
@@ -102,7 +103,7 @@ public class citation {
         if (uri == null) return prop; // no proper url addressed
         url = uri.toNormalform(true);
         prop.put("url", url);
-        
+
         // get the document from the index
         SolrDocument doc;
         try {
@@ -149,11 +150,11 @@ public class citation {
                     sentenceOcc.put(sentence, list);
                 }
             } catch (final Throwable ee) {
-                
+
             }
         }
         sentences.clear(); // we do not need this again
-        
+
         // iterate the sentences
         int i = 0;
         int sentenceNr = 0;
@@ -189,7 +190,7 @@ public class citation {
             sentenceNr++;
         }
         prop.put("sentences", i);
-        
+
         // iterate the citations in order of number of citations
         i = 0;
         for (String u: scores.keyList(false)) {
@@ -207,7 +208,7 @@ public class citation {
             } catch (final MalformedURLException e) {}
         }
         prop.put("citations", i);
-        
+
         // find similar documents from different hosts
         i = 0;
         for (String u: scores.keyList(false)) {
@@ -221,7 +222,7 @@ public class citation {
         }
         prop.put("similar_links", i);
         prop.put("similar", i > 0 ? 1 : 0);
-        
+
         // return rewrite properties
         return prop;
     }
diff --git a/htroot/api/linkstructure.java b/htroot/api/linkstructure.java
index d69271fbe..62e85f92f 100644
--- a/htroot/api/linkstructure.java
+++ b/htroot/api/linkstructure.java
@@ -62,7 +62,8 @@ public class linkstructure {
             if (about.length() == 12 && Base64Order.enhancedCoder.wellformed(ASCII.getBytes(about))) {
                 byte[] urlhash = ASCII.getBytes(about);
                 try {
-                    url = authenticated ? sb.getURL(urlhash) : null;
+                    String u = authenticated ? sb.getURL(urlhash) : null;
+                    url = u == null ? null : new DigestURL(u);
                 } catch (IOException e) {
                     ConcurrentLog.logException(e);
                 }
diff --git a/htroot/api/snapshot.java b/htroot/api/snapshot.java
index 8ffb7a883..a47475810 100644
--- a/htroot/api/snapshot.java
+++ b/htroot/api/snapshot.java
@@ -60,7 +60,7 @@ import net.yacy.server.serverObjects;
 import net.yacy.server.serverSwitch;
 
 public class snapshot {
-    
+
     //width = 1024, height = 1024, density = 300, quality = 75
     private final static int DEFAULT_WIDTH = 1024;
     private final static int DEFAULT_HEIGHT = 1024;
@@ -70,19 +70,19 @@ public class snapshot {
 
     public static Object respond(final RequestHeader header, serverObjects post, final serverSwitch env) {
         final Switchboard sb = (Switchboard) env;
-        
+
     	final serverObjects defaultResponse = new serverObjects();
-    	
+
 
         final boolean authenticated = sb.adminAuthenticated(header) >= 2;
         final String ext = header.get(HeaderFramework.CONNECTION_PROP_EXT, "");
-        
+
         if(ext.isEmpty()) {
 			throw new TemplateProcessingException("Missing extension. Try with rss, xml, json, pdf, png or jpg." + ext,
 					HttpStatus.SC_BAD_REQUEST);
         }
-        
-        
+
+
         if (ext.equals("rss")) {
             // create a report about the content of the snapshot directory
             if (!authenticated) {
@@ -103,9 +103,9 @@ public class snapshot {
             rssfeed.setChannel(new RSSMessage("Snapshot list for host = " + host + ", depth = " + depth + ", order = " + order + ", maxcount = " + maxcount, "", ""));
             for (Map.Entry<String, Revisions> e: iddate.entrySet()) {
                 try {
-                    DigestURL u = e.getValue().url == null ? sb.index.fulltext().getURL(e.getKey()) : new DigestURL(e.getValue().url);
+                    String u = e.getValue().url == null ? sb.index.fulltext().getURL(e.getKey()) : e.getValue().url;
                     if (u == null) continue;
-                    RSSMessage message = new RSSMessage(u.toNormalform(true), "", u, e.getKey());
+                    RSSMessage message = new RSSMessage(u, "", new DigestURL(u), e.getKey());
                     message.setPubDate(e.getValue().dates[0]);
                     rssfeed.addMessage(message);
                 } catch (IOException ee) {
@@ -137,7 +137,8 @@ public class snapshot {
         }
         if (durl == null && urlhash.length() > 0) {
             try {
-                durl = sb.index.fulltext().getURL(urlhash);
+                String u = sb.index.fulltext().getURL(urlhash);
+                durl = u == null ? null : new DigestURL(u);
             } catch (IOException e) {
                 ConcurrentLog.logException(e);
             }
@@ -183,8 +184,8 @@ public class snapshot {
                                     for (Revisions r: entry.getValue()) {
                                         try {
                                             JSONObject metadata = new JSONObject();
-                                            DigestURL u = r.url != null ? new DigestURL(r.url) : sb.index.fulltext().getURL(r.urlhash);
-                                            metadata.put("url", u == null ? "unknown" : u.toNormalform(true));
+                                            String u = r.url != null ? r.url : sb.index.fulltext().getURL(r.urlhash);
+                                            metadata.put("url", u == null ? "unknown" : u);
                                             metadata.put("dates", r.dates);
                                             assert r.depth == entry.getKey().intValue();
                                             metadata.put("depth", entry.getKey().intValue());
@@ -239,9 +240,8 @@ public class snapshot {
                         }
                         if (r != null) {
                             JSONObject metadata = new JSONObject();
-                            DigestURL u;
-                            u = r.url != null ? new DigestURL(r.url) : sb.index.fulltext().getURL(r.urlhash);
-                            metadata.put("url", u == null ? "unknown" : u.toNormalform(true));
+                            String u = r.url != null ? r.url : sb.index.fulltext().getURL(r.urlhash);
+                            metadata.put("url", u == null ? "unknown" : u);
                             metadata.put("dates", r.dates);
                             metadata.put("depth", r.depth);
                             metadata.put("state", state.name());
@@ -256,12 +256,12 @@ public class snapshot {
             if (post.containsKey("callback")) json = post.get("callback") + "([" + json + "]);";
             return new ByteArrayInputStream(UTF8.getBytes(json));
         }
-        
+
         // for the following methods we always need the durl to fetch data
         if (durl == null) {
         	throw new TemplateMissingParameterException("Missing valid url or urlhash parameter");
         }
-        
+
         if (xml) {
             Collection<File> xmlSnapshots = Transactions.findPaths(durl, "xml", Transactions.State.ANY);
             File xmlFile = null;
@@ -277,7 +277,7 @@ public class snapshot {
                 throw new TemplateProcessingException("Could not read the xml snapshot file.");
             }
         }
-        
+
         if (pdf || pngjpg) {
             Collection<File> pdfSnapshots = Transactions.findPaths(durl, "pdf", Transactions.State.INVENTORY);
             File pdfFile = null;
@@ -319,7 +319,7 @@ public class snapshot {
 					throw new TemplateProcessingException("Could not read the pdf snapshot file.");
                 }
             }
-            
+
             if (pngjpg) {
                 int width = Math.min(post.getInt("width", DEFAULT_WIDTH), DEFAULT_WIDTH);
                 int height = Math.min(post.getInt("height", DEFAULT_HEIGHT), DEFAULT_HEIGHT);
@@ -357,7 +357,7 @@ public class snapshot {
                     final MediaTracker mediaTracker = new MediaTracker(new Container());
                     mediaTracker.addImage(scaled, 0);
                     try {mediaTracker.waitForID(0);} catch (final InterruptedException e) {}
-                    
+
 					/*
 					 * Ensure there is no alpha component on the ouput image, as it is pointless
 					 * here and it is not well supported by the JPEGImageWriter from OpenJDK
@@ -369,10 +369,10 @@ public class snapshot {
                     ConcurrentLog.logException(e);
 					throw new TemplateProcessingException("Could not scale the " + ext + " image snapshot file.");
                 }
-    
+
             }
         }
-        
+
 		throw new TemplateProcessingException(
 				"Unsupported extension : " + ext + ". Try with rss, xml, json, pdf, png or jpg.",
 				HttpStatus.SC_BAD_REQUEST);
diff --git a/htroot/api/webstructure.java b/htroot/api/webstructure.java
index 84eec8f53..e1cbf59d9 100644
--- a/htroot/api/webstructure.java
+++ b/htroot/api/webstructure.java
@@ -52,64 +52,64 @@ import net.yacy.server.serverSwitch;
  */
 public class webstructure {
 
-	/**
-	 * <p>Retrieve the locally known web links structure of a specified resource ("about" parameter supplied) or
-	 * the whole computed links structure since install (no parameter supplied)
-	 * or since last start or last call ("latest" parameter supplied).</p>
-	 * <p>Returned object contains the following information :
-	 * <ul>
-	 * 	<li>in all cases :
-	 * 		<ul>
-	 * 			<li>accumulated list of outgoing links to other domains (per host accumulated anchors)</li>
-	 * 		</ul>
-	 * 	</li>
-	 *  <li>when "about" parameter is filled :
-	 *  	<ul>
-	 * 			<li>accumulated list of incoming links from other domains (per host accumulated references)</li>
-	 * 			<li>detailed list of outgoing links (anchors) from document at "about" URL to references</li>
-     * 			<li>detailed list of incoming links (citations) from other documents (their references) - reverse link structure</li>
-     * 		</ul>
-     * 	</li>
-	 * </ul>
-	 * <p>
-	 * Remarks :
-	 * <ul>
-	 * <li>Information detail is limited by {@link WebStructureGraph#maxhosts}, {@link WebStructureGraph#maxref} and {@link WebStructureGraph#MAX_PARSED_ANCHORS} constants.</li>
-	 * <li>Requesting client must be authenticated (as admin or requesting from localhost enabled) otherwise results will be empty</li>
-	 * </ul>
-	 * </p>
-	 * 
-	 * <p>
-	 * Example API calls :
-	 * <ul>
-	 * <li>domain name and index page structure : http://localhost:8090/api/webstructure.xml?about=yacy.net</li>
-	 * <li>domain name structure : http://localhost:8090/api/webstructure.xml?about=yacy.net&documentStructure=false</li>
+    /**
+     * <p>Retrieve the locally known web links structure of a specified resource ("about" parameter supplied) or
+     * the whole computed links structure since install (no parameter supplied)
+     * or since last start or last call ("latest" parameter supplied).</p>
+     * <p>Returned object contains the following information :
+     * <ul>
+     *     <li>in all cases :
+     *         <ul>
+     *             <li>accumulated list of outgoing links to other domains (per host accumulated anchors)</li>
+     *         </ul>
+     *     </li>
+     *  <li>when "about" parameter is filled :
+     *      <ul>
+     *             <li>accumulated list of incoming links from other domains (per host accumulated references)</li>
+     *             <li>detailed list of outgoing links (anchors) from document at "about" URL to references</li>
+     *             <li>detailed list of incoming links (citations) from other documents (their references) - reverse link structure</li>
+     *         </ul>
+     *     </li>
+     * </ul>
+     * <p>
+     * Remarks :
+     * <ul>
+     * <li>Information detail is limited by {@link WebStructureGraph#maxhosts}, {@link WebStructureGraph#maxref} and {@link WebStructureGraph#MAX_PARSED_ANCHORS} constants.</li>
+     * <li>Requesting client must be authenticated (as admin or requesting from localhost enabled) otherwise results will be empty</li>
+     * </ul>
+     * </p>
+     *
+     * <p>
+     * Example API calls :
+     * <ul>
+     * <li>domain name and index page structure : http://localhost:8090/api/webstructure.xml?about=yacy.net</li>
+     * <li>domain name structure : http://localhost:8090/api/webstructure.xml?about=yacy.net&documentStructure=false</li>
      * <li>hosts accumulated structure and specific resource structure : http://localhost:8090/api/webstructure.xml?about=http://yacy.net/fr/API.html</li>
-	 * <li>whole locally known hosts web structure : http://localhost:8090/api/webstructure.xml</li>
-	 * <li>recently locally computed hosts web structure : http://localhost:8090/api/webstructure.xml?latest=</li>
-	 * </ul>
-	 * </p>
-	 * 
-	 * 
-	 * @param header
-	 *            servlet request header
-	 * @param post
-	 *            request parameters. Supported keys :
-	 *            <ul>
-	 *            <li>about : get only links structure about the resource
-	 *            specified as value. Supported values : host hash, URL hash,
-	 *            host name or URL</li>
-	 *            <li>latest (ignored when about parameter is valued): get the structure that have been computed during
-	 *            the current run-time of YaCy, and with each next call only an
-	 *            update to the next list of references.</li>
-	 *            <li>agentName : name of the user agent string used to load the "about" resource</li>
-	 *            <li>documentStructure : set to false when you only want the hosts accumulated references for the "about" resource</li>
-	 *            </ul>
-	 * @param env
-	 *            server environment
-	 * @return the servlet answer object
-	 * @see WebStructureGraph
-	 */
+     * <li>whole locally known hosts web structure : http://localhost:8090/api/webstructure.xml</li>
+     * <li>recently locally computed hosts web structure : http://localhost:8090/api/webstructure.xml?latest=</li>
+     * </ul>
+     * </p>
+     *
+     *
+     * @param header
+     *            servlet request header
+     * @param post
+     *            request parameters. Supported keys :
+     *            <ul>
+     *            <li>about : get only links structure about the resource
+     *            specified as value. Supported values : host hash, URL hash,
+     *            host name or URL</li>
+     *            <li>latest (ignored when about parameter is valued): get the structure that have been computed during
+     *            the current run-time of YaCy, and with each next call only an
+     *            update to the next list of references.</li>
+     *            <li>agentName : name of the user agent string used to load the "about" resource</li>
+     *            <li>documentStructure : set to false when you only want the hosts accumulated references for the "about" resource</li>
+     *            </ul>
+     * @param env
+     *            server environment
+     * @return the servlet answer object
+     * @see WebStructureGraph
+     */
     public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
         final serverObjects prop = new serverObjects();
         final Switchboard sb = (Switchboard) env;
@@ -124,25 +124,26 @@ public class webstructure {
             byte[] urlhash = null;
             Set<String> hostHashes = new HashSet<>();
             if (about.length() == 6 && Base64Order.enhancedCoder.wellformed(ASCII.getBytes(about))) {
-            	hostHashes.add(about);
+                hostHashes.add(about);
             } else if (about.length() == 12 && Base64Order.enhancedCoder.wellformed(ASCII.getBytes(about))) {
-            	urlhash = ASCII.getBytes(about);
-            	hostHashes.add(about.substring(6));
-            	try {
-                    url = authenticated ? sb.getURL(urlhash) : null;
+                urlhash = ASCII.getBytes(about);
+                hostHashes.add(about.substring(6));
+                if (authenticated) try {
+                    String u = sb.getURL(urlhash);
+                    url = u == null ? null : new DigestURL(u);
                 } catch (IOException e) {
                     url = null;
                     ConcurrentLog.logException(e);
                 }
             } else if (about.length() > 0) {
-            	// consider "about" as url or hostname
+                // consider "about" as url or hostname
                 try {
                     url = new DigestURL(about.indexOf("://") >= 0 ? about : "http://" + about); // accept also domains
                     urlhash = url.hash();
                     if(about.indexOf("://") >= 0) {
-                    	hostHashes.add(url.hosthash());
+                        hostHashes.add(url.hosthash());
                     } else {
-                    	hostHashes.addAll(sb.webStructure.hostName2HostHashes(about));
+                        hostHashes.addAll(sb.webStructure.hostName2HostHashes(about));
                     }
                 } catch (final MalformedURLException e) {
                 }
@@ -152,37 +153,37 @@ public class webstructure {
                 prop.put("in", 1);
                 int inCount = 0, outCount = 0;
                 for(final String hostHash: hostHashes) {
-                	WebStructureGraph.StructureEntry sentry = sb.webStructure.outgoingReferences(hostHash);
-                	if (sentry != null && sentry.references.size() > 0) {
-                		reference(prop, "out", outCount, sentry, sb.webStructure);
-                		outCount++;
-                	} else {
-                		prop.put("out_domains", 0);
-                	}
-                	sentry = sb.webStructure.incomingReferences(hostHash);
-                	if (sentry != null && sentry.references.size() > 0) {
-                		reference(prop, "in", inCount, sentry, sb.webStructure);
-                		prop.put("in_domains", 1);
-                		inCount++;
-                	} else {
-                		prop.put("in_domains", 0);
-                	}
+                    WebStructureGraph.StructureEntry sentry = sb.webStructure.outgoingReferences(hostHash);
+                    if (sentry != null && sentry.references.size() > 0) {
+                        reference(prop, "out", outCount, sentry, sb.webStructure);
+                        outCount++;
+                    } else {
+                        prop.put("out_domains", 0);
+                    }
+                    sentry = sb.webStructure.incomingReferences(hostHash);
+                    if (sentry != null && sentry.references.size() > 0) {
+                        reference(prop, "in", inCount, sentry, sb.webStructure);
+                        prop.put("in_domains", 1);
+                        inCount++;
+                    } else {
+                        prop.put("in_domains", 0);
+                    }
                 }
-        		prop.put("out_domains", outCount);
-        		prop.put("in_domains", inCount);
+                prop.put("out_domains", outCount);
+                prop.put("in_domains", inCount);
             }
-            
-			/*
-			 * It is possible not to scrape document and look for citations by
-			 * setting documentStructure parameter to "false"
-			 */
-			boolean documentStructure = true;
-			if (post != null && "false".equals(post.get("documentStructure", "true"))) {
-				documentStructure = false;
-			}
-            
+
+            /*
+             * It is possible not to scrape document and look for citations by
+             * setting documentStructure parameter to "false"
+             */
+            boolean documentStructure = true;
+            if (post != null && "false".equals(post.get("documentStructure", "true"))) {
+                documentStructure = false;
+            }
+
             if (urlhash != null && documentStructure) {
-            	// anchors
+                // anchors
                 prop.put("references", 1);
                 net.yacy.document.Document scraper = null;
                 if (url != null) try {
@@ -201,36 +202,36 @@ public class webstructure {
                     if (url != null) prop.putXML("references_documents_0_urle_url", url.toNormalform(true));
                     int d = 0;
                     Iterator<DigestURL> i = scraper.inboundLinks().keySet().iterator();
-            		while (i.hasNext()) {
-            			DigestURL refurl = i.next();
-                    	byte[] refhash = refurl.hash();
-                    	prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true));
-                    	prop.put("references_documents_0_anchors_" + d + "_hash", refhash);
-                    	prop.put("references_documents_0_anchors_" + d + "_outbound", 0);
-                    	d++;
-            		}
+                    while (i.hasNext()) {
+                        DigestURL refurl = i.next();
+                        byte[] refhash = refurl.hash();
+                        prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true));
+                        prop.put("references_documents_0_anchors_" + d + "_hash", refhash);
+                        prop.put("references_documents_0_anchors_" + d + "_outbound", 0);
+                        d++;
+                    }
                     i = scraper.outboundLinks().keySet().iterator();
-            		while (i.hasNext()) {
-            			DigestURL refurl = i.next();
-                    	byte[] refhash = refurl.hash();
-                    	prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true));
-                    	prop.put("references_documents_0_anchors_" + d + "_hash", refhash);
-                    	prop.put("references_documents_0_anchors_" + d + "_outbound", 1);
-                    	d++;
-            		}
+                    while (i.hasNext()) {
+                        DigestURL refurl = i.next();
+                        byte[] refhash = refurl.hash();
+                        prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true));
+                        prop.put("references_documents_0_anchors_" + d + "_hash", refhash);
+                        prop.put("references_documents_0_anchors_" + d + "_outbound", 1);
+                        d++;
+                    }
                     prop.put("references_documents_0_count", d);
                     prop.put("references_documents_0_anchors", d);
                 } else {
                     prop.put("references_count", 0);
                     prop.put("references_documents", 0);
-            	}
+                }
 
                 // citations
                 prop.put("citations", 1);
                 ReferenceReportCache rrc = sb.index.getReferenceReportCache();
                 ReferenceReport rr = null;
                 try {rr = rrc.getReferenceReport(ASCII.String(urlhash), true);} catch (IOException e) {}
-            	if (rr != null && rr.getInternalCount() > 0 && rr.getExternalCount() > 0) {
+                if (rr != null && rr.getInternalCount() > 0 && rr.getExternalCount() > 0) {
                     prop.put("citations_count", 1);
                     prop.put("citations_documents", 1);
                     prop.put("citations_documents_0_hash", urlhash);
@@ -242,26 +243,26 @@ public class webstructure {
                     HandleSet ids = rr.getInternallIDs();
                     try {ids.putAll(rr.getExternalIDs());} catch (SpaceExceededException e) {}
                     Iterator<byte[]> i = ids.iterator();
-            		while (i.hasNext()) {
-                    	byte[] refhash = i.next();
-                    	DigestURL refurl;
+                    while (i.hasNext()) {
+                        byte[] refhash = i.next();
+                        String refurl;
                         try {
                             refurl = authenticated ? sb.getURL(refhash) : null;
                             prop.put("citations_documents_0_anchors_" + d + "_urle", refurl == null ? 0 : 1);
-                            if (refurl != null) prop.putXML("citations_documents_0_anchors_" + d + "_urle_url", refurl.toNormalform(true));
+                            if (refurl != null) prop.putXML("citations_documents_0_anchors_" + d + "_urle_url", refurl);
                             prop.put("citations_documents_0_anchors_" + d + "_urle_hash", refhash);
                             prop.put("citations_documents_0_anchors_" + d + "_urle_date", GenericFormatter.SHORT_DAY_FORMATTER.format(new Date())); // superfluous?
                             d++;
                         } catch (IOException e) {
                             ConcurrentLog.logException(e);
                         }
-            		}
+                    }
                     prop.put("citations_documents_0_count", d);
                     prop.put("citations_documents_0_anchors", d);
-            	} else {
+                } else {
                     prop.put("citations_count", 0);
                     prop.put("citations_documents", 0);
-            	}
+                }
             }
         } else if (authenticated) {
             // show a complete list of link structure informations in case that the user is authenticated