diff options
Diffstat (limited to 'htroot/api/webstructure.java')
| -rw-r--r-- | htroot/api/webstructure.java | 237 |
1 files changed, 119 insertions, 118 deletions
diff --git a/htroot/api/webstructure.java b/htroot/api/webstructure.java index 84eec8f53..e1cbf59d9 100644 --- a/htroot/api/webstructure.java +++ b/htroot/api/webstructure.java @@ -52,64 +52,64 @@ import net.yacy.server.serverSwitch; */ public class webstructure { - /** - * <p>Retrieve the locally known web links structure of a specified resource ("about" parameter supplied) or - * the whole computed links structure since install (no parameter supplied) - * or since last start or last call ("latest" parameter supplied).</p> - * <p>Returned object contains the following information : - * <ul> - * <li>in all cases : - * <ul> - * <li>accumulated list of outgoing links to other domains (per host accumulated anchors)</li> - * </ul> - * </li> - * <li>when "about" parameter is filled : - * <ul> - * <li>accumulated list of incoming links from other domains (per host accumulated references)</li> - * <li>detailed list of outgoing links (anchors) from document at "about" URL to references</li> - * <li>detailed list of incoming links (citations) from other documents (their references) - reverse link structure</li> - * </ul> - * </li> - * </ul> - * <p> - * Remarks : - * <ul> - * <li>Information detail is limited by {@link WebStructureGraph#maxhosts}, {@link WebStructureGraph#maxref} and {@link WebStructureGraph#MAX_PARSED_ANCHORS} constants.</li> - * <li>Requesting client must be authenticated (as admin or requesting from localhost enabled) otherwise results will be empty</li> - * </ul> - * </p> - * - * <p> - * Example API calls : - * <ul> - * <li>domain name and index page structure : http://localhost:8090/api/webstructure.xml?about=yacy.net</li> - * <li>domain name structure : http://localhost:8090/api/webstructure.xml?about=yacy.net&documentStructure=false</li> + /** + * <p>Retrieve the locally known web links structure of a specified resource ("about" parameter supplied) or + * the whole computed links structure since install (no parameter supplied) + * or since last start or last call ("latest" parameter supplied).</p> + * <p>Returned object contains the following information : + * <ul> + * <li>in all cases : + * <ul> + * <li>accumulated list of outgoing links to other domains (per host accumulated anchors)</li> + * </ul> + * </li> + * <li>when "about" parameter is filled : + * <ul> + * <li>accumulated list of incoming links from other domains (per host accumulated references)</li> + * <li>detailed list of outgoing links (anchors) from document at "about" URL to references</li> + * <li>detailed list of incoming links (citations) from other documents (their references) - reverse link structure</li> + * </ul> + * </li> + * </ul> + * <p> + * Remarks : + * <ul> + * <li>Information detail is limited by {@link WebStructureGraph#maxhosts}, {@link WebStructureGraph#maxref} and {@link WebStructureGraph#MAX_PARSED_ANCHORS} constants.</li> + * <li>Requesting client must be authenticated (as admin or requesting from localhost enabled) otherwise results will be empty</li> + * </ul> + * </p> + * + * <p> + * Example API calls : + * <ul> + * <li>domain name and index page structure : http://localhost:8090/api/webstructure.xml?about=yacy.net</li> + * <li>domain name structure : http://localhost:8090/api/webstructure.xml?about=yacy.net&documentStructure=false</li> * <li>hosts accumulated structure and specific resource structure : http://localhost:8090/api/webstructure.xml?about=http://yacy.net/fr/API.html</li> - * <li>whole locally known hosts web structure : http://localhost:8090/api/webstructure.xml</li> - * <li>recently locally computed hosts web structure : http://localhost:8090/api/webstructure.xml?latest=</li> - * </ul> - * </p> - * - * - * @param header - * servlet request header - * @param post - * request parameters. Supported keys : - * <ul> - * <li>about : get only links structure about the resource - * specified as value. Supported values : host hash, URL hash, - * host name or URL</li> - * <li>latest (ignored when about parameter is valued): get the structure that have been computed during - * the current run-time of YaCy, and with each next call only an - * update to the next list of references.</li> - * <li>agentName : name of the user agent string used to load the "about" resource</li> - * <li>documentStructure : set to false when you only want the hosts accumulated references for the "about" resource</li> - * </ul> - * @param env - * server environment - * @return the servlet answer object - * @see WebStructureGraph - */ + * <li>whole locally known hosts web structure : http://localhost:8090/api/webstructure.xml</li> + * <li>recently locally computed hosts web structure : http://localhost:8090/api/webstructure.xml?latest=</li> + * </ul> + * </p> + * + * + * @param header + * servlet request header + * @param post + * request parameters. Supported keys : + * <ul> + * <li>about : get only links structure about the resource + * specified as value. Supported values : host hash, URL hash, + * host name or URL</li> + * <li>latest (ignored when about parameter is valued): get the structure that have been computed during + * the current run-time of YaCy, and with each next call only an + * update to the next list of references.</li> + * <li>agentName : name of the user agent string used to load the "about" resource</li> + * <li>documentStructure : set to false when you only want the hosts accumulated references for the "about" resource</li> + * </ul> + * @param env + * server environment + * @return the servlet answer object + * @see WebStructureGraph + */ public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { final serverObjects prop = new serverObjects(); final Switchboard sb = (Switchboard) env; @@ -124,25 +124,26 @@ public class webstructure { byte[] urlhash = null; Set<String> hostHashes = new HashSet<>(); if (about.length() == 6 && Base64Order.enhancedCoder.wellformed(ASCII.getBytes(about))) { - hostHashes.add(about); + hostHashes.add(about); } else if (about.length() == 12 && Base64Order.enhancedCoder.wellformed(ASCII.getBytes(about))) { - urlhash = ASCII.getBytes(about); - hostHashes.add(about.substring(6)); - try { - url = authenticated ? sb.getURL(urlhash) : null; + urlhash = ASCII.getBytes(about); + hostHashes.add(about.substring(6)); + if (authenticated) try { + String u = sb.getURL(urlhash); + url = u == null ? null : new DigestURL(u); } catch (IOException e) { url = null; ConcurrentLog.logException(e); } } else if (about.length() > 0) { - // consider "about" as url or hostname + // consider "about" as url or hostname try { url = new DigestURL(about.indexOf("://") >= 0 ? about : "http://" + about); // accept also domains urlhash = url.hash(); if(about.indexOf("://") >= 0) { - hostHashes.add(url.hosthash()); + hostHashes.add(url.hosthash()); } else { - hostHashes.addAll(sb.webStructure.hostName2HostHashes(about)); + hostHashes.addAll(sb.webStructure.hostName2HostHashes(about)); } } catch (final MalformedURLException e) { } @@ -152,37 +153,37 @@ public class webstructure { prop.put("in", 1); int inCount = 0, outCount = 0; for(final String hostHash: hostHashes) { - WebStructureGraph.StructureEntry sentry = sb.webStructure.outgoingReferences(hostHash); - if (sentry != null && sentry.references.size() > 0) { - reference(prop, "out", outCount, sentry, sb.webStructure); - outCount++; - } else { - prop.put("out_domains", 0); - } - sentry = sb.webStructure.incomingReferences(hostHash); - if (sentry != null && sentry.references.size() > 0) { - reference(prop, "in", inCount, sentry, sb.webStructure); - prop.put("in_domains", 1); - inCount++; - } else { - prop.put("in_domains", 0); - } + WebStructureGraph.StructureEntry sentry = sb.webStructure.outgoingReferences(hostHash); + if (sentry != null && sentry.references.size() > 0) { + reference(prop, "out", outCount, sentry, sb.webStructure); + outCount++; + } else { + prop.put("out_domains", 0); + } + sentry = sb.webStructure.incomingReferences(hostHash); + if (sentry != null && sentry.references.size() > 0) { + reference(prop, "in", inCount, sentry, sb.webStructure); + prop.put("in_domains", 1); + inCount++; + } else { + prop.put("in_domains", 0); + } } - prop.put("out_domains", outCount); - prop.put("in_domains", inCount); + prop.put("out_domains", outCount); + prop.put("in_domains", inCount); } - - /* - * It is possible not to scrape document and look for citations by - * setting documentStructure parameter to "false" - */ - boolean documentStructure = true; - if (post != null && "false".equals(post.get("documentStructure", "true"))) { - documentStructure = false; - } - + + /* + * It is possible not to scrape document and look for citations by + * setting documentStructure parameter to "false" + */ + boolean documentStructure = true; + if (post != null && "false".equals(post.get("documentStructure", "true"))) { + documentStructure = false; + } + if (urlhash != null && documentStructure) { - // anchors + // anchors prop.put("references", 1); net.yacy.document.Document scraper = null; if (url != null) try { @@ -201,36 +202,36 @@ public class webstructure { if (url != null) prop.putXML("references_documents_0_urle_url", url.toNormalform(true)); int d = 0; Iterator<DigestURL> i = scraper.inboundLinks().keySet().iterator(); - while (i.hasNext()) { - DigestURL refurl = i.next(); - byte[] refhash = refurl.hash(); - prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true)); - prop.put("references_documents_0_anchors_" + d + "_hash", refhash); - prop.put("references_documents_0_anchors_" + d + "_outbound", 0); - d++; - } + while (i.hasNext()) { + DigestURL refurl = i.next(); + byte[] refhash = refurl.hash(); + prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true)); + prop.put("references_documents_0_anchors_" + d + "_hash", refhash); + prop.put("references_documents_0_anchors_" + d + "_outbound", 0); + d++; + } i = scraper.outboundLinks().keySet().iterator(); - while (i.hasNext()) { - DigestURL refurl = i.next(); - byte[] refhash = refurl.hash(); - prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true)); - prop.put("references_documents_0_anchors_" + d + "_hash", refhash); - prop.put("references_documents_0_anchors_" + d + "_outbound", 1); - d++; - } + while (i.hasNext()) { + DigestURL refurl = i.next(); + byte[] refhash = refurl.hash(); + prop.putXML("references_documents_0_anchors_" + d + "_url", refurl.toNormalform(true)); + prop.put("references_documents_0_anchors_" + d + "_hash", refhash); + prop.put("references_documents_0_anchors_" + d + "_outbound", 1); + d++; + } prop.put("references_documents_0_count", d); prop.put("references_documents_0_anchors", d); } else { prop.put("references_count", 0); prop.put("references_documents", 0); - } + } // citations prop.put("citations", 1); ReferenceReportCache rrc = sb.index.getReferenceReportCache(); ReferenceReport rr = null; try {rr = rrc.getReferenceReport(ASCII.String(urlhash), true);} catch (IOException e) {} - if (rr != null && rr.getInternalCount() > 0 && rr.getExternalCount() > 0) { + if (rr != null && rr.getInternalCount() > 0 && rr.getExternalCount() > 0) { prop.put("citations_count", 1); prop.put("citations_documents", 1); prop.put("citations_documents_0_hash", urlhash); @@ -242,26 +243,26 @@ public class webstructure { HandleSet ids = rr.getInternallIDs(); try {ids.putAll(rr.getExternalIDs());} catch (SpaceExceededException e) {} Iterator<byte[]> i = ids.iterator(); - while (i.hasNext()) { - byte[] refhash = i.next(); - DigestURL refurl; + while (i.hasNext()) { + byte[] refhash = i.next(); + String refurl; try { refurl = authenticated ? sb.getURL(refhash) : null; prop.put("citations_documents_0_anchors_" + d + "_urle", refurl == null ? 0 : 1); - if (refurl != null) prop.putXML("citations_documents_0_anchors_" + d + "_urle_url", refurl.toNormalform(true)); + if (refurl != null) prop.putXML("citations_documents_0_anchors_" + d + "_urle_url", refurl); prop.put("citations_documents_0_anchors_" + d + "_urle_hash", refhash); prop.put("citations_documents_0_anchors_" + d + "_urle_date", GenericFormatter.SHORT_DAY_FORMATTER.format(new Date())); // superfluous? d++; } catch (IOException e) { ConcurrentLog.logException(e); } - } + } prop.put("citations_documents_0_count", d); prop.put("citations_documents_0_anchors", d); - } else { + } else { prop.put("citations_count", 0); prop.put("citations_documents", 0); - } + } } } else if (authenticated) { // show a complete list of link structure informations in case that the user is authenticated |