summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--htroot/BlacklistCleaner_p.html8
-rw-r--r--htroot/BlacklistCleaner_p.java8
-rw-r--r--htroot/ConfigRobotsTxt_p.html48
-rw-r--r--htroot/ConfigRobotsTxt_p.java204
-rw-r--r--htroot/YPStats_p.html301
-rw-r--r--htroot/YPStats_p.java248
-rw-r--r--htroot/env/page.html33
-rw-r--r--htroot/env/templates/submenuConfig.template1
-rw-r--r--source/de/anomic/data/bookmarksDB.java12
9 files changed, 283 insertions, 580 deletions
diff --git a/htroot/BlacklistCleaner_p.html b/htroot/BlacklistCleaner_p.html
index a7fb96f1d..8befc1272 100644
--- a/htroot/BlacklistCleaner_p.html
+++ b/htroot/BlacklistCleaner_p.html
@@ -14,16 +14,14 @@
<form action="BlacklistCleaner_p.html" method="post" enctype="multipart/form-data">#(disabled)#
<fieldset class="selectList"><legend>Check list</legend>
<div style="display:inline;">
- <select name="listNames" size="1">
- #{blacklists}#
- <option value="#[name]#" #(selected)#::selected="selected"#(/selected)#>#[name]#</option>
- #{/blacklists}#
+ <select name="listNames" size="1">#{blacklists}#
+ <option value="#[name]#" #(selected)#::selected="selected"#(/selected)#>#[name]#</option>#{/blacklists}#
</select>
<input type="submit" name="list" value="Check" />
</div>::
<p class="error">The blacklist-cleaner only works for the following blacklist-engines up to now:</p>
<ul>#{engines}#
- <li class="tt" title="#[fullname]#">#[name]#</li>#{/engines}#
+ <li class="tt">#[name]#</li>#{/engines}#
</ul>::
#(/disabled)#
</fieldset>
diff --git a/htroot/BlacklistCleaner_p.java b/htroot/BlacklistCleaner_p.java
index 276803647..d91b57635 100644
--- a/htroot/BlacklistCleaner_p.java
+++ b/htroot/BlacklistCleaner_p.java
@@ -1,4 +1,4 @@
-// Blacklist_p.java
+// BlacklistCleaner_p.java
// -----------------------
// part of YaCy
// (C) by Michael Peter Christen; mc@anomic.de
@@ -162,10 +162,8 @@ public class BlacklistCleaner_p {
}
} else {
prop.put("disabled", 1);
- for (int i=0; i<supportedBLEngines.length; i++) {
- prop.put(DISABLED + "engines_" + i + "_name", supportedBLEngines[i].getSimpleName());
- prop.put(DISABLED + "engines_" + i + "_fullname", supportedBLEngines[i].getName());
- }
+ for (int i=0; i<supportedBLEngines.length; i++)
+ prop.put(DISABLED + "engines_" + i + "_name", supportedBLEngines[i].getName());
prop.put(DISABLED + "engines", supportedBLEngines.length);
}
}
diff --git a/htroot/ConfigRobotsTxt_p.html b/htroot/ConfigRobotsTxt_p.html
new file mode 100644
index 000000000..e723d0462
--- /dev/null
+++ b/htroot/ConfigRobotsTxt_p.html
@@ -0,0 +1,48 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ <title>YaCy '#[clientname]#': Your Personal Profile</title>
+ #%env/templates/metas.template%#
+ </head>
+ <body id="ConfigProfile">
+ #%env/templates/header.template%#
+ #%env/templates/submenuConfig.template%#
+ <h2>Exclude Web-Spiders</h2>
+ <p>
+ </p>
+ #(error)#
+ ::<p class="error">Unable to access the local file: <span class="error tt">#[msg]#</span></p>
+ ::<p class="error">Deletion of <span class="error tt">htroot/robots.txt</span> failed</p>#(/error)#
+ <form method="post" action="ConfigRobotsTxt_p.html" enctype="multipart/form-data" accept-charset="UTF-8">
+ <input type="hidden" name="directories" value="checked" />
+ <input type="hidden" name="locked" value="checked" />
+ <fieldset><legend>Restrict access to</legend>
+ <dl>
+ <dt><label for="all"><em>Entire Peer</em></label></dt>
+ <dd><input type="checkbox" id="all" name="all"#(all.checked)#:: checked="checked"#(/all.checked)# /></dd>
+ <dt><label for="status">Status page</label></dt>
+ <dd><input type="checkbox" id="status" name="status"#(status.checked)#:: checked="checked"#(/status.checked)# /></dd>
+ <dt><label for="network">Network pages</label></dt>
+ <dd><input type="checkbox" id="network" name="network"#(network.checked)#:: checked="checked"#(/network.checked)# /></dd>
+ <dt><label for="surftips">Surftips</label></dt>
+ <dd><input type="checkbox" id="surftips" name="surftips"#(surftips.checked)#:: checked="checked"#(/surftips.checked)# /></dd>
+ <dt><label for="news">News pages</label></dt>
+ <dd><input type="checkbox" id="news" name="news"#(news.checked)#:: checked="checked"#(/news.checked)# /></dd>
+ <dt><label for="blog">Blog</label></dt>
+ <dd><input type="checkbox" id="blog" name="blog"#(blog.checked)#:: checked="checked"#(/blog.checked)# /></dd>
+ <dt><label for="wiki">Wiki</label></dt>
+ <dd><input type="checkbox" id="wiki" name="wiki"#(wiki.checked)#:: checked="checked"#(/wiki.checked)# /></dd>
+ <dt><label for="bookmarks">Public bookmarks</label></dt>
+ <dd><input type="checkbox" id="bookmarks" name="bookmarks"#(bookmarks.checked)#:: checked="checked"#(/bookmarks.checked)# /></dd>
+ <dt><label for="homepage">Home Page</label></dt>
+ <dd><input type="checkbox" id="homepage" name="homepage"#(homepage.checked)#:: checked="checked"#(/homepage.checked)# /></dd>
+ <dt><label for="fileshare">File Share</label></dt>
+ <dd><input type="checkbox" id="fileshare" name="fileshare"#(fileshare.checked)#:: checked="checked"#(/fileshare.checked)# /></dd>
+ <dt>&nbsp;</dt>
+ <dd><input type="submit" name="save" value="Set restrictions" /></dd>
+ </dl>
+ </fieldset>
+ </form>
+ #%env/templates/footer.template%#
+ </body>
+</html> \ No newline at end of file
diff --git a/htroot/ConfigRobotsTxt_p.java b/htroot/ConfigRobotsTxt_p.java
new file mode 100644
index 000000000..c37e45ae7
--- /dev/null
+++ b/htroot/ConfigRobotsTxt_p.java
@@ -0,0 +1,204 @@
+// ConfigRobotsTxt_p.java
+// -----------------------
+// part of YaCy
+// (C) by Michael Peter Christen; mc@anomic.de
+// first published on http://www.anomic.de
+// Frankfurt, Germany, 2004
+//
+// This File is contributed by Franz Brauße
+//
+// $LastChangedDate: $
+// $LastChangedRevision: $
+// $LastChangedBy: $
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//
+// Using this software in any meaning (reading, learning, copying, compiling,
+// running) means that you agree that the Author(s) is (are) not responsible
+// for cost, loss of data or any harm that may be caused directly or indirectly
+// by usage of this softare or this documentation. The usage of this software
+// is on your own risk. The installation and usage (starting/running) of this
+// software may allow other people or application to access your computer and
+// any attached devices and is highly dependent on the configuration of the
+// software which must be done by the user of the software; the author(s) is
+// (are) also not responsible for proper configuration and usage of the
+// software, even if provoked by documentation provided together with
+// the software.
+//
+// Any changes to this file according to the GPL as documented in the file
+// gpl.txt aside this file in the shipment you received can be done to the
+// lines that follows this copyright notice here, but changes must not be
+// done inside the copyright notive above. A re-distribution must contain
+// the intact and unchanged copyright notice.
+// Contributions and changes to the program code must be marked as such.
+
+// You must compile this file with
+// javac -classpath .:../classes ConfigRobotsTxt_p.java
+// if the shell's current path is HTROOT
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import de.anomic.http.httpHeader;
+import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.server.serverObjects;
+import de.anomic.server.serverSwitch;
+import de.anomic.server.servletProperties;
+import de.anomic.server.logging.serverLog;
+import de.anomic.yacy.yacyCore;
+
+public class ConfigRobotsTxt_p {
+
+ public static final Pattern entryBeginPattern = Pattern.compile("# (\\w*) \\((\\d*) entries\\)");
+
+ private static HashMap disallowMap = null;
+
+ private static Map getDisallowMap(String htrootPath) {
+ if (disallowMap == null) {
+ final File htroot = new File(htrootPath);
+ if (!htroot.exists()) return null;
+ disallowMap = new /* <String,String[]> */ HashMap();
+ final ArrayList htrootFiles = new ArrayList();
+ final ArrayList htrootDirs = new ArrayList();
+ final String[] htroots = htroot.list();
+ File file;
+ for (int i=0, dot; i<htroots.length; i++) {
+ if (htroots[i].equals("www")) continue;
+ file = new File(htroot, htroots[i]);
+ if (file.isDirectory()) {
+ htrootDirs.add("/" + file.getName());
+ } else if (
+ (dot = htroots[i].lastIndexOf('.')) < 2 ||
+ htroots[i].charAt(dot - 2) == '_' && htroots[i].charAt(dot - 1) == 'p'
+ ) {
+ htrootFiles.add("/" + file.getName());
+ }
+ }
+
+ disallowMap.put("all", new String[] { "/" } );
+ disallowMap.put("locked", htrootFiles.toArray(new String[htrootFiles.size()]));
+ disallowMap.put("directories", htrootDirs.toArray(new String[htrootDirs.size()]));
+ disallowMap.put("blog", new String[] {
+ "/Blog.html",
+ "/Blog.xml",
+ "/BlogComments.html" } );
+ disallowMap.put("wiki", new String[] { "/Wiki.html" } );
+ disallowMap.put("bookmarks", new String[] { "/Bookmarks.html" } );
+ disallowMap.put("homepage", new String[] { "/www" } );
+ disallowMap.put("fileshare", new String[] { "/share" } );
+ disallowMap.put("surftips", new String[] { "/Surftips.html" } );
+ disallowMap.put("news", new String[] { "/News.html" } );
+ disallowMap.put("status", new String[] { "/Status.html" } );
+ disallowMap.put("network", new String[] {
+ "/Network.html",
+ "/Network.xml",
+ "/Network.csv" } );
+ }
+ return disallowMap;
+ }
+
+ public static servletProperties respond(httpHeader header, serverObjects post, serverSwitch env) {
+ final servletProperties prop = new servletProperties();
+
+ prop.put("address", yacyCore.seedDB.mySeed.getAddress());
+
+ final String htroot = ((plasmaSwitchboard)env).getConfig(plasmaSwitchboard.HTROOT_PATH, plasmaSwitchboard.HTROOT_PATH_DEFAULT);
+ final File robots_txt = new File(htroot + File.separator + "robots.txt");
+ if (!robots_txt.exists()) try {
+ robots_txt.createNewFile();
+ } catch (IOException e) {
+ prop.put("error", 1);
+ prop.put("error_msg", e.getMessage());
+ }
+
+ if (post != null) {
+ if (post.containsKey("save")) {
+ try {
+ if (robots_txt.delete() && robots_txt.createNewFile()) {
+ PrintWriter out = new PrintWriter(new BufferedWriter(new FileWriter(robots_txt)));
+ printHeader(out);
+
+ final Iterator it = getDisallowMap(htroot).entrySet().iterator();
+ Map.Entry entry;
+ while (it.hasNext()) {
+ entry = (Map.Entry)it.next();
+ if (post.containsKey(entry.getKey())) {
+ out.println();
+ printEntry(out, entry);
+ }
+ }
+ out.flush();
+ out.close();
+ } else {
+ prop.put("error", 2);
+ }
+ } catch (IOException e) {
+ serverLog.logSevere("ROBOTS.TXT", "Error writing " + robots_txt, e);
+ prop.put("error", 1);
+ prop.put("error_msg", e.getMessage());
+ }
+ }
+ }
+
+ // read htroot/robots.txt
+ try {
+ BufferedReader br = new BufferedReader(new FileReader(robots_txt));
+ String line;
+ Matcher m;
+ while ((line = br.readLine()) != null) {
+ if ((m = entryBeginPattern.matcher(line)).matches())
+ prop.put(m.group(1) + ".checked", 1);
+ }
+ } catch (IOException e) {
+ prop.put("error", 1);
+ prop.put("error_msg", e.getMessage());
+ }
+
+ return prop;
+ }
+
+ private static void printHeader(PrintWriter out) {
+ out.print("# robots.txt for ");
+ out.print(yacyCore.seedDB.mySeed.getName());
+ out.println(".yacy");
+ out.println();
+ out.println("User-agent: *");
+ }
+
+ private static void printEntry(PrintWriter out, Map.Entry entry) {
+ String[] disallows = (String[])entry.getValue();
+ out.print("# ");
+ out.print(entry.getKey());
+ out.print(" (");
+ out.print(disallows.length);
+ out.println(" entries)");
+
+ for (int i=0; i<disallows.length; i++) {
+ out.print("Disallow: ");
+ out.println(disallows[i]);
+ }
+ }
+}
diff --git a/htroot/YPStats_p.html b/htroot/YPStats_p.html
deleted file mode 100644
index 530c5a52e..000000000
--- a/htroot/YPStats_p.html
+++ /dev/null
@@ -1,301 +0,0 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
-<html>
- <head>
- <title>YaCy '#[clientname]#': YPStats</title>
- #%env/templates/metas.template%#
- <style><!--
- select {
- width: 13em;
- }
- input.url {
- width: 22em;
- }
- input.ypsmall {
- width: 4em;
- }
- -->
- </style>
- </head>
- <body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0">
- #%env/templates/header.template%#
- <br />
- <h2>YPStats - Statistics to generate default settings</h2>
- <p>
- <form action="#" method="GET">
- <label for="0">Select submit url: </label><input class="url" type="text" name="url" value="#[url]#" />
- <input type="submit" name="urlchg" value="change" />
- </form>
- </p>
- <hr />
- <form action="#[url]#" method="post" enctype="multipart/form-data">
- <p>
- <table>
- <tr>
- <td><label for="1">Operating system: </label></td>
- <td><!--<input type="text" name="OS" value="" id="1" />-->
- <select name="OS" id="1">
- <optgroup label="Windows">
- <option label="Win98/ME" value="Win98">Windows 98/ME</option>
- <option label="WinXP" value="WinXP">Windows 2000/XP</option>
- </optgroup>
- <option label="Linux" value="Linux">GNU Linux</option>
- <option label="*BSD" value="BSD">Open-/Free-/NetBSD</option>
- <optgroup label="Mac OS">
- <option value="MacOS&lt;=X">Mac OS &lt;= X</option>
- <option value="MacOSX">Mac OS X</option>
- </optgroup>
- <option value="Solaris">Solaris</option>
- <option value="ReactOS">ReactOS</option>
- </select>
- </td>
- <td></td>
- </tr>
- <tr>
- <td><label for="2">RAM in machine: </label></td>
- <td><input class="ypsmall" type="text" name="RAM" value="" id="2" /> MB</td>
- <td></td>
- </tr>
- <tr>
- <td><label for="3">CPU speed: </label></td>
- <td><input class="ypsmall" type="text" name="CPUSpeed" value="" id="3" /> MHz</td>
- <td></td>
- </tr>
- <tr>
- <td><label for="5">HDD speed: </label></td>
- <td><input class="ypsmall" type="text" name="HDDSpeed" value="" id="5" /> RPM</td>
- <td></td>
- </tr>
- <tr>
- <td><label for="6">HDD cache: </label></td>
- <td><input class="ypsmall" type="text" name="HDDCache" value="" id="6" /> KB</td>
- <td></td>
- </tr>
- <tr>
- <td><label for="7">HDD interface: </label></td>
- <td>
- <select name="HDDInterface" id="7">
- <option value="P-ATA" selected>P-ATA / IDE</option>
- <option value="S-ATA">Serial ATA</option>
- <option value="S-ATA2">Serial ATA 2</option>
- <option value="SCSI">SCSI</option>
- <option value="else">Other</option>
- </select>
- </td>
- <td></td>
- </tr>
- <tr>
- <td><label for="fstype">Filesystem: </label></td>
- <td>
- <select name="FSType">
- <option value="FAT32">FAT32</option>
- <option value="NTFS">NTFS</option>
- <option value="Ext2-3">UFS / Ext2 / Ext3</option>
- <option value="JFS">JFS</option>
- <option value="XFS">(C)XFS</option>
- <option value="Reiser">ReiserFS</option>
- <option value="HFS">HFS / HFS+</option>
- </select>
- </td>
- <td></td>
- </tr>
- <tr>
- <td><label for="seperate">YaCy runs on "own" HDD</label></td>
- <td><input type="checkbox" name="seperateHDD" id="seperate" /></td>
- <td></td>
- <tr>
- <td><label for="8">Peer main usage: </label></td>
- <td>
- <select name="Usage" id="8">
- <option value="0" selected>Proxy</option>
- <option value="1">DHT-Peer</option>
- <option value="2">Crawl-Peer</option>
- <option value="3">No special usage</option>
- </select>
- </td>
- <td></td>
- </tr>
- <tr>
- <td><label for="9">PC only for YaCy </label></td>
- <td><input type="checkbox" name="24-7" id="9" /></td>
- <td></td>
- </tr>
- <tr>
- <td><label for="10">System slowdown: </label></td>
- <td>
- <select name="Slowdown" id="10">
- <option value="0" selected>No slowdown noticable</option>
- <option value="1">Slight slowdown</option>
- <option value="2">Moderate slowdown</option>
- <option value="3">Heavy slowdown</option>
- <option value="4">Working not possible</option>
- </select>
- </td>
- <td></td>
- </tr>
- <tr>
- <td><label for="11">Comments: </label></td>
- <td colspan="2"><textarea name="Comment" rows="10" cols="73" id="11"></textarea></td>
- </tr>
- <tr>
- <td colspan="3">
- <center><input type="submit" name="submit" value="Submit to YPStats" /></center>
- </td>
- </tr>
- </table>
- </p>
- <hr />
- <h3>Data which will be sent to #[url]#:</h3>
- <p>
- <table border="0" cellpadding="2" cellspacing="1">
- <tr class="TableHeader" valign="bottom">
- <td class="small">Thread</td>
- <td class="small">idlesleep</td>
- <td class="small">busysleep</td>
- <td class="small">memprereq</td>
- </tr>
- #{table}#<tr class="TableCellDark">
- <td class="small">#[threadname]#</td>
- <td class="small">#[idlesleep]#<input type="hidden" name="#[threadname]#_idlesleep" value="#[idlesleep]#" /></td>
- <td class="small">#[busysleep]#<input type="hidden" name="#[threadname]#_busysleep" value="#[busysleep]#" /></td>
- <td class="small">#[memprereq]#<input type="hidden" name="#[threadname]#_memprereq" value="#[memprereq]#" /></td>
- </tr>
- #{/table}#
- </table>
- </p>
- <p>
- <table border="0" cellpadding="2" cellspacing="1">
- <tr class="TableCellDark">
- <td class="small">wordCacheMaxCount</td>
- <td class="small">#[wordCacheMaxCount]#<input type="hidden" name="wordCacheMaxCount" value="#[wordCacheMaxCount]#" /></td>
- </tr>
- </table>
- </p>
- <p>
- <table border="0" cellpadding="2" cellspacing="1">
- <tr class="TableHeader" valign="bottom">
- <td class="small">Thread</td>
- <td class="small">maxActive</td>
- <td class="small">maxIdle</td>
- <td class="small">minIdle</td>
- </tr>
- #{pool}#<tr class="TableCellDark">
- <td class="small">#[name]#</td>
- <td class="small">#[maxActive]#<input type="hidden" name="#[name]#_maxActive" value="#[maxActive]#" /></td>
- <td class="small">#[maxIdle]#<input type="hidden" name="#[name]#_maxIdle" value="#[maxIdle]#"/></td>
- <td class="small">#[minIdle]#<input type="hidden" name="#[name]#_minIdle" value="#[minIdle]#" /></td>
- </tr>
- #{/pool}#
- </table>
- </p>
- <p>
- <table border="0" cellpadding="2" cellspacing="1">
- <tr class="TableCellDark">
- <td><span title="-Xmx parameter for Java" class="small">Maximum RAM</span></td>
- <td class="small">#[Xmx]#<input type="hidden" name="Xmx" value="#[Xmx]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td><span title="-Xms parameter for Java" class="small">Initial RAM</span></td>
- <td class="small">#[Xms]#<input type="hidden" name="Xms" value="#[Xms]#" /></td>
- </tr>
- </table>
- </p>
- <p>
- <table border="0" cellpadding="2" cellspacing="1">
- <tr class="TableHeader" valign="bottom">
- <td class="small">Database</td>
- <td class="small">DB-Size</td>
- <td class="small">RAM Cache</td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">RWI</td>
- <td class="small">#[slreqRWI]#<input type="hidden" name="DBSizeRWI" value="#[slreqRWI]#" /></td>
- <td class="small">#[ramCacheRWI]#<input type="hidden" name="ramCacheRWI" value="#[ramCacheRWI]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">HTTP</td>
- <td class="small">#[slreqHTTP]#<input type="hidden" name="DBSizeHTTP" value="#[slreqHTTP]#" /></td>
- <td class="small">#[ramCacheHTTP]#<input type="hidden" name="ramCacheHTTP" value="#[ramCacheHTTP]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">LURL</td>
- <td class="small">#[slreqLURL]#<input type="hidden" name="DBSizeLURL" value="#[slreqLURL]#" /></td>
- <td class="small">#[ramCacheLURL]#<input type="hidden" name="ramCacheLURL" value="#[ramCacheLURL]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">NURL</td>
- <td class="small">#[slreqNURL]#<input type="hidden" name="DBSizeNURL" value="#[slreqNURL]#" /></td>
- <td class="small">#[ramCacheNURL]#<input type="hidden" name="ramCacheNURL" value="#[ramCacheNURL]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">EURL</td>
- <td class="small">#[slreqEURL]#<input type="hidden" name="DBSizeEURL" value="#[slreqEURL]#" /></td>
- <td class="small">#[ramCacheEURL]#<input type="hidden" name="ramCacheEURL" value="#[ramCacheEURL]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">DHT</td>
- <td class="small">#[slreqDHT]#<input type="hidden" name="DBSizeDHT" value="#[slreqDHT]#" /></td>
- <td class="small">#[ramCacheDHT]#<input type="hidden" name="ramCacheDHT" value="#[ramCacheDHT]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">Message</td>
- <td class="small">#[slreqMessage]#<input type="hidden" name="DBSizeMessage" value="#[slreqMessage]#" /></td>
- <td class="small">#[ramCacheMessage]#<input type="hidden" name="ramCacheMessage" value="#[ramCacheMessage]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">Wiki</td>
- <td class="small">#[slreqWiki]#<input type="hidden" name="DBSizeWiki" value="#[slreqWiki]#" /></td>
- <td class="small">#[ramCacheWiki]#<input type="hidden" name="ramCacheWiki" value="#[ramCacheWiki]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">Blog</td>
- <td class="small">#[slreqBlog]#<input type="hidden" name="DBSizeBlog" value="#[slreqBlog]#" /></td>
- <td class="small">#[ramCacheBlog]#<input type="hidden" name="ramCacheBlog" value="#[ramCacheBlog]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">News</td>
- <td class="small">#[slreqNews]#<input type="hidden" name="DBSizeNews" value="#[slreqNews]#" /></td>
- <td class="small">#[ramCacheNews]#<input type="hidden" name="ramCacheNews" value="#[ramCacheNews]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">Robots</td>
- <td class="small">#[slreqRobots]#<input type="hidden" name="DBSizeRobots" value="#[slreqRobots]#" /></td>
- <td class="small">#[ramCacheRobots]#<input type="hidden" name="ramCacheRobots" value="#[ramCacheRobots]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">Profiles</td>
- <td class="small">#[slreqProfiles]#<input type="hidden" name="DBSizeProfiles" value="#[slreqProfiles]#" /></td>
- <td class="small">#[ramCacheProfiles]#<input type="hidden" name="ramCacheProfiles" value="#[ramCacheProfiles]#" /></td>
- </tr>
- </table>
- </p>
- <p>
- <table border="0" cellpadding="2" cellspacing="1">
- <tr class="TableCellDark">
- <td class="small">YaCy-Version</td>
- <td class="small">#[versionpp]#<input type="hidden" name="version" value="#[versionpp]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small"># Links</td>
- <td class="small">#[links]#<input type="hidden" name="links" value="#[links]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small"># RWIs</td>
- <td class="small">#[words]#<input type="hidden" name="words" value="#[words]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">Peer-Hash</td>
- <td class="small">#[hash]#<input type="hidden" name="hash" value="#[hash]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">Java-Version</td>
- <td class="small">#[jversion]#<input type="hidden" name="jversion" value="#[jversion]#" /></td>
- </tr>
- <tr class="TableCellDark">
- <td class="small">Processors</td>
- <td class="small">#[processors]#<input type="hidden" name="CPUCount" value="#[processors]#" /></td>
- </tr>
- </table>
- </p>
- </form>
-#%env/templates/footer.template%#
- </body> \ No newline at end of file
diff --git a/htroot/YPStats_p.java b/htroot/YPStats_p.java
deleted file mode 100644
index 8a0892dec..000000000
--- a/htroot/YPStats_p.java
+++ /dev/null
@@ -1,248 +0,0 @@
-//YPStats_p.java
-//-----------------------
-//Author: Franz Brausse
-//part of YaCy (C) by Michael Peter Christen; mc@anomic.de
-//first published on http://www.anomic.de
-//Frankfurt, Germany, 2004, 2005
-//last major change: 27.06.2006
-//
-//This program is free software; you can redistribute it and/or modify
-//it under the terms of the GNU General Public License as published by
-//the Free Software Foundation; either version 2 of the License, or
-//(at your option) any later version.
-//
-//This program is distributed in the hope that it will be useful,
-//but WITHOUT ANY WARRANTY; without even the implied warranty of
-//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-//GNU General Public License for more details.
-//
-//You should have received a copy of the GNU General Public License
-//along with this program; if not, write to the Free Software
-//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-//Using this software in any meaning (reading, learning, copying, compiling,
-//running) means that you agree that the Author(s) is (are) not responsible
-//for cost, loss of data or any harm that may be caused directly or indirectly
-//by usage of this softare or this documentation. The usage of this software
-//is on your own risk. The installation and usage (starting/running) of this
-//software may allow other people or application to access your computer and
-//any attached devices and is highly dependent on the configuration of the
-//software which must be done by the user of the software; the author(s) is
-//(are) also not responsible for proper configuration and usage of the
-//software, even if provoked by documentation provided together with
-//the software.
-//
-//Any changes to this file according to the GPL as documented in the file
-//gpl.txt aside this file in the shipment you received can be done to the
-//lines that follows this copyright notice here, but changes must not be
-//done inside the copyright notive above. A re-distribution must contain
-//the intact and unchanged copyright notice.
-//Contributions and changes to the program code must be marked as such.
-
-//You must compile this file with
-//javac -classpath .:../classes YPStats_p.java
-//if the shell's current path is HTROOT
-
-import java.util.Iterator;
-
-import org.apache.commons.pool.impl.GenericKeyedObjectPool;
-import org.apache.commons.pool.impl.GenericObjectPool;
-
-import de.anomic.http.httpHeader;
-import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.server.serverCore;
-import de.anomic.server.serverObjects;
-import de.anomic.server.serverSwitch;
-import de.anomic.server.serverThread;
-import de.anomic.yacy.yacyCore;
-import de.anomic.yacy.yacySeed;
-
-public class YPStats_p {
-
- private static final int KB = 1024;
-
- //private static long[] slt,chk;
- //private static String[] ost;
- private static long req /*, usd, bst, god*/;
-
- //private static long usedTotal, currTotal, dfltTotal, goodTotal, bestTotal;
-
- public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) {
- // return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) sb;
- serverObjects prop = new serverObjects();
- //File defaultSettingsFile = new File(switchboard.getRootPath(), "yacy.init");
- //Map defaultSettings = ((post == null) || (!(post.containsKey("submitdefault")))) ? null : serverFileUtils.loadHashMap(defaultSettingsFile);
-
- String url=null;
- if(post!=null && post.containsKey("url")) {
- url=(String)post.get("url");
- } else {
- url="http://ypstats.yacy-forum.de/index.php";
- }
- prop.put("url", url);
-
- Iterator threads = switchboard.threadNames();
- String threadName;
- serverThread thread;
-
- // calculate totals
- long blocktime_total = 0, sleeptime_total = 0, exectime_total = 0;
- while (threads.hasNext()) {
- threadName = (String) threads.next();
- thread = switchboard.getThread(threadName);
- blocktime_total += thread.getBlockTime();
- sleeptime_total += thread.getSleepTime();
- exectime_total += thread.getExecTime();
- }
- if (blocktime_total == 0) blocktime_total = 1;
- if (sleeptime_total == 0) sleeptime_total = 1;
- if (exectime_total == 0) exectime_total = 1;
-
- // set templates for latest news from the threads
- long blocktime, sleeptime, exectime;
- long idlesleep, busysleep, memuse, memprereq;
- int queuesize;
- threads = switchboard.threadNames();
- int c = 0;
- long idleCycles, busyCycles, memshortageCycles;
- while (threads.hasNext()) {
- threadName = (String) threads.next();
- thread = switchboard.getThread(threadName);
-
- // set values to templates
- prop.put("table_" + c + "_threadname", threadName);
- prop.put("table_" + c + "_shortdescr", (thread.getMonitorURL() == null) ? thread.getShortDescription() : "<a href=\"" + thread.getMonitorURL() + "\" class=?\"small\">" + thread.getShortDescription() + "</a>");
- prop.put("table_" + c + "_longdescr", thread.getLongDescription());
- queuesize = thread.getJobCount();
- prop.put("table_" + c + "_queuesize", (queuesize == Integer.MAX_VALUE) ? "unlimited" : Integer.toString(queuesize));
-
- blocktime = thread.getBlockTime();
- sleeptime = thread.getSleepTime();
- exectime = thread.getExecTime();
- memuse = thread.getMemoryUse();
- idleCycles = thread.getIdleCycles();
- busyCycles = thread.getBusyCycles();
- memshortageCycles = thread.getOutOfMemoryCycles();
- prop.put("table_" + c + "_blocktime", blocktime / 1000);
- prop.put("table_" + c + "_blockpercent", Long.toString(100 * blocktime / blocktime_total));
- prop.put("table_" + c + "_sleeptime", sleeptime / 1000);
- prop.put("table_" + c + "_sleeppercent", Long.toString(100 * sleeptime / sleeptime_total));
- prop.put("table_" + c + "_exectime", exectime / 1000);
- prop.put("table_" + c + "_execpercent", Long.toString(100 * exectime / exectime_total));
- prop.put("table_" + c + "_totalcycles", Long.toString(idleCycles + busyCycles + memshortageCycles));
- prop.put("table_" + c + "_idlecycles", Long.toString(idleCycles));
- prop.put("table_" + c + "_busycycles", Long.toString(busyCycles));
- prop.put("table_" + c + "_memscycles", Long.toString(memshortageCycles));
- prop.put("table_" + c + "_sle ", ((idleCycles + busyCycles) == 0) ? "-" : Long.toString(sleeptime / (idleCycles + busyCycles)));
- prop.put("table_" + c + "_execpercycle", (busyCycles == 0) ? "-" : Long.toString(exectime / busyCycles));
- prop.put("table_" + c + "_memusepercycle", (busyCycles == 0) ? "-" : Long.toString(memuse / busyCycles / 1024));
-
- // load with old values
- idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000"));
- busysleep = Long.parseLong(switchboard.getConfig(threadName + "_busysleep", "100"));
- memprereq = Long.parseLong(switchboard.getConfig(threadName + "_memprereq", "0"));
-
- prop.put("table_" + c + "_idlesleep", idlesleep);
- prop.put("table_" + c + "_busysleep", busysleep);
- prop.put("table_" + c + "_memprereq", memprereq / 1024);
-
- c++;
- }
- prop.put("table", c);
-
- // table cache settings
- prop.put("wordCacheMaxCount", switchboard.getConfig("wordCacheMaxCount", "10000"));
-
- // table thread pool settings
- GenericKeyedObjectPool.Config crawlerPoolConfig = switchboard.cacheLoader.getPoolConfig();
- prop.put("pool_0_name","Crawler Pool");
- prop.put("pool_0_maxActive",crawlerPoolConfig.maxActive);
- prop.put("pool_0_maxIdle",crawlerPoolConfig.maxIdle);
- prop.put("pool_0_minIdle",0 /* crawlerPoolConfig.minIdle */);
-
- serverThread httpd = switchboard.getThread("10_httpd");
- GenericObjectPool.Config httpdPoolConfig = ((serverCore)httpd).getPoolConfig();
- prop.put("pool_1_name","httpd Session Pool");
- prop.put("pool_1_maxActive",httpdPoolConfig.maxActive);
- prop.put("pool_1_maxIdle",httpdPoolConfig.maxIdle);
- prop.put("pool_1_minIdle",httpdPoolConfig.minIdle);
-
- GenericObjectPool.Config stackerPoolConfig = switchboard.sbStackCrawlThread.getPoolConfig();
- prop.put("pool_2_name","CrawlStacker Session Pool");
- prop.put("pool_2_maxActive",stackerPoolConfig.maxActive);
- prop.put("pool_2_maxIdle",stackerPoolConfig.maxIdle);
- prop.put("pool_2_minIdle",stackerPoolConfig.minIdle);
- prop.put("pool",3);
-
- // parse initialization memory settings
- String Xmx = sb.getConfig("javastart_Xmx", "Xmx64m").substring(3);
- prop.put("Xmx", Xmx.substring(0, Xmx.length() - 1));
- String Xms = sb.getConfig("javastart_Xms", "Xms10m").substring(3);
- prop.put("Xms", Xms.substring(0, Xms.length() - 1));
-
- plasmaSwitchboard sb1 = (plasmaSwitchboard) sb;
- req = sb1.wordIndex.size();
- putprop(prop, sb, "RWI");
-
- req = sb1.cacheManager.dbSize();
- putprop(prop, sb, "HTTP");
-
- //req = sb1.urlPool.loadedURL.urlHashCache.size();
- putprop(prop, sb, "LURL");
-
- //req = sb1.urlPool.noticeURL.urlHashCache.size();
- putprop(prop, sb, "NURL");
-
- //req = sb1.urlPool.errorURL.urlHashCache.size();
- putprop(prop, sb, "EURL");
-
- req = yacyCore.seedDB.sizeConnected() + yacyCore.seedDB.sizeDisconnected() + yacyCore.seedDB.sizePotential();
- putprop(prop, sb, "DHT");
-
- req = sb1.messageDB.size();
- putprop(prop, sb, "Message");
-
- req = sb1.wikiDB.sizeOfTwo();
- putprop(prop, sb, "Wiki");
-
- req = sb1.blogDB.size();
- putprop(prop, sb, "Blog");
-
- req = yacyCore.newsPool.dbSize();
- putprop(prop, sb, "News");
-
- req = plasmaSwitchboard.robots.size();
- putprop(prop, sb, "Robots");
-
- req = sb1.profiles.size();
- putprop(prop, sb, "Profiles");
-
- prop.put("versionpp", yacy.combined2prettyVersion(sb.getConfig("version","0.1")));
-
- prop.put("links", yacyCore.seedDB.mySeed.get(yacySeed.LCOUNT, "unknown"));
- prop.put("words", yacyCore.seedDB.mySeed.get(yacySeed.ICOUNT, "unknown"));
- /*prop.put("address", yacyCore.seedDB.mySeed.getAddress());
- prop.put("ip", serverCore.publicLocalIP());*/
- prop.put("hash", yacyCore.seedDB.mySeed.hash);
-
- String jversion = System.getProperties().getProperty("java.vendor");
- jversion += " " + System.getProperties().getProperty("java.version");
- prop.put("jversion", jversion);
- prop.put("processors",Runtime.getRuntime().availableProcessors());
-
- // return rewrite values for templates
- return prop;
- }
-
- private static void putprop(serverObjects prop, serverSwitch sb, String db) {
- prop.put("slreq" + db, req);
- prop.put("ramCache" + db, Long.parseLong(sb.getConfig("ramCache" + db, "0")) / KB);
- }
-
- /*
- private static String d(String a, String b) {
- return (a == null) ? b : a;
- }
- */
-} \ No newline at end of file
diff --git a/htroot/env/page.html b/htroot/env/page.html
index 364ea18f6..7ea35076f 100644
--- a/htroot/env/page.html
+++ b/htroot/env/page.html
@@ -1,19 +1,16 @@
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
-<html>
-<head>
-#[header]#
-<!--<title>YaCy '#[clientname]#': #[title]#</title>-->
-<!-- TODO: title cannot be set, yet. -->
-<!-- clientname needs to be set in page.java (this will remove the code in httpdFileHandler,
-if we use supertemplates for everything -->
-#%env/templates/metas.template%#
-</head>
-<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0">
-#%env/templates/header.template%#
-<br><br>
-
-#[page]#
-
-#%env/templates/footer.template%#
-</body>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+ <head>
+ #[header]#
+ <!--<title>YaCy '#[clientname]#': #[title]#</title>-->
+ <!-- TODO: title cannot be set, yet. -->
+ <!-- clientname needs to be set in page.java (this will remove the code in httpdFileHandler,
+ if we use supertemplates for everything -->
+ #%env/templates/metas.template%#
+ </head>
+ <body>
+ #%env/templates/header.template%#
+ #[page]#
+ #%env/templates/footer.template%#
+ </body>
</html>
diff --git a/htroot/env/templates/submenuConfig.template b/htroot/env/templates/submenuConfig.template
index 4224c6182..02144e747 100644
--- a/htroot/env/templates/submenuConfig.template
+++ b/htroot/env/templates/submenuConfig.template
@@ -5,6 +5,7 @@
<li><a href="/ConfigLanguage_p.html" class="MenuItemLink lock">Language</a></li>
<li><a href="/ConfigProfile_p.html" class="MenuItemLink lock">Peer Profile</a></li>
<li><a href="/ConfigSkins_p.html" class="MenuItemLink lock">Interface Skins</a></li>
+ <li><a href="/ConfigRobotsTxt_p.html" class ="MenuItemLink lock">Local robots.txt</a></li>
<li><a href="/ConfigAdvanced_p.html" class="MenuItemLink lock">Advanced</a></li>
</ul>
</div> \ No newline at end of file
diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java
index fbf2776c1..83aa00f54 100644
--- a/source/de/anomic/data/bookmarksDB.java
+++ b/source/de/anomic/data/bookmarksDB.java
@@ -321,12 +321,14 @@ public class bookmarksDB {
}
public Bookmark getBookmark(String urlHash){
try {
- return (Bookmark) bookmarksTable.get(urlHash);
+ kelondroObjectsMapEntry map = (kelondroObjectsMapEntry)bookmarksTable.get(urlHash);
+ if (map instanceof Bookmark) return (Bookmark)map;
+ return new Bookmark(map);
} catch (IOException e) {
return null;
}
-
}
+
public Iterator getBookmarksIterator(boolean priv){
TreeSet set=new TreeSet(new bookmarkComparator(true));
Iterator it=bookmarkIterator(true);
@@ -764,7 +766,11 @@ public class bookmarksDB {
tags=new HashSet();
timestamp=System.currentTimeMillis();
}
-
+
+ public Bookmark(kelondroObjectsMapEntry map) {
+ this(plasmaURL.urlHash((String)map.map().get(BOOKMARK_URL)), map.map());
+ }
+
private Map toMap(){
entry.put(BOOKMARK_TAGS, listManager.hashset2string(tags));
entry.put(BOOKMARK_TIMESTAMP, String.valueOf(this.timestamp));
generated by cgit v1.2.3 (git 2.46.0) at 2026-08-08 10:28:37 +0000