diff options
| author | orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542> | 2005-09-20 10:10:34 +0000 |
|---|---|---|
| committer | orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542> | 2005-09-20 10:10:34 +0000 |
| commit | fb52a82008da0b15d675c3e25a0cb664fe918203 (patch) | |
| tree | 0986a7a5283893f61494454a0d67af494192276d | |
| parent | 0f1bbff459986b94141ec3ff6b21d2783a8ad1f0 (diff) | |
added new performance page for memory settings
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@751 6c8d7289-2bf4-0310-a012-ef5d649a1542
22 files changed, 534 insertions, 71 deletions
diff --git a/htroot/PerformanceMemory_p.html b/htroot/PerformanceMemory_p.html new file mode 100644 index 000000000..78fd2aac6 --- /dev/null +++ b/htroot/PerformanceMemory_p.html @@ -0,0 +1,189 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
+<html>
+<head>
+<title>YaCy '#[clientname]#': Performance Settings for Memory</title>
+#[metas]#
+</head>
+<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0">
+#[header]#
+<br><br>
+<h2>Performance Settings for Memory</h2>
+
+<p>
+<div class=small><b>RAM Cache for Database Files:</b></div>
+<table border="0" cellpadding="2" cellspacing="1">
+<form action="PerformanceMemory_p.html" method="post" enctype="multipart/form-data">
+<tr class="TableHeader" valign="bottom">
+<td class="small">Database<br></td>
+<td class="small">Chunk Size<br>(bytes)</td>
+<td class="small">#Needed Slots<br>(= DB Size)</td>
+<td class="small">#Empty Slots<br> </td>
+<td class="small">#Used Slots<br>High Priority</td>
+<td class="small">#Used Slots<br>Medium Priority</td>
+<td class="small">#Used Slots<br>Low Priority</td>
+<td class="small">Used<br>Size</td>
+<td class="small">Assigned<br>Max-Size</td>
+<td class="small">Recommended<br>Max-Size</td>
+<td class="small">Optimum<br>Max-Size</td>
+<td class="small">Description<br> </td>
+</tr>
+
+<tr class="TableCellDark">
+<td class="small" align="left">RWI Assortment Cluster</td>
+<td class="small" align="right">#[chunkRWI]#</td>
+<td class="small" align="right">#[slreqRWI]#</td>
+<td class="small" align="right">#[slempRWI]#</td>
+<td class="small" align="right">#[slhigRWI]#</td>
+<td class="small" align="right">#[slmedRWI]#</td>
+<td class="small" align="right">#[sllowRWI]#</td>
+<td class="small" align="right">#[usedRWI]#</td>
+<td class="small" align="right"><input name="ramCacheRWI" type="text" align="right" size="8" maxlength="9" value="#[ramCacheRWI]#"> bytes</td>
+<td class="small" align="right">#[goodRWI]#</td>
+<td class="small" align="right">#[bestRWI]#</td>
+<td class="small" align="left">The Assortment Cluster stores most of the page indexes.
+Flushing speed of the temporary RWI cache depends on the size of this file cache. Increasing the space of this
+cache will speed up crawls with a depth > 3.</td>
+</tr>
+
+<tr class="TableCellDark">
+<td class="small" align="left">HTTP Response Header</td>
+<td class="small" align="right">#[chunkHTTP]#</td>
+<td class="small" align="right">#[slreqHTTP]#</td>
+<td class="small" align="right">#[slempHTTP]#</td>
+<td class="small" align="right">#[slhigHTTP]#</td>
+<td class="small" align="right">#[slmedHTTP]#</td>
+<td class="small" align="right">#[sllowHTTP]#</td>
+<td class="small" align="right">#[usedHTTP]#</td>
+<td class="small" align="right"><input name="ramCacheHTTP" type="text" align="right" size="8" maxlength="9" value="#[ramCacheHTTP]#"> bytes</td>
+<td class="small" align="right">#[goodHTTP]#</td>
+<td class="small" align="right">#[bestHTTP]#</td>
+<td class="small" align="left">The Response Header database stores the HTTP heades that other servers send when YaCy retrieves web pages
+during proxy mode, when performing crawls or if it fetches pages for snippet generation.
+Increasing this cache will be most important for a fast proxy mode.</td>
+</tr>
+
+<tr class="TableCellDark">
+<td class="small" align="left">'loaded' URLs</td>
+<td class="small" align="right">#[chunkLURL]#</td>
+<td class="small" align="right">#[slreqLURL]#</td>
+<td class="small" align="right">#[slempLURL]#</td>
+<td class="small" align="right">#[slhigLURL]#</td>
+<td class="small" align="right">#[slmedLURL]#</td>
+<td class="small" align="right">#[sllowLURL]#</td>
+<td class="small" align="right">#[usedLURL]#</td>
+<td class="small" align="right"><input name="ramCacheLURL" type="text" align="right" size="8" maxlength="9" value="#[ramCacheLURL]#"> bytes</td>
+<td class="small" align="right">#[goodLURL]#</td>
+<td class="small" align="right">#[bestLURL]#</td>
+<td class="small" align="left">This is the database that holds the hash/url - relation and properties regarding the url like load date and server date.
+This cache is very important for a fast search process. Increasing the cache size will result in more search results and less IO during DHT transfer.</td>
+</tr>
+
+<tr class="TableCellDark">
+<td class="small" align="left">'noticed' URLs</td>
+<td class="small" align="right">#[chunkNURL]#</td>
+<td class="small" align="right">#[slreqNURL]#</td>
+<td class="small" align="right">#[slempNURL]#</td>
+<td class="small" align="right">#[slhigNURL]#</td>
+<td class="small" align="right">#[slmedNURL]#</td>
+<td class="small" align="right">#[sllowNURL]#</td>
+<td class="small" align="right">#[usedNURL]#</td>
+<td class="small" align="right"><input name="ramCacheNURL" type="text" align="right" size="8" maxlength="9" value="#[ramCacheNURL]#"> bytes</td>
+<td class="small" align="right">#[goodNURL]#</td>
+<td class="small" align="right">#[bestNURL]#</td>
+<td class="small" align="left">A noticed URL is one that was discovered during crawling but was not loaded yet.
+Increasing the cache size will result in faster double-check during URL recognition when doing crawls.</td>
+</tr>
+
+<tr class="TableCellDark">
+<td class="small" align="left">'error' URLs</td>
+<td class="small" align="right">#[chunkEURL]#</td>
+<td class="small" align="right">#[slreqEURL]#</td>
+<td class="small" align="right">#[slempEURL]#</td>
+<td class="small" align="right">#[slhigEURL]#</td>
+<td class="small" align="right">#[slmedEURL]#</td>
+<td class="small" align="right">#[sllowEURL]#</td>
+<td class="small" align="right">#[usedEURL]#</td>
+<td class="small" align="right"><input name="ramCacheEURL" type="text" align="right" size="8" maxlength="9" value="#[ramCacheEURL]#"> bytes</td>
+<td class="small" align="right">#[goodEURL]#</td>
+<td class="small" align="right">#[bestEURL]#</td>
+<td class="small" align="left">URLs that cannot be loaded are stored in this database. It is also used for double-checked during crawling.
+Increasing the cache size will most probably speed up crawling slightly, but not significantly.</td>
+</tr>
+
+<tr class="TableCellDark">
+<td class="small" align="left">DHT Control</td>
+<td class="small" align="right">#[chunkDHT]#</td>
+<td class="small" align="right">#[slreqDHT]#</td>
+<td class="small" align="right">#[slempDHT]#</td>
+<td class="small" align="right">#[slhigDHT]#</td>
+<td class="small" align="right">#[slmedDHT]#</td>
+<td class="small" align="right">#[sllowDHT]#</td>
+<td class="small" align="right">#[usedDHT]#</td>
+<td class="small" align="right"><input name="ramCacheDHT" type="text" align="right" size="8" maxlength="9" value="#[ramCacheDHT]#"> bytes</td>
+<td class="small" align="right">#[goodDHT]#</td>
+<td class="small" align="right">#[bestDHT]#</td>
+<td class="small" align="left">This is simply the cache for the seed-dbs (active, passive, potential).
+This cache is divided into three equal parts.
+Increasing this cache may speed up many functions, but we need to test this to see the effects.</td>
+</tr>
+
+<tr class="TableCellDark">
+<td class="small" align="left">Messages</td>
+<td class="small" align="right">#[chunkMessage]#</td>
+<td class="small" align="right">#[slreqMessage]#</td>
+<td class="small" align="right">#[slempMessage]#</td>
+<td class="small" align="right">#[slhigMessage]#</td>
+<td class="small" align="right">#[slmedMessage]#</td>
+<td class="small" align="right">#[sllowMessage]#</td>
+<td class="small" align="right">#[usedMessage]#</td>
+<td class="small" align="right"><input name="ramCacheMessage" type="text" align="right" size="8" maxlength="9" value="#[ramCacheMessage]#"> bytes</td>
+<td class="small" align="right">#[goodMessage]#</td>
+<td class="small" align="right">#[bestMessage]#</td>
+<td class="small" align="left">The Message cache for peer-to-peer messages. Less important.</td>
+</tr>
+
+<tr class="TableCellDark">
+<td class="small" align="left">Wiki</td>
+<td class="small" align="right">#[chunkWiki]#</td>
+<td class="small" align="right">#[slreqWiki]#</td>
+<td class="small" align="right">#[slempWiki]#</td>
+<td class="small" align="right">#[slhigWiki]#</td>
+<td class="small" align="right">#[slmedWiki]#</td>
+<td class="small" align="right">#[sllowWiki]#</td>
+<td class="small" align="right">#[usedWiki]#</td>
+<td class="small" align="right"><input name="ramCacheWiki" type="text" align="right" size="8" maxlength="9" value="#[ramCacheWiki]#"> bytes</td>
+<td class="small" align="right">#[goodWiki]#</td>
+<td class="small" align="right">#[bestWiki]#</td>
+<td class="small" align="left">The YaCy-Wiki uses a database to store its pages.
+This cache is divided in two parts, one for the wiki database and one for its backup.
+Increasing this cache may speed up access to the wiki pages.</td>
+</tr>
+
+<tr class="TableCellDark">
+<td class="small" align="left">News</td>
+<td class="small" align="right">#[chunkNews]#</td>
+<td class="small" align="right">#[slreqNews]#</td>
+<td class="small" align="right">#[slempNews]#</td>
+<td class="small" align="right">#[slhigNews]#</td>
+<td class="small" align="right">#[slmedNews]#</td>
+<td class="small" align="right">#[sllowNews]#</td>
+<td class="small" align="right">#[usedNews]#</td>
+<td class="small" align="right"><input name="ramCacheNews" type="text" align="right" size="8" maxlength="9" value="#[ramCacheNews]#"> bytes</td>
+<td class="small" align="right">#[goodNews]#</td>
+<td class="small" align="right">#[bestNews]#</td>
+<td class="small" align="left">The News-DB stores property-lists for news that are included in seeds.
+Increasing this cache may speed up the peer-ping.</td>
+</tr>
+
+<tr class="TableCellLight">
+<td class="small" align="left" colspan="18">
+<input type="submit" name="submitcache" value="Submit New Cache Values">
+Changes take effect after re-start of YaCy</td>
+</tr>
+</form>
+</table>
+</p>
+
+#[footer]#
+</body>
+</html>
diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java new file mode 100644 index 000000000..a20770e70 --- /dev/null +++ b/htroot/PerformanceMemory_p.java @@ -0,0 +1,159 @@ +//PerformaceMemory_p.java
+//-----------------------
+//part of YaCy
+//(C) by Michael Peter Christen; mc@anomic.de
+//first published on http://www.anomic.de
+//Frankfurt, Germany, 2005
+//last major change: 19.09.2005
+//
+//This program is free software; you can redistribute it and/or modify
+//it under the terms of the GNU General Public License as published by
+//the Free Software Foundation; either version 2 of the License, or
+//(at your option) any later version.
+//
+//This program is distributed in the hope that it will be useful,
+//but WITHOUT ANY WARRANTY; without even the implied warranty of
+//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+//GNU General Public License for more details.
+//
+//You should have received a copy of the GNU General Public License
+//along with this program; if not, write to the Free Software
+//Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+//
+//Using this software in any meaning (reading, learning, copying, compiling,
+//running) means that you agree that the Author(s) is (are) not responsible
+//for cost, loss of data or any harm that may be caused directly or indirectly
+//by usage of this softare or this documentation. The usage of this software
+//is on your own risk. The installation and usage (starting/running) of this
+//software may allow other people or application to access your computer and
+//any attached devices and is highly dependent on the configuration of the
+//software which must be done by the user of the software; the author(s) is
+//(are) also not responsible for proper configuration and usage of the
+//software, even if provoked by documentation provided together with
+//the software.
+//
+//Any changes to this file according to the GPL as documented in the file
+//gpl.txt aside this file in the shipment you received can be done to the
+//lines that follows this copyright notice here, but changes must not be
+//done inside the copyright notive above. A re-distribution must contain
+//the intact and unchanged copyright notice.
+//Contributions and changes to the program code must be marked as such.
+
+//You must compile this file with
+//javac -classpath .:../classes PerformanceMemory_p.java
+//if the shell's current path is HTROOT
+
+import java.util.Iterator;
+import java.util.Map;
+import java.io.File;
+
+import de.anomic.http.httpHeader;
+import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.server.serverCore;
+import de.anomic.server.serverObjects;
+import de.anomic.server.serverSwitch;
+import de.anomic.server.serverThread;
+import de.anomic.server.serverFileUtils;
+import de.anomic.yacy.yacyCore;
+import de.anomic.yacy.yacySeedDB;
+
+public class PerformanceMemory_p {
+
+ private static int[] slt;
+ private static int req,chk,usd,bst,god;
+
+ public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) {
+ // return variable that accumulates replacements
+ plasmaSwitchboard switchboard = (plasmaSwitchboard) sb;
+ serverObjects prop = new serverObjects();
+ File defaultSettingsFile = new File(switchboard.getRootPath(), "yacy.init");
+ Map defaultSettings = ((post == null) || (!(post.containsKey("submitdefault")))) ? null : serverFileUtils.loadHashMap(defaultSettingsFile);
+
+ if ((post != null) && (post.containsKey("submitcache"))) {
+ sb.setConfig("ramCacheRWI", post.get("ramCacheRWI", "0"));
+ sb.setConfig("ramCacheHTTP", post.get("ramCacheHTTP", "0"));
+ sb.setConfig("ramCacheLURL", post.get("ramCacheLURL", "0"));
+ sb.setConfig("ramCacheNURL", post.get("ramCacheNURL", "0"));
+ sb.setConfig("ramCacheEURL", post.get("ramCacheEURL", "0"));
+ sb.setConfig("ramCacheDHT", post.get("ramCacheDHT", "0"));
+ sb.setConfig("ramCacheMessage", post.get("ramCacheMessage", "0"));
+ sb.setConfig("ramCacheWiki", post.get("ramCacheWiki", "0"));
+ sb.setConfig("ramCacheNews", post.get("ramCacheNews", "0"));
+ }
+
+ req = switchboard.wordIndex.size();
+ chk = switchboard.wordIndex.assortmentsCacheChunkSizeAvg();
+ slt = switchboard.wordIndex.assortmentsCacheFillStatusCml();
+ calc(); putprop(prop, "RWI");
+ prop.put("ramCacheRWI", sb.getConfig("ramCacheRWI", "0"));
+
+ req = switchboard.cacheManager.size();
+ chk = switchboard.cacheManager.dbCacheChunkSize();
+ slt = switchboard.cacheManager.dbCacheFillStatus();
+ calc(); putprop(prop, "HTTP");
+ prop.put("ramCacheHTTP", sb.getConfig("ramCacheHTTP", "0"));
+
+ req = switchboard.urlPool.loadedURL.urlHashCache.size();
+ chk = switchboard.urlPool.loadedURL.urlHashCache.cacheChunkSize();
+ slt = switchboard.urlPool.loadedURL.urlHashCache.cacheFillStatus();
+ calc(); putprop(prop, "LURL");
+ prop.put("ramCacheLURL", sb.getConfig("ramCacheLURL", "0"));
+
+ req = switchboard.urlPool.noticeURL.urlHashCache.size();
+ chk = switchboard.urlPool.noticeURL.urlHashCache.cacheChunkSize();
+ slt = switchboard.urlPool.noticeURL.urlHashCache.cacheFillStatus();
+ calc(); putprop(prop, "NURL");
+ prop.put("ramCacheNURL", sb.getConfig("ramCacheNURL", "0"));
+
+ req = switchboard.urlPool.errorURL.urlHashCache.size();
+ chk = switchboard.urlPool.errorURL.urlHashCache.cacheChunkSize();
+ slt = switchboard.urlPool.errorURL.urlHashCache.cacheFillStatus();
+ calc(); putprop(prop, "EURL");
+ prop.put("ramCacheEURL", sb.getConfig("ramCacheEURL", "0"));
+
+ req = yacyCore.seedDB.sizeConnected() + yacyCore.seedDB.sizeDisconnected() + yacyCore.seedDB.sizePotential();
+ chk = yacyCore.seedDB.dbCacheChunkSize();
+ slt = yacyCore.seedDB.dbCacheFillStatus();
+ calc(); putprop(prop, "DHT");
+ prop.put("ramCacheDHT", sb.getConfig("ramCacheDHT", "0"));
+
+ req = switchboard.messageDB.size();
+ chk = switchboard.messageDB.dbCacheChunkSize();
+ slt = switchboard.messageDB.dbCacheFillStatus();
+ calc(); putprop(prop, "Message");
+ prop.put("ramCacheMessage", sb.getConfig("ramCacheMessage", "0"));
+
+ req = switchboard.wikiDB.sizeOfTwo();
+ chk = switchboard.wikiDB.dbCacheChunkSize();
+ slt = switchboard.wikiDB.dbCacheFillStatus();
+ calc(); putprop(prop, "Wiki");
+ prop.put("ramCacheWiki", sb.getConfig("ramCacheWiki", "0"));
+
+ req = yacyCore.newsPool.dbSize();
+ chk = yacyCore.newsPool.dbCacheChunkSize();
+ slt = yacyCore.newsPool.dbCacheFillStatus();
+ calc(); putprop(prop, "News");
+ prop.put("ramCacheNews", sb.getConfig("ramCacheNews", "0"));
+
+ // return rewrite values for templates
+ return prop;
+ }
+
+ private static void calc() {
+ usd = chk * (slt[1]+slt[2]+slt[3]);
+ bst = (((chk * req) >> 10) + 1) << 10;
+ god = (((bst / (1+slt[1]+slt[2]+slt[3]) * slt[1]) >> 10) + 1) << 10;
+ }
+
+ private static void putprop(serverObjects prop, String db) {
+ prop.put("chunk" + db, chk);
+ prop.put("slreq" + db, req);
+ prop.put("slemp" + db, slt[0]);
+ prop.put("slhig" + db, slt[1]);
+ prop.put("slmed" + db, slt[2]);
+ prop.put("sllow" + db, slt[3]);
+ prop.put("used" + db, usd);
+ prop.put("good" + db, god);
+ prop.put("best" + db, bst);
+ }
+}
diff --git a/htroot/Performance_p.html b/htroot/PerformanceQueues_p.html index 72fb46cd5..a28e2a689 100644 --- a/htroot/Performance_p.html +++ b/htroot/PerformanceQueues_p.html @@ -1,17 +1,17 @@ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
<html>
<head>
-<title>YaCy '#[clientname]#': Performance</title>
+<title>YaCy '#[clientname]#': Performance Settings of Queues and Processes</title>
#[metas]#
</head>
<body marginheight="0" marginwidth="0" leftmargin="0" topmargin="0">
#[header]#
<br><br>
-<h2>Performance</h2>
+<h2>Performance Settings of Queues and Processes</h2>
<p>
<div class=small><b>Scheduled tasks overview and waiting time settings:</b></div>
<table border="0" cellpadding="2" cellspacing="1">
-<form action="Performance_p.html" method="post" enctype="multipart/form-data">
+<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data">
<tr class="TableHeader" valign="bottom">
<td class="small">Thread</td>
<td class="small">Queue Size</td>
@@ -63,7 +63,7 @@ Changes take effect immediately</td> <p>
<div class=small><b>Indexing Cache Settings:</b></div>
-<form action="Performance_p.html" method="post" enctype="multipart/form-data">
+<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data">
<table border="0" cellpadding="5" cellspacing="1" width="100%">
<tr valign="top" class="TableCellDark">
<td class=small>Words in RAM Cache:</td>
@@ -102,7 +102,7 @@ Changes take effect immediately</td> <p>
<div class=small><b>Thread pool settings:</b></div>
-<form action="Performance_p.html" method="post" enctype="multipart/form-data">
+<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data">
<table border="0" cellpadding="2" cellspacing="1" width="100%">
<tr class="TableHeader" valign="bottom">
<td class="small" width="150">Thread Pool</td>
@@ -149,7 +149,7 @@ Changes take effect immediately</td> <p>
<div class=small><b>Proxy Performance Settings:</b></div>
-<form action="Performance_p.html" method="post" enctype="multipart/form-data">
+<form action="PerformanceQueues_p.html" method="post" enctype="multipart/form-data">
<table border="0" cellpadding="5" cellspacing="1" width="100%">
<tr valign="top" class="TableCellDark">
<td class=small>Online Caution Delay (milliseconds):</td>
diff --git a/htroot/Performance_p.java b/htroot/PerformanceQueues_p.java index 3b821311a..695f26cb4 100644 --- a/htroot/Performance_p.java +++ b/htroot/PerformanceQueues_p.java @@ -1,4 +1,4 @@ -//Performace_p.java
+//PerformaceQueues_p.java
//-----------------------
//part of YaCy
//(C) by Michael Peter Christen; mc@anomic.de
@@ -57,7 +57,7 @@ import de.anomic.server.serverSwitch; import de.anomic.server.serverThread;
import de.anomic.server.serverFileUtils;
-public class Performance_p {
+public class PerformanceQueues_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch sb) {
// return variable that accumulates replacements
diff --git a/htroot/env/templates/header.template b/htroot/env/templates/header.template index 4eb8f1022..d6a53515c 100644 --- a/htroot/env/templates/header.template +++ b/htroot/env/templates/header.template @@ -64,7 +64,7 @@ <tr><td class="MenuItem"> <a href="/News.html" accesskey="n" class="MenuItemLink">News</a></td></tr>
<tr><td class="MenuItem"> <img border="0" src="/env/grafics/lock.gif" align="top"> <a href="/ViewLog_p.html" class="MenuItemLink">Log</a></td></tr>
<tr><td class="MenuItem"> <img border="0" src="/env/grafics/lock.gif" align="top"> <a href="/Settings_p.html" class="MenuItemLink">Settings</a></td></tr>
- <tr><td class="MenuItem"> <img border="0" src="/env/grafics/lock.gif" align="top"> <a href="/Performance_p.html" class="MenuItemLink">Performance</a></td></tr>
+ <tr><td class="MenuItem"> <img border="0" src="/env/grafics/lock.gif" align="top"> <a href="/PerformanceQueues_p.html" class="MenuItemLink">Performance</a></td></tr>
<tr><td class="MenuItem"> <img border="0" src="/env/grafics/lock.gif" align="top"> <a href="/Language_p.html" class="MenuItemLink">Language</a></td></tr>
<tr><td class="MenuItem"> <img border="0" src="/env/grafics/lock.gif" align="top"> <a href="/Skins_p.html" class="MenuItemLink">Skins</a></td></tr>
<tr><td class="MenuSpacer"></td></tr>
diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index 9751f0523..e0d2bb45d 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -78,6 +78,18 @@ public class messageBoard { sn = 0;
}
+ public int size() {
+ return database.size();
+ }
+
+ public int dbCacheChunkSize() {
+ return database.cacheChunkSize();
+ }
+
+ public int[] dbCacheFillStatus() {
+ return database.cacheFillStatus();
+ }
+
public void close() throws IOException {
database.close();
}
diff --git a/source/de/anomic/data/wikiBoard.java b/source/de/anomic/data/wikiBoard.java index edb5fd8fe..01fa1193a 100644 --- a/source/de/anomic/data/wikiBoard.java +++ b/source/de/anomic/data/wikiBoard.java @@ -74,20 +74,34 @@ public class wikiBoard { new File(actpath.getParent()).mkdir();
if (datbase == null) {
if (actpath.exists())
- datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb * 0x40));
+ datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb/2 * 0x40));
else
- datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb * 0x400, keyLength, recordSize));
+ datbase = new kelondroMap(new kelondroDyn(actpath, bufferkb/2 * 0x400, keyLength, recordSize));
}
new File(bkppath.getParent()).mkdir();
if (bkpbase == null) {
if (bkppath.exists())
- bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb * 0x400));
+ bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb/2 * 0x400));
else
- bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb * 0x400, keyLength + dateFormat.length(), recordSize));
+ bkpbase = new kelondroMap(new kelondroDyn(bkppath, bufferkb/2 * 0x400, keyLength + dateFormat.length(), recordSize));
}
sn = 0;
}
+ public int sizeOfTwo() {
+ return datbase.size() + bkpbase.size();
+ }
+
+ public int dbCacheChunkSize() {
+ return (datbase.cacheChunkSize() + bkpbase.cacheChunkSize()) / 2;
+ }
+
+ public int[] dbCacheFillStatus() {
+ int[] a = datbase.cacheFillStatus();
+ int[] b = bkpbase.cacheFillStatus();
+ return new int[]{a[0] + b[0], a[1] + b[1], a[2] + b[2], a[3] + b[3]};
+ }
+
public void close() throws IOException {
datbase.close();
bkpbase.close();
diff --git a/source/de/anomic/kelondro/kelondroMap.java b/source/de/anomic/kelondro/kelondroMap.java index 9308d0590..38e333f32 100644 --- a/source/de/anomic/kelondro/kelondroMap.java +++ b/source/de/anomic/kelondro/kelondroMap.java @@ -127,6 +127,14 @@ public class kelondroMap { return dyn.columnSize(0);
}
+ public int cacheChunkSize() {
+ return dyn.cacheChunkSize();
+ }
+
+ public int[] cacheFillStatus() {
+ return dyn.cacheFillStatus();
+ }
+
public synchronized void set(String key, Map newMap) throws IOException {
// update elementCount
if ((sortfields != null) || (accfields != null)) {
diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index ea9d42466..f08333ece 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -159,19 +159,7 @@ public class kelondroRecords { //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename));
//kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
- this.XcacheSize = (int) (buffersize / ((long) (overhead + columns[0])));
- if (XcacheSize <= 0) {
- XcacheSize = 0;
- this.XcacheHeaders = null;
- this.XcacheScore = null;
- } else {
- // initialize cache
- // in case that we have more cache space than elements
- // we dont need cache control by the XcacheScore
- this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
- this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster();
- }
- this.XcacheStartup = System.currentTimeMillis();
+ initCache(buffersize);
}
public kelondroRecords(kelondroRA ra, long buffersize /* bytes */,
@@ -179,16 +167,7 @@ public class kelondroRecords { int[] columns, int FHandles, int txtProps, int txtPropWidth) throws IOException {
this.filename = null;
init(ra, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
- this.XcacheSize = (int) (buffersize / ((long) (overhead + columns[0])));
- if (XcacheSize <= 0) {
- XcacheSize = 0;
- this.XcacheHeaders = null;
- this.XcacheScore = null;
- } else {
- this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
- this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster();
- }
- this.XcacheStartup = System.currentTimeMillis();
+ initCache(buffersize);
}
private void init(kelondroRA ra, short ohbytec, short ohhandlec,
@@ -275,31 +254,13 @@ public class kelondroRecords { //kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
init(raf);
- this.XcacheSize = (int) (buffersize / ((long) (overhead + COLWIDTHS[0])));
- if (XcacheSize <= 0) {
- XcacheSize = 0;
- this.XcacheHeaders = null;
- this.XcacheScore = null;
- } else {
- this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
- this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster();
- }
- this.XcacheStartup = System.currentTimeMillis();
+ initCache(buffersize);
}
public kelondroRecords(kelondroRA ra, long buffersize) throws IOException{
this.filename = null;
init(ra);
- this.XcacheSize = (int) (buffersize / ((long) (overhead + COLWIDTHS[0])));
- if (XcacheSize <= 0) {
- XcacheSize = 0;
- this.XcacheHeaders = null;
- this.XcacheScore = null;
- } else {
- this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
- this.XcacheScore = (this.XcacheSize > size()) ? null : new kelondroMScoreCluster();
- }
- this.XcacheStartup = System.currentTimeMillis();
+ initCache(buffersize);
}
private void init(kelondroRA ra) throws IOException {
@@ -350,7 +311,38 @@ public class kelondroRecords { this.tailchunksize = this.recordsize - this.headchunksize;
}
+ private void initCache(long buffersize) {
+ if (buffersize <= 0) {
+ this.XcacheSize = 0;
+ this.XcacheHeaders = null;
+ this.XcacheScore = null;
+ } else {
+ if ((buffersize / cacheChunkSize(false)) > size()) {
+ this.XcacheSize = (int) (buffersize / cacheChunkSize(false));
+ this.XcacheScore = null;
+ } else {
+ this.XcacheSize = (int) (buffersize / cacheChunkSize(true));
+ this.XcacheScore = new kelondroMScoreCluster();
+ }
+ this.XcacheHeaders = new HashMap[]{new HashMap(), new HashMap(), new HashMap()};
+ }
+ this.XcacheStartup = System.currentTimeMillis();
+ }
+
+ private int cacheChunkSize(boolean cacheControl) {
+ return this.headchunksize + 14 + ((cacheControl) ? 16 : 0);
+ }
+
+ public int cacheChunkSize() {
+ return cacheChunkSize(this.XcacheScore != null);
+ }
+ public int[] cacheFillStatus() {
+ if (XcacheHeaders == null) return new int[]{0,0,0,0};
+ return new int[]{XcacheSize - (XcacheHeaders[CP_HIGH].size() + XcacheHeaders[CP_MEDIUM].size() + XcacheHeaders[CP_LOW].size()), XcacheHeaders[CP_HIGH].size(), XcacheHeaders[CP_MEDIUM].size(), XcacheHeaders[CP_LOW].size()};
+ }
+
+
protected Node newNode() throws IOException {
return new Node();
}
diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 637535906..62bb75bbd 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -135,6 +135,14 @@ public final class plasmaHTCache { }
}
+ public int dbCacheChunkSize() {
+ return responseHeaderDB.cacheChunkSize();
+ }
+
+ public int[] dbCacheFillStatus() {
+ return responseHeaderDB.cacheFillStatus();
+ }
+
public void push(Entry entry) {
synchronized (this.cacheStack) {
this.cacheStack.add(entry);
diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 6f2bdda7f..18c96eb81 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -154,7 +154,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public static plasmaURLPattern urlBlacklist;
// storage management
- private File cachePath; // do we need that ?
+ private File cachePath;
private File plasmaPath;
public File listsPath;
public plasmaURLPool urlPool;
diff --git a/source/de/anomic/plasma/plasmaURL.java b/source/de/anomic/plasma/plasmaURL.java index 7c905c36d..39cb4c1d6 100644 --- a/source/de/anomic/plasma/plasmaURL.java +++ b/source/de/anomic/plasma/plasmaURL.java @@ -93,7 +93,7 @@ public class plasmaURL { }
// the class object
- protected kelondroTree urlHashCache;
+ public kelondroTree urlHashCache;
private HashSet existsIndex;
public plasmaURL() throws IOException {
diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 1c433355b..8799b3a2f 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -82,7 +82,15 @@ public final class plasmaWordIndex { public int[] assortmentSizes() {
return ramCache.assortmentsSizes();
}
-
+
+ public int assortmentsCacheChunkSizeAvg() {
+ return ramCache.assortmentsCacheChunkSizeAvg();
+ }
+
+ public int[] assortmentsCacheFillStatusCml() {
+ return ramCache.assortmentsCacheFillStatusCml();
+ }
+
public void setMaxWords(int maxWords) {
ramCache.setMaxWords(maxWords);
}
diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortment.java b/source/de/anomic/plasma/plasmaWordIndexAssortment.java index bbff454c4..3380530fe 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortment.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortment.java @@ -213,6 +213,14 @@ public final class plasmaWordIndexAssortment { return assortments.size();
}
+ public int cacheChunkSize() {
+ return assortments.cacheChunkSize();
+ }
+
+ public int[] cacheFillStatus() {
+ return assortments.cacheFillStatus();
+ }
+
public void close() {
try {
assortments.close();
diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java b/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java index 35fed67a5..ec969f4a8 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortmentCluster.java @@ -190,7 +190,22 @@ public final class plasmaWordIndexAssortmentCluster { for (int i = 0; i < clusterCount; i++) sizes[i] = assortments[i].size();
return sizes;
}
-
+
+ public int cacheChunkSizeAvg() {
+ int total = 0;
+ for (int i = 0; i < clusterCount; i++) total += assortments[i].cacheChunkSize();
+ return total / clusterCount;
+ }
+
+ public int[] cacheFillStatusCml() {
+ int[] a, cml = new int[]{0, 0, 0, 0};
+ for (int i = 0; i < clusterCount; i++) {
+ a = assortments[i].cacheFillStatus();
+ for (int j = 0; j < 4; j++) cml[j] += a[j];
+ }
+ return cml;
+ }
+
public void close() {
for (int i = 0; i < clusterCount; i++) assortments[i].close();
}
diff --git a/source/de/anomic/plasma/plasmaWordIndexCache.java b/source/de/anomic/plasma/plasmaWordIndexCache.java index fb0aea3fe..6bf2de41c 100644 --- a/source/de/anomic/plasma/plasmaWordIndexCache.java +++ b/source/de/anomic/plasma/plasmaWordIndexCache.java @@ -265,6 +265,14 @@ public final class plasmaWordIndexCache implements plasmaWordIndexInterface { return assortmentCluster.sizes();
}
+ public int assortmentsCacheChunkSizeAvg() {
+ return assortmentCluster.cacheChunkSizeAvg();
+ }
+
+ public int[] assortmentsCacheFillStatusCml() {
+ return assortmentCluster.cacheFillStatusCml();
+ }
+
public int size() {
return java.lang.Math.max(assortmentCluster.sizeTotal(), java.lang.Math.max(backend.size(), cache.size()));
}
diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index acbedc2d5..cd4c910d8 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -153,20 +153,20 @@ public class yacyCore { yacyDBPath = new File(sb.getRootPath(), sb.getConfig("yacyDB", "DATA/YACYDB"));
if (!(yacyDBPath.exists())) yacyDBPath.mkdir();
- // read memory amount
- int mem = Integer.parseInt(switchboard.getConfig("ramCacheDHT", "1024")) / 1024;
- log.logConfig("DHT Cache memory = " + mem + " KB");
-
// create or init seed cache
+ int memDHT = Integer.parseInt(switchboard.getConfig("ramCacheDHT", "1024")) / 1024;
+ log.logConfig("DHT Cache memory = " + memDHT + " KB");
seedDB = new yacySeedDB(
sb,
new File(yacyDBPath, "seed.new.db"),
new File(yacyDBPath, "seed.old.db"),
new File(yacyDBPath, "seed.pot.db"),
- mem);
+ memDHT);
// create or init news database
- newsPool = new yacyNewsPool(yacyDBPath, 1024);
+ int memNews = Integer.parseInt(switchboard.getConfig("ramCacheNews", "1024")) / 1024;
+ log.logConfig("News Cache memory = " + memNews + " KB");
+ newsPool = new yacyNewsPool(yacyDBPath, memNews);
loadSeedUploadMethods();
diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index d8cb17d85..82ac59961 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -89,6 +89,14 @@ public class yacyNewsDB { news = createDB(path, bufferkb); } + public int dbCacheChunkSize() { + return news.cacheChunkSize(); + } + + public int[] dbCacheFillStatus() { + return news.cacheFillStatus(); + } + public void close() { if (news != null) try {news.close();} catch (IOException e) {} news = null; diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index 5f299761f..1e2c060d0 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -99,6 +99,18 @@ public class yacyNewsPool { maxDistribution = 30; } + public int dbSize() { + return newsDB.size(); + } + + public int dbCacheChunkSize() { + return newsDB.dbCacheChunkSize(); + } + + public int[] dbCacheFillStatus() { + return newsDB.dbCacheFillStatus(); + } + public void publishMyNews(yacyNewsRecord record) throws IOException { // this shall be called if our peer generated a new news record and wants to publish it if (newsDB.get(record.id()) == null) { diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index e2142e53c..86806d566 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -147,10 +147,21 @@ public final class yacySeedDB { } catch (IOException e) {}
}
+ public int dbCacheChunkSize() {
+ return (seedActiveDB.cacheChunkSize() + seedPassiveDB.cacheChunkSize() + seedPotentialDB.cacheChunkSize()) / 3;
+ }
+
+ public int[] dbCacheFillStatus() {
+ int[] ac = seedActiveDB.cacheFillStatus();
+ int[] pa = seedPassiveDB.cacheFillStatus();
+ int[] po = seedPotentialDB.cacheFillStatus();
+ return new int[]{ac[0] + pa[0] + po[0], ac[1] + pa[1] + po[1], ac[2] + pa[2] + po[2], ac[3] + pa[3] + po[3]};
+ }
+
private synchronized kelondroMap openSeedTable(File seedDBFile) throws IOException {
if (seedDBFile.exists()) try {
// open existing seed database
- return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB * 0x400), sortFields, accFields);
+ return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB/3 * 0x400), sortFields, accFields);
} catch (kelondroException e) {
// if we have an error, we start with a fresh database
if (seedDBFile.exists()) seedDBFile.delete();
@@ -160,7 +171,7 @@ public final class yacySeedDB { }
// create new seed database
new File(seedDBFile.getParent()).mkdir();
- return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB * 0x400, commonHashLength, 480), sortFields, accFields);
+ return new kelondroMap(new kelondroDyn(seedDBFile, seedDBBufferKB/3 * 0x400, commonHashLength, 480), sortFields, accFields);
}
private synchronized kelondroMap resetSeedTable(kelondroMap seedDB, File seedDBFile) {
diff --git a/source/yacy.java b/source/yacy.java index 7496f0d81..b64ab53c0 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -166,8 +166,9 @@ public final class yacy { * the main threads.
*
* @param homePath Root-path where all information is to be found.
+ * @param startupFree free memory at startup time, to be used later for statistics
*/
- private static void startup(String homePath) {
+ private static void startup(String homePath, long startupFree) {
long startup = yacyCore.universalTime();
try {
// start up
@@ -201,6 +202,9 @@ public final class yacy { plasmaSwitchboard sb = new plasmaSwitchboard(homePath, "yacy.init", "DATA/SETTINGS/httpProxy.conf");
+ // save information about available memory at startup time
+ sb.setConfig("memoryFreeAfterStartup", startupFree); // for statistics and memory configuration
+
// hardcoded, forced, temporary value-migration
sb.setConfig("htTemplatePath", "htroot/env/templates");
sb.setConfig("parseableExt", "html,htm,txt,php,shtml,asp");
@@ -413,6 +417,9 @@ public final class yacy { Runtime run = Runtime.getRuntime();
run.addShutdownHook(new shutdownHookThread(Thread.currentThread(), sb));
+ // save information about available memory after all initializations
+ System.gc(); sb.setConfig("memoryFreeAfterInit", Runtime.getRuntime().freeMemory());
+
// wait for server shutdown
try {
sb.waitForShutdown();
@@ -766,13 +773,14 @@ public final class yacy { * @param args Given arguments from the command line.
*/
public static void main(String args[]) {
+ System.gc(); long startupFree = Runtime.getRuntime().freeMemory();
String applicationRoot = System.getProperty("user.dir").replace('\\', '/');
//System.out.println("args.length=" + args.length);
//System.out.print("args=["); for (int i = 0; i < args.length; i++) System.out.print(args[i] + ", "); System.out.println("]");
if ((args.length >= 1) && ((args[0].equals("-startup")) || (args[0].equals("-start")))) {
// normal start-up of yacy
if (args.length == 2) applicationRoot= args[1];
- startup(applicationRoot);
+ startup(applicationRoot, startupFree);
} else if ((args.length >= 1) && ((args[0].equals("-shutdown")) || (args[0].equals("-stop")))) {
// normal shutdown of yacy
if (args.length == 2) applicationRoot= args[1];
@@ -800,7 +808,7 @@ public final class yacy { cleanwordlist(args[1], minlength, maxlength);
} else {
if (args.length == 1) applicationRoot= args[0];
- startup(applicationRoot);
+ startup(applicationRoot, startupFree);
}
}
}
@@ -461,6 +461,9 @@ ramCacheMessage = 8192 # ram cache for wiki.db
ramCacheWiki = 8192
+# ram cache for news1.db
+ramCacheNews = 8192
+
# performance properties for the word index cache
# wordCacheMax is the number of word indexes that shall be held in the
# ram cache during indexing. When YaCy is shut down, this cache must be
|