diff options
56 files changed, 737 insertions, 1416 deletions
diff --git a/build.properties b/build.properties index 86a18e21f..3d9a2cc7d 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.4 javacTarget=1.4
# Release Configuration
-releaseVersion=0.506
+releaseVersion=0.507
releaseFile=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
#releaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
releaseDir=yacy_dev_v${releaseVersion}_${DSTAMP}_${releaseNr}
diff --git a/htroot/PerformanceMemory_p.html b/htroot/PerformanceMemory_p.html index 309c7197a..7283cdade 100644 --- a/htroot/PerformanceMemory_p.html +++ b/htroot/PerformanceMemory_p.html @@ -58,355 +58,135 @@ </tr>
</table>
- <form action="PerformanceMemory_p.html" method="post" enctype="multipart/form-data">
- <fieldset>
+ <p><form action="PerformanceMemory_p.html" method="post" enctype="multipart/form-data">
+ <fieldset border="0">
<legend>Memory Settings</legend>
- <dl>
- <dt><label for="Xmx">Maximum used memory</label></dt>
- <dd><input name="Xmx" id="Xmx" type="text" size="4" maxlength="4" value="#[Xmx]#" /> MByte</dd>
- <dt><label for="Xms">Minimal used memory</label></dt>
- <dd><input name="Xms" id="Xms" type="text" size="4" maxlength="4" value="#[Xms]#" /> MByte</dd>
- <dt><input type="submit" name="setStartup" value="Set" /></dt>
- <dd>Changes take effect after <strong>restart</strong> of YaCy</dd>
- </dl>
+ <table border="0" cellpadding="2" cellspacing="1" width="60%">
+ <colgroup><col width="20%"><col width="40%"></colgroup>
+ <tr>
+ <td class="TableHeader" align="right"><label for="Xmx">Memory reserved for JVM</label> </td>
+ <td class="TableCellDark" align="right"><input name="Xmx" id="Xmx" type="text" size="4" maxlength="4" value="#[Xmx]#" /> MByte</td>
+ </tr><tr>
+ <td #(setStartupCommit)#class="TableCellLight"::class="commit"::class="error"#(/setStartupCommit)# colspan="2" align="right">
+ #(setStartupCommit)#::Accepted change. This will take effect after <strong>restart</strong> of YaCy#(/setStartupCommit)#
+ <input type="submit" name="setStartup" value="Set" /></td>
+ </tr>
+ </table>
</fieldset>
- </form>
+ </form></p>
- <p><strong>RAM Cache for Database Files:</strong></p>
- <form action="PerformanceMemory_p.html" method="post" enctype="multipart/form-data">
- <table border="0" cellpadding="2" cellspacing="1">
- <tr class="TableHeader" valign="bottom">
- <td rowspan="2">Database<br /> </td>
- <td rowspan="2">DB Size<br /> </td>
- <td colspan="4">Node Cache</td>
- <td colspan="5">Object Cache</td>
- <td colspan="4">Memory Occupation (KBytes)</td>
- <td rowspan="2">Description<br /> </td>
- </tr>
-
- <tr class="TableHeader" valign="bottom">
- <td>Chunk<br />Size<br />(bytes)</td>
- <td>Empty<br />(avail.)<br />Slots</td>
- <td>Used<br />(filled)<br />Slots</td>
- <td>Node-Cache<br />Hit:Miss<br />Uniq:Doub<br />Del:Flush<br /></td>
- <td>Chunk<br />Size<br />(bytes)</td>
- <td>Max<br />Size<br />Hit/Miss</td>
- <td> <br />Hit-Size<br />Miss-Size</td>
- <td>Hit-Cache<br />Hit:Miss<br />Uniq:Doub<br />Del:Flush<br /></td>
- <td>Miss-Cache<br />Hit:Miss<br />Uniq:Doub<br />Del:Flush<br /></td>
- <td>Used Now</td>
- <td>Assigned Max</td>
- <td>Default Max</td>
- <td>Best Max</td>
- </tr>
- #(useRWICache)#::
- <tr class="TableCellDark">
- <td align="left">RWI Assortment Cluster</td>
- <td align="right">#[slreqRWI]#</td>
- <td align="right">#[nodszRWI]#</td>
- <td align="right">#[slempRWI]#</td>
- <td align="right">#[slfilRWI]#</td>
- <td align="right">#[slhittmissRWI]#<br />#[sluniqdoubRWI]#<br />#[slflushRWI]#</td>
- <td align="right">#[ochunksizRWI]#</td>
- <td align="right">#[ochunkmaxRWI]#<br />#[omisscmaxRWI]#</td>
- <td align="right">#[ohitsizeRWI]#<br />#[omisssizeRWI]#</td>
- <td align="right">#[ohittmissRWI]#<br />#[ouniqdoubRWI]#<br />#[oflushRWI]#</td>
- <td align="right">#[nhittmissRWI]#<br />#[nuniqdoubRWI]#<br />#[nflushRWI]#</td>
- <td align="right">#[usedRWI]#</td>
- <td align="right"><input name="ramCacheRWI" type="text" size="6" maxlength="7" value="#[ramCacheRWI]#" /></td>
- <td align="right">#[dfltRWI]#</td>
- <td align="right">#[bestRWI]#</td>
- <td align="left">The Assortment Cluster stores most of the page indexes.
- Flushing speed of the temporary RWI cache depends on the size of this file cache. Increasing the space of this
- cache will speed up crawls with a depth > 3.</td>
- </tr>#(/useRWICache)#
-
- <tr class="TableCellDark">
- <td align="left">HTTP Response Header</td>
- <td align="right">#[slreqHTTP]#</td>
- <td align="right">#[nodszHTTP]#</td>
- <td align="right">#[slempHTTP]#</td>
- <td align="right">#[slfilHTTP]#</td>
- <td align="right">#[slhittmissHTTP]#<br />#[sluniqdoubHTTP]#<br />#[slflushHTTP]#</td>
- <td align="right">#[ochunksizHTTP]#</td>
- <td align="right">#[ochunkmaxHTTP]#<br />#[omisscmaxHTTP]#</td>
- <td align="right">#[ohitsizeHTTP]#<br />#[omisssizeHTTP]#</td>
- <td align="right">#[ohittmissHTTP]#<br />#[ouniqdoubHTTP]#<br />#[oflushHTTP]#</td>
- <td align="right">#[nhittmissHTTP]#<br />#[nuniqdoubHTTP]#<br />#[nflushHTTP]#</td>
- <td align="right">#[usedHTTP]#</td>
- <td align="right"><input name="ramCacheHTTP" type="text" size="6" maxlength="7" value="#[ramCacheHTTP]#" /></td>
- <td align="right">#[dfltHTTP]#</td>
- <td align="right">#[bestHTTP]#</td>
- <td align="left">The Response Header database stores the HTTP heades that other servers send when YaCy retrieves web pages
- during proxy mode, when performing crawls or if it fetches pages for snippet generation.
- Increasing this cache will be most important for a fast proxy mode.</td>
- </tr>
-
- <tr class="TableCellDark">
- <td align="left">'loaded' URLs</td>
- <td align="right">#[slreqLURL]#</td>
- <td align="right">#[nodszLURL]#</td>
- <td align="right">#[slempLURL]#</td>
- <td align="right">#[slfilLURL]#</td>
- <td align="right">#[slhittmissLURL]#<br />#[sluniqdoubLURL]#<br />#[slflushLURL]#</td>
- <td align="right">#[ochunksizLURL]#</td>
- <td align="right">#[ochunkmaxLURL]#<br />#[omisscmaxLURL]#</td>
- <td align="right">#[ohitsizeLURL]#<br />#[omisssizeLURL]#</td>
- <td align="right">#[ohittmissLURL]#<br />#[ouniqdoubLURL]#<br />#[oflushLURL]#</td>
- <td align="right">#[nhittmissLURL]#<br />#[nuniqdoubLURL]#<br />#[nflushLURL]#</td>
- <td align="right">#[usedLURL]#</td>
- <td align="right"><input name="ramCacheLURL" type="text" size="6" maxlength="7" value="#[ramCacheLURL]#" /></td>
- <td align="right">#[dfltLURL]#</td>
- <td align="right">#[bestLURL]#</td>
- <td align="left">This is the database that holds the hash/url - relation and properties regarding the url like load date and server date.
- This cache is very important for a fast search process. Increasing the cache size will result in more search results and less IO during DHT transfer.</td>
- </tr>
- #(usePreNURLCache)#::
- <tr class="TableCellDark">
- <td align="left">'pre-noticed' URLs</td>
- <td align="right">#[slreqPreNURL]#</td>
- <td align="right">#[nodszPreNURL]#</td>
- <td align="right">#[slempPreNURL]#</td>
- <td align="right">#[slfilPreNURL]#</td>
- <td align="right">#[slhittmissPreNURL]#<br />#[sluniqdoubPreNURL]#<br />#[slflushPreNURL]#</td>
- <td align="right">#[ochunksizPreNURL]#</td>
- <td align="right">#[ochunkmaxPreNURL]#<br />#[omisscmaxPreNURL]#</td>
- <td align="right">#[ohitsizePreNURL]#<br />#[omisssizePreNURL]#</td>
- <td align="right">#[ohittmissPreNURL]#<br />#[ouniqdoubPreNURL]#<br />#[oflushPreNURL]#</td>
- <td align="right">#[nhittmissPreNURL]#<br />#[nuniqdoubPreNURL]#<br />#[nflushPreNURL]#</td>
- <td align="right">#[usedPreNURL]#</td>
- <td align="right"><input name="ramCachePreNURL" type="text" size="6" maxlength="7" value="#[ramCachePreNURL]#" /></td>
- <td align="right">#[dfltPreNURL]#</td>
- <td align="right">#[bestPreNURL]#</td>
- <td align="left"></td>
- </tr>#(/usePreNURLCache)#
- #(useNURLCache)#::
- <tr class="TableCellDark">
- <td align="left">'noticed' URLs</td>
- <td align="right">#[slreqNURL]#</td>
- <td align="right">#[nodszNURL]#</td>
- <td align="right">#[slempNURL]#</td>
- <td align="right">#[slfilNURL]#</td>
- <td align="right">#[slhittmissNURL]#<br />#[sluniqdoubNURL]#<br />#[slflushNURL]#</td>
- <td align="right">#[ochunksizNURL]#</td>
- <td align="right">#[ochunkmaxNURL]#<br />#[omisscmaxNURL]#</td>
- <td align="right">#[ohitsizeNURL]#<br />#[omisssizeNURL]#</td>
- <td align="right">#[ohittmissNURL]#<br />#[ouniqdoubNURL]#<br />#[oflushNURL]#</td>
- <td align="right">#[nhittmissNURL]#<br />#[nuniqdoubNURL]#<br />#[nflushNURL]#</td>
- <td align="right">#[usedNURL]#</td>
- <td align="right"><input name="ramCacheNURL" type="text" size="6" maxlength="7" value="#[ramCacheNURL]#" /></td>
- <td align="right">#[dfltNURL]#</td>
- <td align="right">#[bestNURL]#</td>
- <td align="left">A noticed URL is one that was discovered during crawling but was not loaded yet.
- Increasing the cache size will result in faster double-check during URL recognition when doing crawls.</td>
- </tr>#(/useNURLCache)#
- #(useEURLCache)#::
- <tr class="TableCellDark">
- <td align="left">'error' URLs</td>
- <td align="right">#[slreqEURL]#</td>
- <td align="right">#[nodszEURL]#</td>
- <td align="right">#[slempEURL]#</td>
- <td align="right">#[slfilEURL]#</td>
- <td align="right">#[slhittmissEURL]#<br />#[sluniqdoubEURL]#<br />#[slflushEURL]#</td>
- <td align="right">#[ochunksizEURL]#</td>
- <td align="right">#[ochunkmaxEURL]#<br />#[omisscmaxEURL]#</td>
- <td align="right">#[ohitsizeEURL]#<br />#[omisssizeEURL]#</td>
- <td align="right">#[ohittmissEURL]#<br />#[ouniqdoubEURL]#<br />#[oflushEURL]#</td>
- <td align="right">#[nhittmissEURL]#<br />#[nuniqdoubEURL]#<br />#[nflushEURL]#</td>
- <td align="right">#[usedEURL]#</td>
- <td align="right"><input name="ramCacheEURL" type="text" size="6" maxlength="7" value="#[ramCacheEURL]#" /></td>
- <td align="right">#[dfltEURL]#</td>
- <td align="right">#[bestEURL]#</td>
- <td align="left">URLs that cannot be loaded are stored in this database. It is also used for double-checked during crawling.
- Increasing the cache size will most probably speed up crawling slightly, but not significantly.</td>
- </tr>#(/useEURLCache)#
-
- <tr class="TableCellDark">
- <td align="left">DHT Control</td>
- <td align="right">#[slreqDHT]#</td>
- <td align="right">#[nodszDHT]#</td>
- <td align="right">#[slempDHT]#</td>
- <td align="right">#[slfilDHT]#</td>
- <td align="right">#[slhittmissDHT]#<br />#[sluniqdoubDHT]#<br />#[slflushDHT]#</td>
- <td align="right">#[ochunksizDHT]#</td>
- <td align="right">#[ochunkmaxDHT]#<br />#[omisscmaxDHT]#</td>
- <td align="right">#[ohitsizeDHT]#<br />#[omisssizeDHT]#</td>
- <td align="right">#[ohittmissDHT]#<br />#[ouniqdoubDHT]#<br />#[oflushDHT]#</td>
- <td align="right">#[nhittmissDHT]#<br />#[nuniqdoubDHT]#<br />#[nflushDHT]#</td>
- <td align="right">#[usedDHT]#</td>
- <td align="right"><input name="ramCacheDHT" type="text" size="6" maxlength="7" value="#[ramCacheDHT]#" /></td>
- <td align="right">#[dfltDHT]#</td>
- <td align="right">#[bestDHT]#</td>
- <td align="left">This is simply the cache for the seed-dbs (active, passive, potential).
- This cache is divided into three equal parts.
- Increasing this cache may speed up many functions, but we need to test this to see the effects.</td>
- </tr>
-
- <tr class="TableCellDark">
- <td align="left">Messages</td>
- <td align="right">#[slreqMessage]#</td>
- <td align="right">#[nodszMessage]#</td>
- <td align="right">#[slempMessage]#</td>
- <td align="right">#[slfilMessage]#</td>
- <td align="right">#[slhittmissMessage]#<br />#[sluniqdoubMessage]#<br />#[slflushMessage]#</td>
- <td align="right">#[ochunksizMessage]#</td>
- <td align="right">#[ochunkmaxMessage]#<br />#[omisscmaxMessage]#</td>
- <td align="right">#[ohitsizeMessage]#<br />#[omisssizeMessage]#</td>
- <td align="right">#[ohittmissMessage]#<br />#[ouniqdoubMessage]#<br />#[oflushMessage]#</td>
- <td align="right">#[nhittmissMessage]#<br />#[nuniqdoubMessage]#<br />#[nflushMessage]#</td>
- <td align="right">#[usedMessage]#</td>
- <td align="right"><input name="ramCacheMessage" type="text" size="6" maxlength="7" value="#[ramCacheMessage]#" /></td>
- <td align="right">#[dfltMessage]#</td>
- <td align="right">#[bestMessage]#</td>
- <td align="left">The Message cache for peer-to-peer messages. Less important.</td>
- </tr>
-
- <tr class="TableCellDark">
- <td align="left">Wiki</td>
- <td align="right">#[slreqWiki]#</td>
- <td align="right">#[nodszWiki]#</td>
- <td align="right">#[slempWiki]#</td>
- <td align="right">#[slfilWiki]#</td>
- <td align="right">#[slhittmissWiki]#<br />#[sluniqdoubWiki]#<br />#[slflushWiki]#</td>
- <td align="right">#[ochunksizWiki]#</td>
- <td align="right">#[ochunkmaxWiki]#<br />#[omisscmaxWiki]#</td>
- <td align="right">#[ohitsizeWiki]#<br />#[omisssizeWiki]#</td>
- <td align="right">#[ohittmissWiki]#<br />#[ouniqdoubWiki]#<br />#[oflushWiki]#</td>
- <td align="right">#[nhittmissWiki]#<br />#[nuniqdoubWiki]#<br />#[nflushWiki]#</td>
- <td align="right">#[usedWiki]#</td>
- <td align="right"><input name="ramCacheWiki" type="text" size="6" maxlength="7" value="#[ramCacheWiki]#" /></td>
- <td align="right">#[dfltWiki]#</td>
- <td align="right">#[bestWiki]#</td>
- <td align="left">The YaCy-Wiki uses a database to store its pages.
- This cache is divided in two parts, one for the wiki database and one for its backup.
- Increasing this cache may speed up access to the wiki pages.</td>
- </tr>
-
- <tr class="TableCellDark">
- <td align="left">Blog</td>
- <td align="right">#[slreqBlog]#</td>
- <td align="right">#[nodszBlog]#</td>
- <td align="right">#[slempBlog]#</td>
- <td align="right">#[slfilBlog]#</td>
- <td align="right">#[slhittmissBlog]#<br />#[sluniqdoubBlog]#<br />#[slflushBlog]#</td>
- <td align="right">#[ochunksizBlog]#</td>
- <td align="right">#[ochunkmaxBlog]#</td>
- <td align="right">#[ohitsizeBlog]#<br />#[omisssizeBlog]#</td>
- <td align="right">#[ohittmissBlog]#<br />#[ouniqdoubBlog]#<br />#[oflushBlog]#</td>
- <td align="right">#[nhittmissBlog]#<br />#[nuniqdoubBlog]#<br />#[nflushBlog]#</td>
- <td align="right">#[usedBlog]#</td>
- <td align="right"><input name="ramCacheBlog" type="text" size="6" maxlength="7" value="#[ramCacheBlog]#" /></td>
- <td align="right">#[dfltBlog]#</td>
- <td align="right">#[bestBlog]#</td>
- <td align="left">The YaCy-Blog uses a database to store its entries.
- Increasing this cache may speed up access to the Blog.</td>
- </tr>
-
- <tr class="TableCellDark">
- <td align="left">News</td>
- <td align="right">#[slreqNews]#</td>
- <td align="right">#[nodszNews]#</td>
- <td align="right">#[slempNews]#</td>
- <td align="right">#[slfilNews]#</td>
- <td align="right">#[slhittmissNews]#<br />#[sluniqdoubNews]#<br />#[slflushNews]#</td>
- <td align="right">#[ochunksizNews]#</td>
- <td align="right">#[ochunkmaxNews]#</td>
- <td align="right">#[ohitsizeNews]#<br />#[omisssizeNews]#</td>
- <td align="right">#[ohittmissNews]#<br />#[ouniqdoubNews]#<br />#[oflushNews]#</td>
- <td align="right">#[nhittmissNews]#<br />#[nuniqdoubNews]#<br />#[nflushNews]#</td>
- <td align="right">#[usedNews]#</td>
- <td align="right"><input name="ramCacheNews" type="text" size="6" maxlength="7" value="#[ramCacheNews]#" /></td>
- <td align="right">#[dfltNews]#</td>
- <td align="right">#[bestNews]#</td>
- <td align="left">The News-DB stores property-lists for news that are included in seeds.
- Increasing this cache may speed up the peer-ping.</td>
- </tr>
-
- <tr class="TableCellDark">
- <td align="left">robots.txt DB</td>
- <td align="right">#[slreqRobots]#</td>
- <td align="right">#[nodszRobots]#</td>
- <td align="right">#[slempRobots]#</td>
- <td align="right">#[slfilRobots]#</td>
- <td align="right">#[slhittmissRobots]#<br />#[sluniqdoubRobots]#<br />#[slflushRobots]#</td>
- <td align="right">#[ochunksizRobots]#</td>
- <td align="right">#[ochunkmaxRobots]#</td>
- <td align="right">#[ohitsizeRobots]#<br />#[omisssizeRobots]#</td>
- <td align="right">#[ohittmissRobots]#<br />#[ouniqdoubRobots]#<br />#[oflushRobots]#</td>
- <td align="right">#[nhittmissRobots]#<br />#[nuniqdoubRobots]#<br />#[nflushRobots]#</td>
- <td align="right">#[usedRobots]#</td>
- <td align="right"><input name="ramCacheRobots" type="text" size="6" maxlength="7" value="#[ramCacheRobots]#" /></td>
- <td align="right">#[dfltRobots]#</td>
- <td align="right">#[bestRobots]#</td>
- <td align="left">The robots.txt DB stores downloaded records from robots.txt files.
- Increasing this cache may speed up validation if crawling of the URL is allowed.</td>
- </tr>
-
- <tr class="TableCellDark">
- <td align="left">Crawl Profiles</td>
- <td align="right">#[slreqProfiles]#</td>
- <td align="right">#[nodszProfiles]#</td>
- <td align="right">#[slempProfiles]#</td>
- <td align="right">#[slfilProfiles]#</td>
- <td align="right">#[slhittmissProfiles]#<br />#[sluniqdoubProfiles]#<br />#[slflushProfiles]#</td>
- <td align="right">#[ochunksizProfiles]#</td>
- <td align="right">#[ochunkmaxProfiles]#</td>
- <td align="right">#[ohitsizeProfiles]#<br />#[omisssizeProfiles]#</td>
- <td align="right">#[ohittmissProfiles]#<br />#[ouniqdoubProfiles]#<br />#[oflushProfiles]#</td>
- <td align="right">#[nhittmissProfiles]#<br />#[nuniqdoubProfiles]#<br />#[nflushProfiles]#</td>
- <td align="right">#[usedProfiles]#</td>
- <td align="right"><input name="ramCacheProfiles" type="text" size="6" maxlength="7" value="#[ramCacheProfiles]#" /></td>
- <td align="right">#[dfltProfiles]#</td>
- <td align="right">#[bestProfiles]#</td>
- <td align="left">The profile database stores properties for each crawl that is started on the local peer.
- Increasing this cache may speed up crawling, but not much space is needed, so the effect may be low.</td>
- </tr>
-
- <tr class="TableCellSummary">
- <td align="left" colspan="11">Totals</td>
- <td align="right">#[usedTotal]# MB</td>
- <td align="right">#[currTotal]# MB</td>
- <td align="right">#[dfltTotal]# MB</td>
- <td align="right">#[bestTotal]# MB</td>
- <td align="left">Sum of memory amounts</td>
- </tr>
-
- <tr class="TableCellLight">
- <td align="right" colspan="12">Re-Configuration:</td>
- <td align="center"><input type="submit" name="setCustom" value="Set" /><br />these<br />custom<br />values<br /> </td>
- <td align="center"><input type="submit" name="setDefault" value="Set" /><br />all<br />default<br />values<br /> </td>
- <td align="center"><input type="submit" name="setBest" value="Set" /><br />all<br />optimum<br />values<br /> </td>
- <td align="left">Changes take effect after <strong>re-start</strong> of YaCy</td>
- </tr>
-
- </table>
- </form>
+ <p><strong>FlexTable RAM Index:</strong></p>
+ <table border="0" cellpadding="2" cellspacing="1">
+ <tr class="TableHeader" valign="bottom">
+ <td>Table</td>
+ <td>Chunk Size</td>
+ <td>Count</td>
+ <td>Used Memory</td>
+ </tr>
+ #{TableList}#
+ <tr class="TableCellLight">
+ <td align="left">#[tableIndexPath]#</td>
+ <td align="right">#[tableIndexChunkSize]#</td>
+ <td align="right">#[tableIndexCount]#</td>
+ <td align="right">#[tableIndexMem]# MB</td>
+ </tr>
+ #{/TableList}#
+ <tr class="TableCellDark">
+ <td colspan="4">Total Mem = #[TableIndexTotalMem]# MB</td>
+ </tr>
+ </table></p>
- <!--
- <p><strong>Write Cache Object Allocation:</strong></p>
+ <p><strong>Node Caches:</strong></p>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
- <td> <br />Chunk Sizes</td>
- #{sizes}#
- <td>#[chunk]#</td>
- #{/sizes}#
+ <td>Table</td>
+ <td>Chunk Size</td>
+ <td>Count</td>
+ <td>Used Memory</td>
+ <td>Read Hit</td>
+ <td>Read Miss</td>
+ <td>Write Unique</td>
+ <td>Write Double</td>
+ <td>Deletes</td>
+ <td>Flushes</td>
</tr>
+ #{NodeList}#
+ <tr class="TableCellLight">
+ <td align="left">#[nodeCachePath]#</td>
+ <td align="right">#[nodeChunkSize]#</td>
+ <td align="right">#[nodeCacheCount]#</td>
+ <td align="right">#[nodeCacheMem]# MB</td>
+ <td align="right">#[nodeCacheReadHit]#</td>
+ <td align="right">#[nodeCacheReadMiss]#</td>
+ <td align="right">#[nodeCacheWriteUnique]#</td>
+ <td align="right">#[nodeCacheWriteDouble]#</td>
+ <td align="right">#[nodeCacheDeletes]#</td>
+ <td align="right">#[nodeCacheFlushes]#</td>
+ </tr>
+ #{/NodeList}#
<tr class="TableCellDark">
- <td class="TableHeader" align="left">now alive in write cache</td>
- #{alive}#
- <td>#[count]#</td>
- #{/alive}#
+ <td colspan="10">Total Mem = #[nodeCacheTotalMem]# MB; Stop Grow when less than #[nodeCacheStopGrow]# MB available left; Start Shrink when less than #[nodeCacheStartShrink]# MB availabe left</td>
</tr>
+ </table></p>
+
+ <p><strong>Object Read Caches:</strong></p>
+ <table border="0" cellpadding="2" cellspacing="1">
+ <tr class="TableHeader" valign="bottom">
+ <td></td>
+ <td colspan="9">Read Hit Cache</td>
+ <td colspan="9">Read Miss Cache</td>
+ </tr>
+ <tr class="TableHeader" valign="bottom">
+ <td>Table</td>
+ <td>Chunk Size</td>
+ <td>Count</td>
+ <td>Used Memory</td>
+ <td>Read Hit</td>
+ <td>Read Miss</td>
+ <td>Write Unique</td>
+ <td>Write Double</td>
+ <td>Deletes</td>
+ <td>Flushes</td>
+ <td>Chunk Size</td>
+ <td>Count</td>
+ <td>Used Memory</td>
+ <td>Read Hit</td>
+ <td>Read Miss</td>
+ <td>Write Unique</td>
+ <td>Write Double</td>
+ <td>Deletes</td>
+ <td>Flushes</td>
+ </tr>
+ #{ObjectList}#
+ <tr class="TableCellLight">
+ <td align="left">#[objectCachePath]#</td>
+ <td align="right">#[objectHitChunkSize]#</td>
+ <td align="right">#[objectHitCacheCount]#</td>
+ <td align="right">#[objectHitCacheMem]# MB</td>
+ <td align="right">#[objectHitCacheReadHit]#</td>
+ <td align="right">#[objectHitCacheReadMiss]#</td>
+ <td align="right">#[objectHitCacheWriteUnique]#</td>
+ <td align="right">#[objectHitCacheWriteDouble]#</td>
+ <td align="right">#[objectHitCacheDeletes]#</td>
+ <td align="right">#[objectHitCacheFlushes]#</td>
+ <td align="right">#[objectMissChunkSize]#</td>
+ <td align="right">#[objectMissCacheCount]#</td>
+ <td align="right">#[objectMissCacheMem]# MB</td>
+ <td align="right">#[objectMissCacheReadHit]#</td>
+ <td align="right">#[objectMissCacheReadMiss]#</td>
+ <td align="right">#[objectMissCacheWriteUnique]#</td>
+ <td align="right">#[objectMissCacheWriteDouble]#</td>
+ <td align="right">#[objectMissCacheDeletes]#</td>
+ <td align="right">#[objectMissCacheFlushes]#</td>
+ </tr>
+ #{/ObjectList}#
<tr class="TableCellDark">
- <td class="TableHeader" align="left">currently held in write buffer heap</td>
- #{heap}#
- <td>#[count]#</td>
- #{/heap}#
+ <td colspan="19">Total Mem: #[objectHitCacheTotalMem]# MB (hit), #[objectMissCacheTotalMem]# MB (miss); Stop Grow when less than #[objectCacheStopGrow]# MB available left; Start Shrink when less than #[objectCacheStartShrink]# MB availabe left</td>
</tr>
- </table>
- -->
+ </table></p>
+
<!-- other cache sizes -->
- <p><strong>Other caching structures:</strong></p>
+ <p><strong>Other Caching Structures:</strong></p>
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
<td>Type</td>
@@ -424,7 +204,7 @@ <td>HashBlacklistedCache</td>
<td>#[blacklistcache.size]#</td>
</tr>
- </table>
+ </table></p>
#%env/templates/footer.template%#
</body>
</html>
diff --git a/htroot/PerformanceMemory_p.java b/htroot/PerformanceMemory_p.java index cb052e6da..fadb93626 100644 --- a/htroot/PerformanceMemory_p.java +++ b/htroot/PerformanceMemory_p.java @@ -45,16 +45,19 @@ //import java.util.Iterator;
import java.io.File;
+import java.util.Iterator;
import java.util.Map;
import de.anomic.http.httpHeader;
import de.anomic.http.httpc;
+import de.anomic.kelondro.kelondroCache;
+import de.anomic.kelondro.kelondroFlexTable;
+import de.anomic.kelondro.kelondroRecords;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
//import de.anomic.kelondro.kelondroObjectSpace;
public class PerformanceMemory_p {
@@ -63,73 +66,20 @@ public class PerformanceMemory_p { private static final long MB = 1024 * KB;
private static Map defaultSettings = null;
- private static int chk, obj;
- private static int[] slt;
- private static long[] ost;
- private static long req, usd, bst;
-
- private static long usedTotal, currTotal, dfltTotal, bestTotal;
-
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
- plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if (defaultSettings == null) {
defaultSettings = serverFileUtils.loadHashMap(new File(env.getRootPath(), "yacy.init"));
}
prop.put("gc", 0);
- String set = "";
if (post != null) {
- if (post.containsKey("setCustom")) {
- env.setConfig("ramCacheRWI", Long.parseLong(post.get("ramCacheRWI", "0")) * KB);
- env.setConfig("ramCacheHTTP", Long.parseLong(post.get("ramCacheHTTP", "0")) * KB);
- env.setConfig("ramCacheLURL", Long.parseLong(post.get("ramCacheLURL", "0")) * KB);
- env.setConfig("ramCacheNURL", Long.parseLong(post.get("ramCacheNURL", "0")) * KB);
- env.setConfig("ramCacheEURL", Long.parseLong(post.get("ramCacheEURL", "0")) * KB);
- env.setConfig("ramCacheDHT", Long.parseLong(post.get("ramCacheDHT", "0")) * KB);
- env.setConfig("ramCacheMessage", Long.parseLong(post.get("ramCacheMessage", "0")) * KB);
- env.setConfig("ramCacheWiki", Long.parseLong(post.get("ramCacheWiki", "0")) * KB);
- env.setConfig("ramCacheBlog", Long.parseLong(post.get("ramCacheBlog", "0")) * KB);
- env.setConfig("ramCacheNews", Long.parseLong(post.get("ramCacheNews", "0")) * KB);
- env.setConfig("ramCacheRobots", Long.parseLong(post.get("ramCacheRobots", "0")) * KB);
- env.setConfig("ramCacheProfiles", Long.parseLong(post.get("ramCacheProfiles", "0")) * KB);
- env.setConfig("ramCachePreNURL", Long.parseLong(post.get("ramCachePreNURL", "0")) * KB);
- }
- if (post.containsKey("setDefault")) {
- env.setConfig("ramCacheRWI", Long.parseLong((String) defaultSettings.get("ramCacheRWI")));
- env.setConfig("ramCacheHTTP", Long.parseLong((String) defaultSettings.get("ramCacheHTTP")));
- env.setConfig("ramCacheLURL", Long.parseLong((String) defaultSettings.get("ramCacheLURL")));
- env.setConfig("ramCacheNURL", Long.parseLong((String) defaultSettings.get("ramCacheNURL")));
- env.setConfig("ramCacheEURL", Long.parseLong((String) defaultSettings.get("ramCacheEURL")));
- env.setConfig("ramCacheDHT", Long.parseLong((String) defaultSettings.get("ramCacheDHT")));
- env.setConfig("ramCacheMessage", Long.parseLong((String) defaultSettings.get("ramCacheMessage")));
- env.setConfig("ramCacheWiki", Long.parseLong((String) defaultSettings.get("ramCacheWiki")));
- env.setConfig("ramCacheWiki", Long.parseLong((String) defaultSettings.get("ramCacheBlog")));
- env.setConfig("ramCacheNews", Long.parseLong((String) defaultSettings.get("ramCacheNews")));
- env.setConfig("ramCacheRobots", Long.parseLong((String) defaultSettings.get("ramCacheRobots")));
- env.setConfig("ramCacheProfiles", Long.parseLong((String) defaultSettings.get("ramCacheProfiles")));
- env.setConfig("ramCachePreNURL", Long.parseLong((String) defaultSettings.get("ramCachePreNURL")));
- }
- if (post.containsKey("setGood")) set = "setGood";
- if (post.containsKey("setBest")) set = "setBest";
- if (post.containsKey("gc")) {
- Runtime.getRuntime().gc();
- prop.put("gc", 1);
- }
-
- int xms = 10;
- if (post.containsKey("Xms")) {
- try { xms = Integer.valueOf(post.get("Xms", "10")).intValue(); } catch (NumberFormatException e){}
- env.setConfig("javastart_Xms", "Xms" + xms + "m");
- }
- int xmx = 64; // default maximum heap size
+ int xmx = 96; // default maximum heap size
if (post.containsKey("Xmx")) {
try { xmx = Integer.valueOf(post.get("Xmx", "64")).intValue(); } catch (NumberFormatException e){}
- // initial heap size must be less than or equal to the maximum heap size
- if (xmx < xms) xmx = xms;
env.setConfig("javastart_Xmx", "Xmx" + xmx + "m");
- } else if (xmx < xms) {
- env.setConfig("javastart_Xmx", "Xmx" + xms + "m");
+ env.setConfig("javastart_Xms", "Xms" + xmx + "m");
+ prop.put("setStartupCommit", 1);
}
}
@@ -161,158 +111,155 @@ public class PerformanceMemory_p { prop.put("memoryUsedAfterInitAGC", (memoryTotalAfterInitAGC - memoryFreeAfterInitAGC) / KB);
prop.put("memoryUsedNow", (memoryTotalNow - memoryFreeNow) / MB);
- usedTotal = 0;
- currTotal = 0;
- dfltTotal = 0;
- bestTotal = 0;
-
- prop.put("useRWICache", 0);
-
- req = sb.cacheManager.dbSize();
- chk = sb.cacheManager.cacheNodeChunkSize();
- obj = sb.cacheManager.cacheObjectChunkSize();
- slt = sb.cacheManager.cacheNodeStatus();
- ost = sb.cacheManager.cacheObjectStatus();
- putprop(prop, env, "", "HTTP", set);
+ // write table for FlexTable index sizes
+ Iterator i = kelondroFlexTable.filenames();
+ String filename;
+ Map map;
+ int p, c = 0;
+ long mem, totalmem = 0;
+ while (i.hasNext()) {
+ filename = (String) i.next();
+ map = (Map) kelondroFlexTable.memoryStats(filename);
+ p = filename.indexOf("DATA");
+ mem = Long.parseLong((String) map.get("tableIndexMem"));
+ totalmem += mem;
+ prop.put("TableList_" + c + "_tableIndexPath", filename.substring(p));
+ prop.put("TableList_" + c + "_tableIndexChunkSize", map.get("tableIndexChunkSize"));
+ prop.put("TableList_" + c + "_tableIndexCount", map.get("tableIndexCount"));
+ prop.put("TableList_" + c + "_tableIndexMem", mem / (1024 * 1024));
+ c++;
+ }
+ prop.put("TableList", c);
+ prop.put("TableIndexTotalMem", totalmem / (1024 * 1024));
- req = sb.wordIndex.loadedURL.size();
- chk = sb.wordIndex.loadedURL.cacheNodeChunkSize();
- obj = sb.wordIndex.loadedURL.cacheObjectChunkSize();
- slt = sb.wordIndex.loadedURL.cacheNodeStatus();
- ost = sb.wordIndex.loadedURL.cacheObjectStatus();
- putprop(prop, env, "", "LURL", set);
- if (sb.sbStackCrawlThread.getDBType() != de.anomic.plasma.plasmaCrawlStacker.QUEUE_DB_TYPE_TREE) {
- prop.put("usePreNURLCache", 0);
- } else {
- prop.put("usePreNURLCache", 1);
- req = sb.sbStackCrawlThread.size();
- chk = sb.sbStackCrawlThread.cacheNodeChunkSize();
- obj = sb.sbStackCrawlThread.cacheObjectChunkSize();
- slt = sb.sbStackCrawlThread.cacheNodeStatus();
- ost = sb.sbStackCrawlThread.cacheObjectStatus();
- putprop(prop, env, "usePreNURLCache", "PreNURL", set);
- }
+ /*
+ * <p><strong>FlexTable RAM Index:</strong></p>
+ <table border="0" cellpadding="2" cellspacing="1">
+ <tr class="TableHeader" valign="bottom">
+ <td>Table</td>
+ <td>Chunk Size</td>
+ <td>Count</td>
+ <td>Used Memory</td>
+ </tr>
+ #{TableList}#
+ <tr class="TableCellLight">
+ <td align="left">#[tableIndexPath]#</td>
+ <td align="right">#[tableIndexChunkSize]#</td>
+ <td align="right">#[tableIndexCount]#</td>
+ <td align="right">#[tableIndexMem]# MB</td>
+ </tr>
+ #{/TableList}#
+ <tr class="TableCellDark">
+ <td colspan="10">Total Mem = #[TableIndexTotalMem]# MB</td>
+ </tr>
+ </table></p>
+ */
- if (sb.noticeURL.getUseNewDB()) {
- prop.put("useNURLCache", 0);
- } else {
- prop.put("useNURLCache", 1);
- req = sb.noticeURL.size();
- chk = sb.noticeURL.cacheNodeChunkSize();
- obj = sb.noticeURL.cacheObjectChunkSize();
- slt = sb.noticeURL.cacheNodeStatus();
- ost = sb.noticeURL.cacheObjectStatus();
- putprop(prop, env, "useNURLCache", "NURL", set);
+ // write node cache table
+ i = kelondroRecords.filenames();
+ c = 0;
+ totalmem = 0;
+ while (i.hasNext()) {
+ filename = (String) i.next();
+ map = (Map) kelondroRecords.memoryStats(filename);
+ p = filename.indexOf("DATA");
+ mem = Long.parseLong((String) map.get("nodeCacheMem"));
+ totalmem += mem;
+ prop.put("NodeList_" + c + "_nodeCachePath", filename.substring(p));
+ prop.put("NodeList_" + c + "_nodeChunkSize", map.get("nodeChunkSize"));
+ prop.put("NodeList_" + c + "_nodeCacheCount", map.get("nodeCacheCount"));
+ prop.put("NodeList_" + c + "_nodeCacheMem", mem / (1024 * 1024));
+ prop.put("NodeList_" + c + "_nodeCacheReadHit", map.get("nodeCacheReadHit"));
+ prop.put("NodeList_" + c + "_nodeCacheReadMiss", map.get("nodeCacheReadMiss"));
+ prop.put("NodeList_" + c + "_nodeCacheWriteUnique", map.get("nodeCacheWriteUnique"));
+ prop.put("NodeList_" + c + "_nodeCacheWriteDouble", map.get("nodeCacheWriteDouble"));
+ prop.put("NodeList_" + c + "_nodeCacheDeletes", map.get("nodeCacheDeletes"));
+ prop.put("NodeList_" + c + "_nodeCacheFlushes", map.get("nodeCacheFlushes"));
+ c++;
}
+ prop.put("NodeList", c);
+ prop.put("nodeCacheStopGrow", kelondroRecords.getMemStopGrow() / (1024 * 1024));
+ prop.put("nodeCacheStartShrink", kelondroRecords.getMemStartShrink() / (1024 * 1024));
+ prop.put("nodeCacheTotalMem", totalmem / (1024 * 1024));
- if (sb.errorURL.getUseNewDB()) {
- prop.put("useEURLCache", 0);
- } else {
- prop.put("useEURLCache", 1);
- req = sb.errorURL.size();
- chk = sb.errorURL.cacheNodeChunkSize();
- obj = sb.errorURL.cacheObjectChunkSize();
- slt = sb.errorURL.cacheNodeStatus();
- ost = sb.errorURL.cacheObjectStatus();
- putprop(prop, env, "useEURLCache", "EURL", set);
+ // write object cache table
+ i = kelondroCache.filenames();
+ c = 0;
+ long hitmem, missmem, totalhitmem = 0, totalmissmem = 0;
+ while (i.hasNext()) {
+ filename = (String) i.next();
+ map = (Map) kelondroCache.memoryStats(filename);
+ p = filename.indexOf("DATA");
+ prop.put("ObjectList_" + c + "_objectCachePath", filename.substring(p));
+
+ // hit cache
+ hitmem = Long.parseLong((String) map.get("objectHitMem"));
+ totalhitmem += hitmem;
+ prop.put("ObjectList_" + c + "_objectHitChunkSize", map.get("objectHitChunkSize"));
+ prop.put("ObjectList_" + c + "_objectHitCacheCount", map.get("objectHitCacheCount"));
+ prop.put("ObjectList_" + c + "_objectHitCacheMem", hitmem / (1024 * 1024));
+ prop.put("ObjectList_" + c + "_objectHitCacheReadHit", map.get("objectHitCacheReadHit"));
+ prop.put("ObjectList_" + c + "_objectHitCacheReadMiss", map.get("objectHitCacheReadMiss"));
+ prop.put("ObjectList_" + c + "_objectHitCacheWriteUnique", map.get("objectHitCacheWriteUnique"));
+ prop.put("ObjectList_" + c + "_objectHitCacheWriteDouble", map.get("objectHitCacheWriteDouble"));
+ prop.put("ObjectList_" + c + "_objectHitCacheDeletes", map.get("objectHitCacheDeletes"));
+ prop.put("ObjectList_" + c + "_objectHitCacheFlushes", map.get("objectHitCacheFlushes"));
+
+ // miss cache
+ missmem = Long.parseLong((String) map.get("objectMissMem"));
+ totalmissmem += missmem;
+ prop.put("ObjectList_" + c + "_objectMissChunkSize", map.get("objectMissChunkSize"));
+ prop.put("ObjectList_" + c + "_objectMissCacheCount", map.get("objectMissCacheCount"));
+ prop.put("ObjectList_" + c + "_objectMissCacheMem", missmem / (1024 * 1024));
+ prop.put("ObjectList_" + c + "_objectMissCacheReadHit", map.get("objectMissCacheReadHit"));
+ prop.put("ObjectList_" + c + "_objectMissCacheReadMiss", map.get("objectMissCacheReadMiss"));
+ prop.put("ObjectList_" + c + "_objectMissCacheWriteUnique", map.get("objectMissCacheWriteUnique"));
+ prop.put("ObjectList_" + c + "_objectMissCacheWriteDouble", map.get("objectMissCacheWriteDouble"));
+ prop.put("ObjectList_" + c + "_objectMissCacheDeletes", map.get("objectMissCacheDeletes"));
+ prop.put("ObjectList_" + c + "_objectMissCacheFlushes", map.get("objectMissCacheFlushes"));
+
+ c++;
}
-
- req = yacyCore.seedDB.sizeConnected() + yacyCore.seedDB.sizeDisconnected() + yacyCore.seedDB.sizePotential();
- chk = yacyCore.seedDB.cacheNodeChunkSize();
- obj = yacyCore.seedDB.cacheObjectChunkSize();
- slt = yacyCore.seedDB.cacheNodeStatus();
- ost = yacyCore.seedDB.cacheObjectStatus();
- putprop(prop, env, "", "DHT", set);
-
- req = sb.messageDB.size();
- chk = sb.messageDB.cacheNodeChunkSize();
- obj = sb.messageDB.cacheObjectChunkSize();
- slt = sb.messageDB.cacheNodeStatus();
- ost = sb.messageDB.cacheObjectStatus();
- putprop(prop, env, "", "Message", set);
-
- req = sb.wikiDB.sizeOfTwo();
- chk = sb.wikiDB.cacheNodeChunkSize();
- obj = sb.wikiDB.cacheObjectChunkSize();
- slt = sb.wikiDB.cacheNodeStatus();
- ost = sb.wikiDB.cacheObjectStatus();
- putprop(prop, env, "", "Wiki", set);
-
- req = sb.blogDB.size();
- chk = sb.blogDB.cacheNodeChunkSize();
- obj = sb.blogDB.cacheObjectChunkSize();
- slt = sb.blogDB.cacheNodeStatus();
- ost = sb.blogDB.cacheObjectStatus();
- putprop(prop, env, "", "Blog", set);
-
- req = yacyCore.newsPool.dbSize();
- chk = yacyCore.newsPool.cacheNodeChunkSize();
- obj = yacyCore.newsPool.cacheObjectChunkSize();
- slt = yacyCore.newsPool.cacheNodeStatus();
- ost = yacyCore.newsPool.cacheObjectStatus();
- putprop(prop, env, "", "News", set);
-
- req = plasmaSwitchboard.robots.size();
- chk = plasmaSwitchboard.robots.cacheNodeChunkSize();
- obj = plasmaSwitchboard.robots.cacheObjectChunkSize();
- slt = plasmaSwitchboard.robots.cacheNodeStatus();
- ost = plasmaSwitchboard.robots.cacheObjectStatus();
- putprop(prop, env, "", "Robots", set);
-
- req = sb.profiles.size();
- chk = sb.profiles.cacheNodeChunkSize();
- obj = sb.profiles.cacheObjectChunkSize();
- slt = sb.profiles.cacheNodeStatus();
- ost = sb.profiles.cacheObjectStatus();
- putprop(prop, env, "", "Profiles", set);
-
- prop.put("usedTotal", usedTotal / MB);
- prop.put("currTotal", currTotal / MB);
- prop.put("dfltTotal", dfltTotal / MB);
- prop.put("bestTotal", bestTotal / MB);
-
+ prop.put("ObjectList", c);
+ prop.put("objectCacheStopGrow", kelondroCache.getMemStopGrow() / (1024 * 1024));
+ prop.put("objectCacheStartShrink", kelondroCache.getMemStartShrink() / (1024 * 1024));
+ prop.put("objectHitCacheTotalMem", totalhitmem / (1024 * 1024));
+ prop.put("objectMissCacheTotalMem", totalmissmem / (1024 * 1024));
+ /*
+ #{ObjectList}#
+ <tr class="TableCellLight">
+ <td align="left">#[objectHitCachePath]#</td>
+ <td align="right">#[objectHitChunkSize]#</td>
+ <td align="right">#[objectHitCacheCount]#</td>
+ <td align="right">#[objectHitCacheMem]# MB</td>
+ <td align="right">#[objectHitCacheReadHit]#</td>
+ <td align="right">#[objectHitCacheReadMiss]#</td>
+ <td align="right">#[objectHitCacheWriteUnique]#</td>
+ <td align="right">#[objectHitCacheWriteDouble]#</td>
+ <td align="right">#[objectHitCacheDeletes]#</td>
+ <td align="right">#[objectHitCacheFlushes]#</td>
+ <td align="right">#[objectMissChunkSize]#</td>
+ <td align="right">#[objectMissCacheCount]#</td>
+ <td align="right">#[objectMissCacheMem]# MB</td>
+ <td align="right">#[objectMissCacheReadHit]#</td>
+ <td align="right">#[objectMissCacheReadMiss]#</td>
+ <td align="right">#[objectMissCacheWriteUnique]#</td>
+ <td align="right">#[objectMissCacheWriteDouble]#</td>
+ <td align="right">#[objectMissCacheDeletes]#</td>
+ <td align="right">#[objectMissCacheFlushes]#</td>
+ </tr>
+ #{/ObjectList}#
+ <tr class="TableCellDark">
+ <td colspan="10">Total Mem: #[objectHitCacheTotalMem]# MB (hit), #[objectMissCacheTotalMem]# MB (miss); Stop Grow when #[objectCacheStopGrow]# MB available left; Start Shrink when #[objectCacheStartShrink]# MB availabe left</td>
+ </tr>
+ </table></p>
+ */
// parse initialization memory settings
String Xmx = env.getConfig("javastart_Xmx", "Xmx64m").substring(3);
prop.put("Xmx", Xmx.substring(0, Xmx.length() - 1));
String Xms = env.getConfig("javastart_Xms", "Xms10m").substring(3);
prop.put("Xms", Xms.substring(0, Xms.length() - 1));
-
- /*
- // create statistics about write cache object space
- int chunksizes = ((kelondroObjectSpace.statAlive().size() > 0) &&
- (kelondroObjectSpace.statHeap().size() > 0)) ?
- Math.max(
- ((Integer) kelondroObjectSpace.statAlive().lastKey()).intValue(),
- ((Integer) kelondroObjectSpace.statHeap().lastKey()).intValue()
- ) : 0;
- int[] statAlive = new int[chunksizes];
- int[] statHeap = new int[chunksizes];
- for (int i = 0; i < chunksizes; i++) { statAlive[i] = 0; statHeap[i] = 0; }
- Map.Entry entry;
- Iterator i = kelondroObjectSpace.statAlive().entrySet().iterator();
- while (i.hasNext()) {
- entry = (Map.Entry) i.next();
- statAlive[((Integer) entry.getKey()).intValue() - 1] = ((Integer) entry.getValue()).intValue();
- }
- i = kelondroObjectSpace.statHeap().entrySet().iterator();
- while (i.hasNext()) {
- entry = (Map.Entry) i.next();
- statHeap[((Integer) entry.getKey()).intValue() - 1] = ((Integer) entry.getValue()).intValue();
- }
- int c = 0;
- for (int j = 0; j < chunksizes; j++) {
- if ((statAlive[j] > 0) || (statHeap[j] > 0)) {
- prop.put("sizes_" + c + "_chunk", Integer.toString(j + 1));
- prop.put("alive_" + c + "_count", Integer.toString(statAlive[j]));
- prop.put("heap_" + c + "_count", Integer.toString(statHeap[j]));
- c++;
- }
- }
- prop.put("sizes", Integer.toString(c));
- prop.put("alive", Integer.toString(c));
- prop.put("heap" , Integer.toString(c));
- */
// other caching structures
long amount = httpc.nameCacheHitSize();
@@ -324,37 +271,4 @@ public class PerformanceMemory_p { // return rewrite values for templates
return prop;
}
-
- private static void putprop(serverObjects prop, serverSwitch env, String wdb, String db, String set) {
- if ((slt == null) || (ost == null)) return;
- usd = chk * slt[1] + obj * ost[2] /*hit*/ + 12 * ost[3] /*miss*/;
- bst = (((((long) chk) * ((long) req)) >> 10) + 1) << 10;
- if (set.equals("setBest")) env.setConfig("ramCache" + db, bst);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "nodsz" + db, chk);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "ochunksiz" + db, obj);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "slreq" + db, req);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "slemp" + db, slt[0] - slt[1]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "slfil" + db, slt[1]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "slhittmiss" + db, slt[4] + ":" + slt[5]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "sluniqdoub" + db, slt[6] + ":" + slt[7]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "slflush" + db, slt[8] + ":" + slt[9]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "ochunkmax" + db, ost[0]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "omisscmax" + db, ost[1]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "ohitsize" + db, ost[2]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "omisssize" + db, ost[3]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "ohittmiss" + db, ost[7] + ":" + ost[8]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "ouniqdoub" + db, ost[9] + ":" + ost[10]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "oflush" + db, ost[11] + ":" + ost[12]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "nhittmiss" + db, ost[13] + ":" + ost[14]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "nuniqdoub" + db, ost[15] + ":" + ost[16]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "nflush" + db, ost[17] + ":" + ost[18]);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "used" + db, usd / KB);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "best" + db, bst / KB);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "dflt" + db, Long.parseLong((String) defaultSettings.get("ramCache" + db)) / KB);
- prop.put(wdb + ((wdb.length() > 0) ? ("_") : ("")) + "ramCache" + db, Long.parseLong(env.getConfig("ramCache" + db, "0")) / KB);
- usedTotal += usd;
- currTotal += Long.parseLong(env.getConfig("ramCache" + db, "0"));
- dfltTotal += Long.parseLong((String) defaultSettings.get("ramCache" + db));
- bestTotal += bst;
- }
}
diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index 0107bd20b..2f3b095ca 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -173,7 +173,7 @@ public class PerformanceQueues_p { prop.put("table_" + c + "_busysleep", busysleep);
prop.put("table_" + c + "_memprereq", memprereq / 1024);
// disallow setting of memprereq for indexer to prevent db from throwing OOMs
- prop.put("table_" + c + "_disabled", (threadName.endsWith("_indexing")) ? 1 : 0);
+ prop.put("table_" + c + "_disabled", /*(threadName.endsWith("_indexing")) ? 1 :*/ 0);
c++;
}
prop.put("table", c);
diff --git a/htroot/Status.html b/htroot/Status.html index 138fc5616..89abdffc0 100644 --- a/htroot/Status.html +++ b/htroot/Status.html @@ -12,7 +12,8 @@ <p>Welcome to YaCy!</p>
<form action="Steering.html" method="get" class="PeerControl">
- <fieldset>#(restartEnabled)#::
+ <fieldset>
+ #(restartEnabled)#::
<button type="submit" name="restart" value="Restart">
<img src="/env/grafics/lock.gif" alt="Only the administrator may restart this peer" />
Restart
diff --git a/htroot/env/base.css b/htroot/env/base.css index 66007f85f..74374083e 100644 --- a/htroot/env/base.css +++ b/htroot/env/base.css @@ -115,6 +115,11 @@ tt, *.tt { font-weight:bold; } +.commit { + font-weight:bold; + color:green; +} + .help { font-size:90%; } diff --git a/source/dbtest.java b/source/dbtest.java index 1bfc24d94..4bb071c5b 100644 --- a/source/dbtest.java +++ b/source/dbtest.java @@ -35,8 +35,6 @@ public class dbtest { public final static int keylength = 12;
public final static int valuelength = 223; // sum of all data length as defined in plasmaURL
- //public final static long buffer = 0;
- public final static long buffer = 8192 * 1024; // 8 MB buffer
public final static long preload = 1000; // 1 second
public static byte[] dummyvalue2 = new byte[valuelength];
static {
@@ -178,22 +176,21 @@ public class dbtest { kelondroRow testRow = new kelondroRow("byte[] key-" + keylength + ", byte[] dummy-" + keylength + ", value-" + valuelength, kelondroBase64Order.enhancedCoder, 0);
if (dbe.equals("kelondroTree")) {
File tablefile = new File(tablename + ".kelondro.db");
- table = new kelondroCache(new kelondroTree(tablefile, buffer / 2, preload, testRow), buffer / 2, true, false);
+ table = new kelondroCache(new kelondroTree(tablefile, true, preload, testRow), true, false);
}
if (dbe.equals("kelondroSplittedTree")) {
File tablepath = new File(tablename).getParentFile();
tablename = new File(tablename).getName();
table = new kelondroSplittedTree(tablepath, tablename, kelondroBase64Order.enhancedCoder,
- buffer, preload,
- 8, testRow, 1, 80);
+ preload, 8, testRow, 1, 80);
}
if (dbe.equals("kelondroFlexTable")) {
File tablepath = new File(tablename).getParentFile();
- table = new kelondroFlexTable(tablepath, new File(tablename).getName(), buffer, preload, testRow);
+ table = new kelondroFlexTable(tablepath, new File(tablename).getName(), preload, testRow);
}
if (dbe.equals("kelondroFlexSplitTable")) {
File tablepath = new File(tablename).getParentFile();
- table = new kelondroFlexSplitTable(tablepath, new File(tablename).getName(), buffer, preload, testRow);
+ table = new kelondroFlexSplitTable(tablepath, new File(tablename).getName(), preload, testRow);
}
if (dbe.equals("mysql")) {
table = new dbTable("mysql", testRow);
@@ -226,7 +223,7 @@ public class dbtest { key = randomHash(random);
table.put(table.row().newEntry(new byte[][]{key, key, dummyvalue2}));
if (i % 1000 == 0) {
- System.out.println(i + " entries. " + ((table instanceof kelondroTree) ? ((kelondroTree) table).cacheNodeStatusString() : ""));
+ System.out.println(i + " entries. ");
if (table instanceof kelondroTree) {
profiles = ((kelondroTree) table).profiles();
System.out.println("Cache Delta: " + kelondroProfile.delta(profiles[0], cacheProfileAcc).toString());
@@ -457,7 +454,11 @@ final class dbTable implements kelondroIndex { throw new Exception ("Unable to establish a database connection: " + e.getMessage(),e);
}
- }
+ }
+
+ public String filename() {
+ return "dbtest." + theDBConnection.hashCode();
+ }
public void close() throws IOException {
try {
diff --git a/source/de/anomic/data/blogBoard.java b/source/de/anomic/data/blogBoard.java index 97db40be4..f933ebf96 100644 --- a/source/de/anomic/data/blogBoard.java +++ b/source/de/anomic/data/blogBoard.java @@ -81,10 +81,10 @@ public class blogBoard { private kelondroMapObjects datbase = null;
- public blogBoard(File actpath, int bufferkb, long preloadTime) {
+ public blogBoard(File actpath, long preloadTime) {
new File(actpath.getParent()).mkdir();
if (datbase == null) {
- datbase = new kelondroMapObjects(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_', true, false), 500);
+ datbase = new kelondroMapObjects(kelondroDyn.open(actpath, true, true, preloadTime, keyLength, recordSize, '_', true, false), 500);
}
}
@@ -92,22 +92,6 @@ public class blogBoard { return datbase.size();
}
- public int cacheNodeChunkSize() {
- return datbase.cacheNodeChunkSize();
- }
-
- public int cacheObjectChunkSize() {
- return datbase.cacheObjectChunkSize();
- }
-
- public int[] cacheNodeStatus() {
- return datbase.cacheNodeStatus();
- }
-
- public long[] cacheObjectStatus() {
- return datbase.cacheObjectStatus();
- }
-
public void close() {
try {datbase.close();} catch (IOException e) {}
}
diff --git a/source/de/anomic/data/blogBoardComments.java b/source/de/anomic/data/blogBoardComments.java index 9ba230de7..510e57177 100644 --- a/source/de/anomic/data/blogBoardComments.java +++ b/source/de/anomic/data/blogBoardComments.java @@ -81,10 +81,10 @@ public class blogBoardComments { private kelondroMapObjects datbase = null;
- public blogBoardComments(File actpath, int bufferkb, long preloadTime) {
+ public blogBoardComments(File actpath, long preloadTime) {
new File(actpath.getParent()).mkdir();
if (datbase == null) {
- datbase = new kelondroMapObjects(kelondroDyn.open(actpath, bufferkb / 2 * 0x40, preloadTime, keyLength, recordSize, '_', false, false), 500);
+ datbase = new kelondroMapObjects(kelondroDyn.open(actpath, true, true, preloadTime, keyLength, recordSize, '_', false, false), 500);
}
}
@@ -92,22 +92,6 @@ public class blogBoardComments { return datbase.size();
}
- public int cacheNodeChunkSize() {
- return datbase.cacheNodeChunkSize();
- }
-
- public int cacheObjectChunkSize() {
- return datbase.cacheObjectChunkSize();
- }
-
- public int[] cacheNodeStatus() {
- return datbase.cacheNodeStatus();
- }
-
- public long[] cacheObjectStatus() {
- return datbase.cacheObjectStatus();
- }
-
public void close() {
try {datbase.close();} catch (IOException e) {}
}
diff --git a/source/de/anomic/data/bookmarksDB.java b/source/de/anomic/data/bookmarksDB.java index d092878f7..3cef6f026 100644 --- a/source/de/anomic/data/bookmarksDB.java +++ b/source/de/anomic/data/bookmarksDB.java @@ -129,23 +129,23 @@ public class bookmarksDB { return new Date(); } - public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile, int bufferkb, long preloadTime) { + public bookmarksDB(File bookmarksFile, File tagsFile, File datesFile, long preloadTime) { // bookmarks tagCache=new HashMap(); bookmarkCache=new HashMap(); bookmarksFile.getParentFile().mkdirs(); //this.bookmarksTable = new kelondroMap(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false)); - this.bookmarksTable = new kelondroObjects(kelondroDyn.open(bookmarksFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false), 1000); + this.bookmarksTable = new kelondroObjects(kelondroDyn.open(bookmarksFile, true, true, preloadTime, 12, 256, '_', true, false), 1000); // tags tagsFile.getParentFile().mkdirs(); boolean tagsFileExisted = tagsFile.exists(); - this.tagsTable = new kelondroMapObjects(kelondroDyn.open(tagsFile, bufferkb * 1024, preloadTime, 12, 256, '_', true, false), 500); + this.tagsTable = new kelondroMapObjects(kelondroDyn.open(tagsFile, true, true, preloadTime, 12, 256, '_', true, false), 500); if (!tagsFileExisted) rebuildTags(); // dates boolean datesExisted = datesFile.exists(); - this.datesTable = new kelondroMapObjects(kelondroDyn.open(datesFile, bufferkb * 1024, preloadTime, 20, 256, '_', true, false), 500); + this.datesTable = new kelondroMapObjects(kelondroDyn.open(datesFile, true, true, preloadTime, 20, 256, '_', true, false), 500); if (!datesExisted) rebuildDates(); } diff --git a/source/de/anomic/data/messageBoard.java b/source/de/anomic/data/messageBoard.java index 0c06b704e..103f3eb5f 100644 --- a/source/de/anomic/data/messageBoard.java +++ b/source/de/anomic/data/messageBoard.java @@ -67,10 +67,10 @@ public class messageBoard { private kelondroMapObjects database = null;
private int sn = 0;
- public messageBoard(File path, int bufferkb, long preloadTime) {
+ public messageBoard(File path, long preloadTime) {
new File(path.getParent()).mkdir();
if (database == null) {
- database = new kelondroMapObjects(kelondroDyn.open(path, bufferkb * 0x400, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true, false), 500);
+ database = new kelondroMapObjects(kelondroDyn.open(path, true, true, preloadTime, categoryLength + dateFormat.length() + 2, recordSize, '_', true, false), 500);
}
sn = 0;
}
@@ -79,22 +79,6 @@ public class messageBoard { return database.size();
}
- public int cacheNodeChunkSize() {
- return database.cacheNodeChunkSize();
- }
-
- public int cacheObjectChunkSize() {
- return database.cacheObjectChunkSize();
- }
-
- public int[] cacheNodeStatus() {
- return database.cacheNodeStatus();
- }
-
- public long[] cacheObjectStatus() {
- return database.cacheObjectStatus();
- }
-
public void close() {
try {database.close();} catch (IOException e) {}
}
diff --git a/source/de/anomic/data/userDB.java b/source/de/anomic/data/userDB.java index 83b6410a4..c25f1c50a 100644 --- a/source/de/anomic/data/userDB.java +++ b/source/de/anomic/data/userDB.java @@ -68,27 +68,17 @@ public final class userDB { kelondroMapObjects userTable;
private final File userTableFile;
- private final int bufferkb;
private long preloadTime;
private HashMap ipUsers = new HashMap();
private HashMap cookieUsers = new HashMap();
- public userDB(File userTableFile, int bufferkb, long preloadTime) {
+ public userDB(File userTableFile, long preloadTime) {
this.userTableFile = userTableFile;
- this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
userTableFile.getParentFile().mkdirs();
- this.userTable = new kelondroMapObjects(kelondroDyn.open(userTableFile, bufferkb * 1024, preloadTime, 128, 256, '_', true, false), 10);
+ this.userTable = new kelondroMapObjects(kelondroDyn.open(userTableFile, true, true, preloadTime, 128, 256, '_', true, false), 10);
}
- public int dbCacheNodeChunkSize() {
- return userTable.cacheNodeChunkSize();
- }
-
- public int[] dbCacheNodeStatus() {
- return userTable.cacheNodeStatus();
- }
-
void resetDatabase() {
// deletes the database and creates a new one
if (userTable != null) try {
@@ -96,7 +86,7 @@ public final class userDB { } catch (IOException e) {}
if (!(userTableFile.delete())) throw new RuntimeException("cannot delete user database");
userTableFile.getParentFile().mkdirs();
- userTable = new kelondroMapObjects(kelondroDyn.open(userTableFile, this.bufferkb, preloadTime, 256, 512, '_', true, false), 10);
+ userTable = new kelondroMapObjects(kelondroDyn.open(userTableFile, true, true, preloadTime, 256, 512, '_', true, false), 10);
}
public void close() {
diff --git a/source/de/anomic/data/wikiBoard.java b/source/de/anomic/data/wikiBoard.java index cf5fc7040..620d98e26 100644 --- a/source/de/anomic/data/wikiBoard.java +++ b/source/de/anomic/data/wikiBoard.java @@ -54,7 +54,6 @@ import java.util.TimeZone; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroMapObjects;
-import de.anomic.kelondro.kelondroRecords;
public class wikiBoard {
@@ -69,14 +68,14 @@ public class wikiBoard { private kelondroMapObjects bkpbase = null;
private static HashMap authors = new HashMap();
- public wikiBoard(File actpath, File bkppath, int bufferkb, long preloadTime) {
+ public wikiBoard(File actpath, File bkppath, long preloadTime) {
new File(actpath.getParent()).mkdirs();
if (datbase == null) {
- datbase = new kelondroMapObjects(kelondroDyn.open(actpath, bufferkb / 2 * 0x400, preloadTime, keyLength, recordSize, '_', true, false), 500);
+ datbase = new kelondroMapObjects(kelondroDyn.open(actpath, true, true, preloadTime, keyLength, recordSize, '_', true, false), 500);
}
new File(bkppath.getParent()).mkdirs();
if (bkpbase == null) {
- bkpbase = new kelondroMapObjects(kelondroDyn.open(bkppath, bufferkb / 2 * 0x400, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true, false), 500);
+ bkpbase = new kelondroMapObjects(kelondroDyn.open(bkppath, true, true, preloadTime, keyLength + dateFormat.length(), recordSize, '_', true, false), 500);
}
}
@@ -88,28 +87,6 @@ public class wikiBoard { return datbase.size();
}
- public int cacheNodeChunkSize() {
- int db = datbase.cacheNodeChunkSize();
- int bk = bkpbase.cacheNodeChunkSize();
- return (db + bk) / 2;
- }
-
- public int cacheObjectChunkSize() {
- int db = datbase.cacheObjectChunkSize();
- int bk = bkpbase.cacheObjectChunkSize();
- return (db + bk) / 2;
- }
-
- public int[] cacheNodeStatus() {
- int[] a = datbase.cacheNodeStatus();
- int[] b = bkpbase.cacheNodeStatus();
- return kelondroRecords.cacheCombinedStatus(new int[][]{a, b}, 2);
- }
-
- public long[] cacheObjectStatus() {
- return datbase.cacheObjectStatus();
- }
-
public void close() {
try {datbase.close();} catch (IOException e) {}
try {bkpbase.close();} catch (IOException e) {}
diff --git a/source/de/anomic/index/indexCollectionRI.java b/source/de/anomic/index/indexCollectionRI.java index a7b118cea..6999ae7c9 100644 --- a/source/de/anomic/index/indexCollectionRI.java +++ b/source/de/anomic/index/indexCollectionRI.java @@ -44,14 +44,13 @@ public class indexCollectionRI implements indexRI { kelondroCollectionIndex collectionIndex; - public indexCollectionRI(File path, String filenameStub, long buffersize, long preloadTime, int maxpartition, kelondroRow payloadrow) { + public indexCollectionRI(File path, String filenameStub, long preloadTime, int maxpartition, kelondroRow payloadrow) { try { collectionIndex = new kelondroCollectionIndex( path, filenameStub, 12 /*keyLength*/, kelondroBase64Order.enhancedCoder, - buffersize, preloadTime, 4 /*loadfactor*/, maxpartition, diff --git a/source/de/anomic/index/indexRAMRI.java b/source/de/anomic/index/indexRAMRI.java index ea4a8f542..66f421b1f 100644 --- a/source/de/anomic/index/indexRAMRI.java +++ b/source/de/anomic/index/indexRAMRI.java @@ -111,7 +111,7 @@ public final class indexRAMRI implements indexRI { kelondroBufferedRA writeBuffer = null;
if (false /*serverMemory.available() > 50 * bufferStructureBasis.objectsize() * cache.size()*/) {
writeBuffer = new kelondroBufferedRA();
- dumpArray = new kelondroFixedWidthArray(writeBuffer, bufferStructureBasis, 0);
+ dumpArray = new kelondroFixedWidthArray(writeBuffer, indexDumpFile.getCanonicalPath(), bufferStructureBasis, 0);
log.logInfo("started dump of ram cache: " + cache.size() + " words; memory-enhanced write");
} else {
dumpArray = new kelondroFixedWidthArray(indexDumpFile, bufferStructureBasis, 0);
@@ -188,7 +188,7 @@ public final class indexRAMRI implements indexRI { kelondroBufferedRA readBuffer = null;
if (false /*serverMemory.available() > indexDumpFile.length() * 2*/) {
readBuffer = new kelondroBufferedRA(new serverByteBuffer(serverFileUtils.read(indexDumpFile)));
- dumpArray = new kelondroFixedWidthArray(readBuffer, bufferStructureBasis, 0);
+ dumpArray = new kelondroFixedWidthArray(readBuffer, indexDumpFile.getCanonicalPath(), bufferStructureBasis, 0);
log.logInfo("started restore of ram cache '" + indexArrayFileName + "', " + dumpArray.size() + " word/URL relations; memory-enhanced read");
} else {
dumpArray = new kelondroFixedWidthArray(indexDumpFile, bufferStructureBasis, 0);
@@ -228,7 +228,8 @@ public final class indexRAMRI implements indexRI { }
if (readBuffer != null) readBuffer.close();
dumpArray.close();
- log.logConfig("finished restor " + cache.size() + " words in " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds");
+ dumpArray = null;
+ log.logConfig("finished restore: " + cache.size() + " words in " + ((System.currentTimeMillis() - startTime) / 1000) + " seconds");
} catch (kelondroException e) {
// restore failed
log.logSevere("failed restore of indexCache array dump: " + e.getMessage(), e);
diff --git a/source/de/anomic/kelondro/kelondroBytesIntMap.java b/source/de/anomic/kelondro/kelondroBytesIntMap.java index 40afc43f9..b0b4cc369 100644 --- a/source/de/anomic/kelondro/kelondroBytesIntMap.java +++ b/source/de/anomic/kelondro/kelondroBytesIntMap.java @@ -37,6 +37,10 @@ public class kelondroBytesIntMap { this.ki = ki; } + public kelondroRow row() throws IOException { + return ki.row(); + } + public synchronized int geti(byte[] key) throws IOException { assert (key != null); //assert (!(serverLog.allZero(key))); diff --git a/source/de/anomic/kelondro/kelondroCache.java b/source/de/anomic/kelondro/kelondroCache.java index 1d36b7b25..016e06ce5 100644 --- a/source/de/anomic/kelondro/kelondroCache.java +++ b/source/de/anomic/kelondro/kelondroCache.java @@ -29,11 +29,13 @@ package de.anomic.kelondro; import java.io.IOException; import java.util.Date; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import de.anomic.kelondro.kelondroRow.Entry; -import de.anomic.server.serverMemory; public class kelondroCache implements kelondroIndex { @@ -45,17 +47,22 @@ public class kelondroCache implements kelondroIndex { // - a write buffer for rows that are known to be contained in target // furthermore, if we access a kelondroFlexTable, we can use the ram index of the underlying index + // static object tracker; stores information about object cache usage + private static final TreeMap objectTracker = new TreeMap(); + private static long memStopGrow = 4000000; // a limit for the node cache to stop growing if less than this memory amount is available + private static long memStartShrink = 2000000; // a limit for the node cache to start with shrinking if less than this memory amount is available + + // class objects private kelondroRowSet readHitCache; private kelondroRowSet readMissCache; private kelondroRowSet writeBufferUnique; // entries of that buffer are not contained in index private kelondroRowSet writeBufferDoubles; // entries of that buffer shall overwrite entries in index private kelondroIndex index; private kelondroRow keyrow; - private int maxrecords, maxmiss; private int readHit, readMiss, writeUnique, writeDouble, cacheDelete, cacheFlush; private int hasnotHit, hasnotMiss, hasnotUnique, hasnotDouble, hasnotDelete, hasnotFlush; - public kelondroCache(kelondroIndex backupIndex, long buffersize, boolean read, boolean write) throws IOException { + public kelondroCache(kelondroIndex backupIndex, boolean read, boolean write) throws IOException { assert write == false; this.index = backupIndex; this.keyrow = new kelondroRow(new kelondroColumn[]{index.row().column(index.row().primaryKey)}, index.row().objectOrder, index.row().primaryKey); @@ -63,8 +70,6 @@ public class kelondroCache implements kelondroIndex { this.readMissCache = (read) ? new kelondroRowSet(this.keyrow, 0) : null; this.writeBufferUnique = (write) ? new kelondroRowSet(index.row(), 0) : null; this.writeBufferDoubles = (write) ? new kelondroRowSet(index.row(), 0) : null; - this.maxmiss = (read) ? (int) (buffersize / 10 / index.row().column(index.row().primaryKey).cellwidth()) : 0; - this.maxrecords = (int) ((buffersize - maxmiss * index.row().column(index.row().primaryKey).cellwidth()) / index.row().objectsize()); this.readHit = 0; this.readMiss = 0; this.writeUnique = 0; @@ -77,17 +82,7 @@ public class kelondroCache implements kelondroIndex { this.hasnotDouble = 0; this.hasnotDelete = 0; this.hasnotFlush = 0; - } - - - public final int cacheNodeChunkSize() { - // returns the size that the node cache uses for a single entry - return index.cacheNodeChunkSize(); - } - - public final int[] cacheNodeStatus() { - // a collection of different node cache status values - return index.cacheNodeStatus(); + objectTracker.put(backupIndex.filename(), this); } public final int cacheObjectChunkSize() { @@ -98,61 +93,6 @@ public class kelondroCache implements kelondroIndex { } } - public long[] cacheObjectStatus() { - return new long[]{ - (long) maxrecords, - (long) maxmiss, - (long) ((readHitCache == null) ? 0 : readHitCache.size()), - (long) ((readMissCache == null) ? 0 : readMissCache.size()), - 0, // this.maxAge - 0, // minAge() - 0, // maxAge() - (long) readHit, - (long) readMiss, - (long) writeUnique, - (long) writeDouble, - (long) cacheDelete, - (long) cacheFlush, - (long) hasnotHit, - (long) hasnotMiss, - (long) hasnotUnique, - (long) hasnotDouble, - (long) hasnotDelete, - (long) hasnotFlush - }; - } - - private static long[] combinedStatus(long[] a, long[] b) { - return new long[]{ - a[0] + b[0], - a[1] + b[1], - a[2] + b[2], - a[3] + b[3], - Math.max(a[4], b[4]), - Math.min(a[5], b[5]), - Math.max(a[6], b[6]), - a[7] + b[7], - a[8] + b[8], - a[9] + b[9], - a[10] + b[10], - a[11] + b[11], - a[12] + b[12], - a[13] + b[13], - a[14] + b[14], - a[15] + b[15], - a[16] + b[16], - a[17] + b[17], - a[18] + b[18] - }; - } - - public static long[] combinedStatus(long[][] a, int l) { - if ((a == null) || (a.length == 0) || (l == 0)) return null; - if ((a.length >= 1) && (l == 1)) return a[0]; - if ((a.length >= 2) && (l == 2)) return combinedStatus(a[0], a[1]); - return combinedStatus(combinedStatus(a, l - 1), a[l - 1]); - } - public int writeBufferSize() { return ((writeBufferUnique == null) ? 0 : writeBufferUnique.size()) + @@ -163,6 +103,63 @@ public class kelondroCache implements kelondroIndex { return index.profile(); // TODO: implement own profile and merge with global } + public static void setCacheGrowStati(long memStopGrowNew, long memStartShrinkNew) { + memStopGrow = memStopGrowNew; + memStartShrink = memStartShrinkNew; + } + + public static long getMemStopGrow() { + return memStopGrow ; + } + + public static long getMemStartShrink() { + return memStartShrink ; + } + + public static final Iterator filenames() { + // iterates string objects; all file names from record tracker + return objectTracker.keySet().iterator(); + } + + public static final Map memoryStats(String filename) { + // returns a map for each file in the tracker; + // the map represents properties for each record oobjects, + // i.e. for cache memory allocation + kelondroCache theObjectsCache = (kelondroCache) objectTracker.get(filename); + return theObjectsCache.memoryStats(); + } + + private final Map memoryStats() { + // returns statistical data about this object + HashMap map = new HashMap(); + map.put("objectHitChunkSize", (readHitCache == null) ? "0" : Integer.toString(readHitCache.rowdef.objectsize)); + map.put("objectHitCacheCount", (readHitCache == null) ? "0" : Integer.toString(readHitCache.size())); + map.put("objectHitMem", (readHitCache == null) ? "0" : Integer.toString((int) (readHitCache.rowdef.objectsize * readHitCache.size() * kelondroRowCollection.growfactor))); + map.put("objectHitCacheReadHit", Integer.toString(readHit)); + map.put("objectHitCacheReadMiss", Integer.toString(readMiss)); + map.put("objectHitCacheWriteUnique", Integer.toString(writeUnique)); + map.put("objectHitCacheWriteDouble", Integer.toString(writeDouble)); + map.put("objectHitCacheDeletes", Integer.toString(cacheDelete)); + map.put("objectHitCacheFlushes", Integer.toString(cacheFlush)); + + map.put("objectMissChunkSize", (readMissCache == null) ? "0" : Integer.toString(readMissCache.rowdef.objectsize)); + map.put("objectMissCacheCount", (readMissCache == null) ? "0" : Integer.toString(readMissCache.size())); + map.put("objectMissMem", (readMissCache == null) ? "0" : Integer.toString((int) (readMissCache.rowdef.objectsize * readMissCache.size() * kelondroRowCollection.growfactor))); + map.put("objectMissCacheReadHit", Integer.toString(hasnotHit)); + map.put("objectMissCacheReadMiss", Integer.toString(hasnotMiss)); + map.put("objectMissCacheWriteUnique", Integer.toString(hasnotUnique)); + map.put("objectMissCacheWriteDouble", Integer.toString(hasnotDouble)); + map.put("objectMissCacheDeletes", Integer.toString(hasnotDelete)); + map.put("objectMissCacheFlushes", Integer.toString(hasnotFlush)); + + // future feature .. map.put("objectElderTimeRead", index.profile().) + return map; + } + + private int cacheGrowStatus() { + return kelondroRecords.cacheGrowStatus(memStopGrow, memStartShrink); + } + private void flushUnique() throws IOException { if (writeBufferUnique == null) return; synchronized (writeBufferUnique) { @@ -241,25 +238,20 @@ public class kelondroCache implements kelondroIndex { ((writeBufferDoubles == null) ? 0 : writeBufferDoubles.size()); } - private boolean shortMemory() { - return (serverMemory.available() < 2000000); - } - private void checkMissSpace() { - if ((readMissCache != null) && - ((readMissCache.size() >= maxmiss) || (shortMemory())) + if ((readMissCache != null) && (cacheGrowStatus() < 1) ) {readMissCache.clear(); readMissCache.trim();} } private void checkHitSpace() throws IOException { - int s; - if ((s = sumRecords()) >= maxrecords) flushDoubles(s / 4); - if ((s = sumRecords()) >= maxrecords) flushUnique(s / 4); - if (((s = sumRecords()) >= maxrecords) && (readHitCache != null)) { + int s = sumRecords(); + if (cacheGrowStatus() < 2) {flushDoubles(s / 4); s = sumRecords();} + if (cacheGrowStatus() < 2) {flushUnique(s / 4); s = sumRecords();} + if ((cacheGrowStatus() < 2) && (readHitCache != null)) { readHitCache.clear(); readHitCache.trim(); } - if (shortMemory()) { + if (cacheGrowStatus() < 1) { flushUnique(); flushDoubles(); if (readHitCache != null) { @@ -642,4 +634,8 @@ public class kelondroCache implements kelondroIndex { return index.size() + ((writeBufferUnique == null) ? 0 : writeBufferUnique.size()); } + public String filename() { + return index.filename(); + } + } diff --git a/source/de/anomic/kelondro/kelondroCollectionIndex.java b/source/de/anomic/kelondro/kelondroCollectionIndex.java index 40276922d..fe9f53616 100644 --- a/source/de/anomic/kelondro/kelondroCollectionIndex.java +++ b/source/de/anomic/kelondro/kelondroCollectionIndex.java @@ -109,9 +109,7 @@ public class kelondroCollectionIndex { } public kelondroCollectionIndex(File path, String filenameStub, int keyLength, kelondroOrder indexOrder, - long buffersize, long preloadTime, - int loadfactor, int maxpartitions, - kelondroRow rowdef) throws IOException { + long preloadTime, int loadfactor, int maxpartitions, kelondroRow rowdef) throws IOException { // the buffersize is number of bytes that are only used if the kelondroFlexTable is backed up with a kelondroTree this.path = path; this.filenameStub = filenameStub; @@ -125,7 +123,7 @@ public class kelondroCollectionIndex { boolean ramIndexGeneration = false; boolean fileIndexGeneration = !(new File(path, filenameStub + ".index").exists()); if (ramIndexGeneration) index = new kelondroRowSet(indexRow(keyLength, indexOrder), 0); - if (fileIndexGeneration) index = new kelondroFlexTable(path, filenameStub + ".index", buffersize, preloadTime, indexRow(keyLength, indexOrder)); + if (fileIndexGeneration) index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keyLength, indexOrder)); // open array files this.arrays = new HashMap(); // all entries will be dynamically created with getArray() @@ -135,7 +133,7 @@ public class kelondroCollectionIndex { } // open/create index table - if (index == null) index = openIndexFile(path, filenameStub, indexOrder, buffersize, preloadTime, loadfactor, rowdef); + if (index == null) index = openIndexFile(path, filenameStub, indexOrder, preloadTime, loadfactor, rowdef); } private void openAllArrayFiles(boolean indexGeneration, kelondroOrder indexOrder) throws IOException { @@ -200,10 +198,9 @@ public class kelondroCollectionIndex { } private kelondroIndex openIndexFile(File path, String filenameStub, kelondroOrder indexOrder, - long buffersize, long preloadTime, - int loadfactor, kelondroRow rowdef) throws IOException { + long preloadTime, int loadfactor, kelondroRow rowdef) throws IOException { // open/create index table - kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", buffersize / 2, preloadTime, indexRow(keylength, indexOrder)), buffersize / 2, true, false); + kelondroIndex theindex = new kelondroCache(new kelondroFlexTable(path, filenameStub + ".index", preloadTime, indexRow(keylength, indexOrder)), true, false); // save/check property file for this array File propfile = propertyFile(path, filenameStub, loadfactor, rowdef.objectsize()); @@ -1021,13 +1018,12 @@ public class kelondroCollectionIndex { File path = new File(args[0]); String filenameStub = args[1]; - long buffersize = 10000000; long preloadTime = 10000; try { // initialize collection index kelondroCollectionIndex collectionIndex = new kelondroCollectionIndex( path, filenameStub, 9 /*keyLength*/, - kelondroNaturalOrder.naturalOrder, buffersize, preloadTime, + kelondroNaturalOrder.naturalOrder, preloadTime, 4 /*loadfactor*/, 7, rowdef); // fill index with values @@ -1055,7 +1051,7 @@ public class kelondroCollectionIndex { // printout of index collectionIndex.close(); - kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", buffersize, preloadTime, kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder)); + kelondroFlexTable index = new kelondroFlexTable(path, filenameStub + ".index", preloadTime, kelondroCollectionIndex.indexRow(9, kelondroNaturalOrder.naturalOrder)); index.print(); index.close(); } catch (IOException e) { diff --git a/source/de/anomic/kelondro/kelondroDyn.java b/source/de/anomic/kelondro/kelondroDyn.java index 2becf5f2b..b2322de72 100644 --- a/source/de/anomic/kelondro/kelondroDyn.java +++ b/source/de/anomic/kelondro/kelondroDyn.java @@ -72,20 +72,20 @@ public class kelondroDyn { private kelondroObjectBuffer buffer;
private kelondroRow rowdef;
- public kelondroDyn(File file, long buffersize /*bytes*/, long preloadTime, int key, int nodesize, char fillChar, boolean usetree, boolean writebuffer) throws IOException {
- this(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree, writebuffer);
+ public kelondroDyn(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key, int nodesize, char fillChar, boolean usetree, boolean writebuffer) throws IOException {
+ this(file, useNodeCache, useObjectCache, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree, writebuffer);
}
- public kelondroDyn(File file, long buffersize /* bytes */, long preloadTime, int key,
+ public kelondroDyn(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key,
int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree, boolean writebuffer) throws IOException {
// creates or opens a dynamic tree
rowdef = new kelondroRow("byte[] key-" + (key + counterlen) + ", byte[] node-" + nodesize, objectOrder, 0);
if (usetree) {
- kelondroTree tree = new kelondroTree(file, buffersize / 2, preloadTime, rowdef, 1, 8);
- this.index = new kelondroCache(tree, buffersize / 2, true, writebuffer);
+ kelondroTree tree = new kelondroTree(file, useNodeCache, preloadTime, rowdef, 1, 8);
+ this.index = (useObjectCache) ? (kelondroIndex) new kelondroCache(tree, true, writebuffer) : (kelondroIndex) tree;
} else {
- kelondroFlexTable table = new kelondroFlexTable(file.getParentFile(), file.getName(), buffersize / 2, -1, rowdef);
- this.index = new kelondroCache(table, buffersize / 2, true, writebuffer);
+ kelondroFlexTable table = new kelondroFlexTable(file.getParentFile(), file.getName(), 10000, rowdef);
+ this.index = (useObjectCache) ? (kelondroIndex) new kelondroCache(table, true, writebuffer) : (kelondroIndex) table;
}
this.keylen = index.row().width(0) - counterlen;
this.reclen = index.row().width(1);
@@ -95,20 +95,20 @@ public class kelondroDyn { buffer = new kelondroObjectBuffer(file.toString());
}
- public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key, int nodesize, char fillChar, boolean usetree, boolean writebuffer) {
- return open(file, buffersize, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree, writebuffer);
+ public static final kelondroDyn open(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key, int nodesize, char fillChar, boolean usetree, boolean writebuffer) {
+ return open(file, useNodeCache, useObjectCache, preloadTime, key, nodesize, fillChar, new kelondroNaturalOrder(true), usetree, writebuffer);
}
- public static final kelondroDyn open(File file, long buffersize /* bytes */, long preloadTime, int key,
+ public static final kelondroDyn open(File file, boolean useNodeCache, boolean useObjectCache, long preloadTime, int key,
int nodesize, char fillChar, kelondroOrder objectOrder, boolean usetree, boolean writebuffer) {
// opens new or existing file; in case that any error occur the file is deleted again and it is tried to create the file again
// if that fails, the method returns null
try {
- return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder, usetree, writebuffer);
+ return new kelondroDyn(file, useNodeCache, useObjectCache, preloadTime, key, nodesize, fillChar, objectOrder, usetree, writebuffer);
} catch (IOException e) {
file.delete();
try {
- return new kelondroDyn(file, buffersize, preloadTime, key, nodesize, fillChar, objectOrder, usetree, writebuffer);
+ return new kelondroDyn(file, useNodeCache, useObjectCache, preloadTime, key, nodesize, fillChar, objectOrder, usetree, writebuffer);
} catch (IOException ee) {
serverLog.logSevere("kelondroDyn", "cannot open or create file " + file.toString());
e.printStackTrace();
@@ -132,22 +132,6 @@ public class kelondroDyn { return this.rowdef;
}
- public int cacheNodeChunkSize() {
- return index.cacheNodeChunkSize();
- }
-
- public int cacheObjectChunkSize() {
- return index.cacheObjectChunkSize();
- }
-
- public int[] cacheNodeStatus() {
- return index.cacheNodeStatus();
- }
-
- public long[] cacheObjectStatus() {
- return index.cacheObjectStatus();
- }
-
public synchronized int sizeDyn() throws IOException {
//this.segmentCount = 0;
//Iterator i = keys(true); while (i.hasNext()) segmentCount++;
@@ -489,7 +473,7 @@ public class kelondroDyn { if (args.length == 1) {
// open a db and list keys
try {
- kelondroDyn kd = new kelondroDyn(new File(args[0]), 0x100000, 0, 4 ,100, '_', false, false);
+ kelondroDyn kd = new kelondroDyn(new File(args[0]), true, true, 0, 4 ,100, '_', false, false);
System.out.println(kd.sizeDyn() + " elements in DB");
Iterator i = kd.dynKeys(true, false);
while (i.hasNext())
@@ -506,7 +490,7 @@ public class kelondroDyn { File f = new File(args[3]);
kelondroDyn kd;
try {
- kd = new kelondroDyn(db, 0x100000, 0, 80, 200, '_', false, false);
+ kd = new kelondroDyn(db, true, true, 0, 80, 200, '_', false, false);
if (writeFile)
kd.readFile(key, f);
else
diff --git a/source/de/anomic/kelondro/kelondroDynTree.java b/source/de/anomic/kelondro/kelondroDynTree.java index 8d5c55729..7b414adc0 100644 --- a/source/de/anomic/kelondro/kelondroDynTree.java +++ b/source/de/anomic/kelondro/kelondroDynTree.java @@ -62,24 +62,22 @@ public class kelondroDynTree { private int maxcountBuffer = 1000, maxsizeBuffer = 100;
//private long maxageCache = 60000, cycletimeCache = 10000;
private long maxageBuffer = 60000, cycletimeBuffer = 10000;
- private long buffersize = 0;
private long preloadTime = 0;
// data structures for the cache and buffer
private Hashtable buffer, cache;
private long cycleBuffer;
- public kelondroDynTree(File file, long buffersize, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar) throws IOException {
+ public kelondroDynTree(File file, long preloadTime, int keylength, int nodesize, kelondroRow rowdef, char fillChar) throws IOException {
// creates or opens a DynTree
this.file = file;
- this.buffersize = buffersize;
this.preloadTime = preloadTime;
this.rowdef = rowdef;
this.buffer = new Hashtable();
this.cache = new Hashtable();
//this.cycleCache = Long.MIN_VALUE;
this.cycleBuffer = Long.MIN_VALUE;
- this.table = new kelondroDyn(file, buffersize, preloadTime, keylength, nodesize, fillChar, true, false);
+ this.table = new kelondroDyn(file, true, true, preloadTime, keylength, nodesize, fillChar, true, false);
this.treeRAHandles = new Hashtable();
}
@@ -107,7 +105,7 @@ public class kelondroDynTree { kelondroRA ra = table.getRA(key); // works always, even with no-existing entry
treeRAHandles.put(key, ra);
try {
- return new kelondroTree(ra, buffersize, preloadTime, rowdef, false);
+ return new kelondroTree(ra, this.file.getCanonicalPath() + "#" + key, true, preloadTime, rowdef, false);
} catch (RuntimeException e) {
throw new IOException(e.getMessage());
}
@@ -117,7 +115,7 @@ public class kelondroDynTree { if (table.existsDyn(key)) {
kelondroRA ra = table.getRA(key);
treeRAHandles.put(key, ra);
- return new kelondroTree(ra, buffersize, preloadTime);
+ return new kelondroTree(ra, this.file.getCanonicalPath() + "#" + key, true, preloadTime);
}
return null;
}
@@ -298,10 +296,10 @@ public class kelondroDynTree { System.out.println("start");
File file = new File("D:\\bin\\testDyn.db");
if (file.exists()) {
- kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_');
+ kelondroDynTree dt = new kelondroDynTree(file, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_');
System.out.println("opened: table keylength=" + dt.table.row().width(0) + ", sectorsize=" + dt.table.row().width(1) + ", " + dt.table.sizeDyn() + " entries.");
} else {
- kelondroDynTree dt = new kelondroDynTree(file, 0x100000L, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_');
+ kelondroDynTree dt = new kelondroDynTree(file, 0, 16, 512, new kelondroRow("byte[] a-10, byte[] b-20, byte[] c-30", kelondroNaturalOrder.naturalOrder, 0), '_');
String name;
kelondroTree t;
kelondroRow.Entry line;
diff --git a/source/de/anomic/kelondro/kelondroFixedWidthArray.java b/source/de/anomic/kelondro/kelondroFixedWidthArray.java index 068a94c38..5ee96562b 100644 --- a/source/de/anomic/kelondro/kelondroFixedWidthArray.java +++ b/source/de/anomic/kelondro/kelondroFixedWidthArray.java @@ -63,7 +63,7 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro public kelondroFixedWidthArray(File file, kelondroRow rowdef, int intprops) throws IOException {
// this creates a new array
- super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
+ super(file, false, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
if (!(super.fileExisted)) {
for (int i = 0; i < intprops; i++) {
setHandle(i, new Handle(NUL));
@@ -76,9 +76,9 @@ public class kelondroFixedWidthArray extends kelondroRecords implements kelondro markedRemoved = new TreeSet();
}
- public kelondroFixedWidthArray(kelondroRA ra, kelondroRow rowdef, int intprops) throws IOException {
+ public kelondroFixedWidthArray(kelondroRA ra, String filename, kelondroRow rowdef, int intprops) throws IOException {
// this creates a new array
- super(ra, 0, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, false);
+ super(ra, filename, false, 0, thisOHBytes, thisOHHandles, rowdef, intprops, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, false);
for (int i = 0; i < intprops; i++) {
setHandle(i, new Handle(0));
}
diff --git a/source/de/anomic/kelondro/kelondroFlexSplitTable.java b/source/de/anomic/kelondro/kelondroFlexSplitTable.java index 1da711792..07c3ae947 100644 --- a/source/de/anomic/kelondro/kelondroFlexSplitTable.java +++ b/source/de/anomic/kelondro/kelondroFlexSplitTable.java @@ -44,9 +44,8 @@ public class kelondroFlexSplitTable implements kelondroIndex { private kelondroRow rowdef; private File path; private String tablename; - private long buffersize; - public kelondroFlexSplitTable(File path, String tablename, long buffersize, long preloadTime, kelondroRow rowdef) throws IOException { + public kelondroFlexSplitTable(File path, String tablename, long preloadTime, kelondroRow rowdef) throws IOException { this.path = path; this.tablename = tablename; this.rowdef = rowdef; @@ -96,21 +95,15 @@ public class kelondroFlexSplitTable implements kelondroIndex { // open next biggest table t.remove(maxf); date = maxf.substring(tablename.length() + 1); - if (buffersize >= maxram * 2) { - // this will cause usage of a complete RAM index - table = new kelondroCache(new kelondroFlexTable(path, maxf, maxram * 2, preloadTime, rowdef), maxram / 10, true, false); - buffersize -= maxram * 2; - buffersize -= maxram / 10; - } else { - // this will cause a generation of a file index - table = new kelondroFlexTable(path, maxf, buffersize / (t.size() + 1), preloadTime, rowdef); - buffersize -= buffersize / (t.size() + 1); - } + table = new kelondroCache(new kelondroFlexTable(path, maxf, preloadTime, rowdef), true, false); tables.put(date, table); } - System.out.println("*** remaining buffer RAM (not used): " + buffersize); } - + + public String filename() { + return new File(path, tablename).toString(); + } + private static final Calendar thisCalendar = Calendar.getInstance(); public static final String dateSuffix(Date date) { int month, year; @@ -204,7 +197,7 @@ public class kelondroFlexSplitTable implements kelondroIndex { kelondroIndex table = (kelondroIndex) tables.get(suffix); if (table == null) { // make new table - table = new kelondroFlexTable(path, tablename + "." + suffix, buffersize / (tables.size() + 1), -1, rowdef); + table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef); tables.put(suffix, table); } table.put(row); @@ -235,7 +228,7 @@ public class kelondroFlexSplitTable implements kelondroIndex { kelondroIndex table = (kelondroIndex) tables.get(suffix); if (table == null) { // make new table - table = new kelondroFlexTable(path, tablename + "." + suffix, buffersize / (tables.size() + 1), -1, rowdef); + table = new kelondroFlexTable(path, tablename + "." + suffix, -1, rowdef); tables.put(suffix, table); } table.addUnique(row, entryDate); diff --git a/source/de/anomic/kelondro/kelondroFlexTable.java b/source/de/anomic/kelondro/kelondroFlexTable.java index 49c235897..eece0ab0a 100644 --- a/source/de/anomic/kelondro/kelondroFlexTable.java +++ b/source/de/anomic/kelondro/kelondroFlexTable.java @@ -29,24 +29,30 @@ import java.io.File; import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import de.anomic.server.serverMemory;
import de.anomic.server.logging.serverLog;
public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondroIndex {
+ // static tracker objects
+ private static TreeMap tableTracker = new TreeMap();
+
+ // class objects
protected kelondroBytesIntMap index;
private boolean RAMIndex;
- public kelondroFlexTable(File path, String tablename, long buffersize, long preloadTime, kelondroRow rowdef) throws IOException {
+ public kelondroFlexTable(File path, String tablename, long preloadTime, kelondroRow rowdef) throws IOException {
// the buffersize applies to a possible load of the ram-index
// if the ram is not sufficient, a tree file is generated
// if, and only if a tree file exists, the preload time is applied
super(path, tablename, rowdef);
- long neededRAM = (super.row().column(0).cellwidth() + 4) * super.size();
+ long neededRAM = (long) ((super.row().column(0).cellwidth() + 4) * super.size() * kelondroRowCollection.growfactor);
File newpath = new File(path, tablename);
File indexfile = new File(newpath, "col.000.index");
@@ -57,7 +63,7 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr System.out.println("*** Last Startup time: " + stt + " milliseconds");
long start = System.currentTimeMillis();
- if (buffersize >= neededRAM) {
+ if (serverMemory.available(neededRAM, true)) {
// we can use a RAM index
if (indexfile.exists()) {
@@ -75,18 +81,19 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr + " index entries initialized and sorted from "
+ super.col[0].size() + " keys.");
RAMIndex = true;
+ tableTracker.put(this.filename(), this);
} else {
// too less ram for a ram index
if (indexfile.exists()) {
// use existing index file
System.out.println("*** Using File index " + indexfile);
- ki = new kelondroCache(kelondroTree.open(indexfile, buffersize / 3 * 2, preloadTime, treeIndexRow(rowdef.width(0), rowdef.objectOrder, rowdef.primaryKey), 2, 80), buffersize / 3, true, false);
+ ki = new kelondroCache(kelondroTree.open(indexfile, true, preloadTime, treeIndexRow(rowdef.width(0), rowdef.objectOrder, rowdef.primaryKey), 2, 80), true, false);
RAMIndex = false;
} else {
// generate new index file
System.out.println("*** Generating File index for " + size() + " entries from " + indexfile);
- System.out.println("*** Cause: too less RAM (" + (buffersize / 1024 / 1024) + " MB) configured. Assign at least " + (neededRAM / 1024 / 1024) + " MB buffersize to enable a RAM index.");
- ki = initializeTreeIndex(indexfile, buffersize, preloadTime, rowdef.objectOrder, rowdef.primaryKey);
+ System.out.println("*** Cause: too less RAM (" + serverMemory.available() + " MB) configured. Assign at least " + (neededRAM / 1024 / 1024) + " MB buffersize to enable a RAM index.");
+ ki = initializeTreeIndex(indexfile, preloadTime, rowdef.objectOrder, rowdef.primaryKey);
System.out.println(" -done-");
System.out.println(ki.size() + " entries indexed from " + super.col[0].size() + " keys.");
@@ -146,8 +153,8 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr return ri;
}
- private kelondroIndex initializeTreeIndex(File indexfile, long buffersize, long preloadTime, kelondroOrder objectOrder, int primaryKey) throws IOException {
- kelondroIndex treeindex = new kelondroCache(new kelondroTree(indexfile, buffersize / 3 * 2, preloadTime, treeIndexRow(rowdef.width(0), objectOrder, primaryKey), 2, 80), buffersize / 3, true, false);
+ private kelondroIndex initializeTreeIndex(File indexfile, long preloadTime, kelondroOrder objectOrder, int primaryKey) throws IOException {
+ kelondroIndex treeindex = new kelondroCache(new kelondroTree(indexfile, true, preloadTime, treeIndexRow(rowdef.width(0), objectOrder, primaryKey), 2, 80), true, false);
Iterator content = super.col[0].contentNodes(-1);
kelondroRecords.Node node;
kelondroRow.Entry indexentry;
@@ -313,27 +320,43 @@ public class kelondroFlexTable extends kelondroFlexWidthArray implements kelondr return index.profile();
}
- public final int cacheObjectChunkSize() {
- // dummy method
- return -1;
+ public static final Iterator filenames() {
+ // iterates string objects; all file names from record tracker
+ return tableTracker.keySet().iterator();
}
-
- public long[] cacheObjectStatus() {
- // dummy method
- return null;
+
+ public static final kelondroProfile profileStats(String filename) {
+ // returns a map for each file in the tracker;
+ // the map represents properties for each record oobjects,
+ // i.e. for cache memory allocation
+ kelondroFlexTable theFlexTable = (kelondroFlexTable) tableTracker.get(filename);
+ return theFlexTable.profile();
}
- public final int cacheNodeChunkSize() {
- // returns the size that the node cache uses for a single entry
- return -1;
+ public static final Map memoryStats(String filename) {
+ // returns a map for each file in the tracker;
+ // the map represents properties for each record oobjects,
+ // i.e. for cache memory allocation
+ kelondroFlexTable theFlexTable = (kelondroFlexTable) tableTracker.get(filename);
+ return theFlexTable.memoryStats();
}
- public final int[] cacheNodeStatus() {
- // a collection of different node cache status values
- return new int[]{0,0,0,0,0,0,0,0,0,0};
+ private final Map memoryStats() {
+ // returns statistical data about this object
+ HashMap map = new HashMap();
+ try {
+ map.put("tableIndexChunkSize", (!RAMIndex) ? "0" : Integer.toString(index.row().objectsize));
+ map.put("tableIndexCount", (!RAMIndex) ? "0" : Integer.toString(index.size()));
+ map.put("tableIndexMem", (!RAMIndex) ? "0" : Integer.toString((int) (index.row().objectsize * index.size() * kelondroRowCollection.growfactor)));
+ } catch (IOException e) {
+ }
+ return map;
}
public synchronized void close() throws IOException {
+ if (tableTracker.remove(this.filename) == null) {
+ serverLog.logWarning("kelondroFlexTable", "close(): file '" + this.filename + "' was not tracked with record tracker.");
+ }
index.close();
super.close();
}
diff --git a/source/de/anomic/kelondro/kelondroFlexWidthArray.java b/source/de/anomic/kelondro/kelondroFlexWidthArray.java index bac64972f..16310e6cd 100644 --- a/source/de/anomic/kelondro/kelondroFlexWidthArray.java +++ b/source/de/anomic/kelondro/kelondroFlexWidthArray.java @@ -40,6 +40,7 @@ public class kelondroFlexWidthArray implements kelondroArray { protected kelondroFixedWidthArray[] col;
protected kelondroRow rowdef;
protected String tablename;
+ protected String filename;
public kelondroFlexWidthArray(File path, String tablename, kelondroRow rowdef) throws IOException {
this.rowdef = rowdef;
@@ -61,6 +62,7 @@ public class kelondroFlexWidthArray implements kelondroArray { tabledir.mkdirs();
tabledir.mkdir();
}
+ this.filename = tabledir.getCanonicalPath();
// save/check property file for this array
File propfile = new File(tabledir, "properties");
@@ -109,6 +111,10 @@ public class kelondroFlexWidthArray implements kelondroArray { }
}
+ public final String filename() {
+ return this.filename;
+ }
+
public static int staticsize(File path, String tablename) {
// check if table directory exists
diff --git a/source/de/anomic/kelondro/kelondroIndex.java b/source/de/anomic/kelondro/kelondroIndex.java index df884fc5a..37fa51360 100644 --- a/source/de/anomic/kelondro/kelondroIndex.java +++ b/source/de/anomic/kelondro/kelondroIndex.java @@ -57,6 +57,7 @@ import java.util.List; public interface kelondroIndex { + public String filename(); // returns a unique identified for this index; can be a real or artificial file name public int size() throws IOException; public kelondroProfile profile(); public kelondroRow row() throws IOException; @@ -72,12 +73,4 @@ public interface kelondroIndex { public kelondroRow.Entry removeOne() throws IOException; public Iterator rows(boolean up, boolean rotating, byte[] firstKey) throws IOException; public void close() throws IOException; - - // statistics for node caches - public int cacheNodeChunkSize(); // the size that the node cache uses for a single entry - public int[] cacheNodeStatus(); // a collection of different node cache status values - - // statistics for object caches - public int cacheObjectChunkSize(); // the size of an object entry - public long[] cacheObjectStatus(); // a collection of different object cache status values } diff --git a/source/de/anomic/kelondro/kelondroMapTable.java b/source/de/anomic/kelondro/kelondroMapTable.java index 8233fa90f..96cc9eb95 100644 --- a/source/de/anomic/kelondro/kelondroMapTable.java +++ b/source/de/anomic/kelondro/kelondroMapTable.java @@ -70,19 +70,19 @@ public class kelondroMapTable { public void declareMaps(
String tablename, int keysize, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar) throws IOException {
- declareMaps(tablename, keysize, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0x800, 0);
+ declareMaps(tablename, keysize, nodesize, cacheslots, sortfields, longaccfields, doubleaccfields, fillChar, 0);
}
public void declareMaps(
String tablename, int keysize, int nodesize, int cacheslots,
String[] sortfields, String[] longaccfields, String[] doubleaccfields, char fillChar,
- long buffersize /*bytes*/, long preloadTime) throws IOException {
+ long preloadTime) throws IOException {
if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared twice.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareMap: table '" + tablename + "' declared already in other context.");
File tablefile = new File(tablesPath, "table." + tablename + ".mdb");
kelondroDyn dyn;
if (!(tablefile.exists())) tablefile.getParentFile().mkdirs();
- dyn = new kelondroDyn(tablefile, buffersize, preloadTime, keysize, nodesize, fillChar, true, false);
+ dyn = new kelondroDyn(tablefile, true, true, preloadTime, keysize, nodesize, fillChar, true, false);
kelondroMapObjects map = new kelondroMapObjects(dyn, cacheslots, sortfields, longaccfields, doubleaccfields, null, null);
mTables.put(tablename, map);
}
@@ -91,7 +91,7 @@ public class kelondroMapTable { if (mTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared already in other context.");
if (tTables.containsKey(tablename)) throw new RuntimeException("kelondroTables.declareTree: table '" + tablename + "' declared twice.");
File tablefile = new File(tablesPath, "table." + tablename + ".tdb");
- kelondroIndex Tree = new kelondroCache(kelondroTree.open(tablefile, buffersize / 2, preloadTime, rowdef), buffersize / 2, true, false);
+ kelondroIndex Tree = new kelondroCache(kelondroTree.open(tablefile, true, preloadTime, rowdef), true, false);
tTables.put(tablename, Tree);
}
diff --git a/source/de/anomic/kelondro/kelondroObjects.java b/source/de/anomic/kelondro/kelondroObjects.java index 11f9c7cfa..274f19637 100644 --- a/source/de/anomic/kelondro/kelondroObjects.java +++ b/source/de/anomic/kelondro/kelondroObjects.java @@ -52,22 +52,6 @@ public class kelondroObjects { return dyn.row().width(0); } - public int cacheNodeChunkSize() { - return dyn.cacheNodeChunkSize(); - } - - public int cacheObjectChunkSize() { - return dyn.cacheObjectChunkSize(); - } - - public int[] cacheNodeStatus() { - return dyn.cacheNodeStatus(); - } - - public long[] cacheObjectStatus() { - return dyn.cacheObjectStatus(); - } - public synchronized void set(String key, kelondroObjectsEntry newMap) throws IOException { assert (key != null); assert (key.length() > 0); diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index d85949930..a5e0df16a 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -71,11 +71,14 @@ package de.anomic.kelondro; import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.StringTokenizer;
+import java.util.TreeMap;
import java.util.TreeSet;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -87,7 +90,6 @@ public class kelondroRecords { // constants
protected final static int NUL = Integer.MIN_VALUE; // the meta value for the kelondroRecords' NUL abstraction
- private final static long memBlock = 500000; // do not fill cache further if the amount of available memory is less that this
public final static boolean useWriteBuffer = false; // currently only used during development/testing
// memory calculation
@@ -123,6 +125,11 @@ public class kelondroRecords { // POS_TXTPROPS: TXTPROPC * TXTPROPW : an array of TXTPROPC byte arrays of width TXTPROPW that can hold any string
// POS_NODES : (USEDC + FREEC) * (overhead + sum(all: COLWIDTHS)) : Node Objects
+ // static supervision objects: recognize and coordinate all activites
+ private static TreeMap recordTracker = new TreeMap(); // a String/filename - kelondroRecords mapping
+ private static long memStopGrow = 4000000; // a limit for the node cache to stop growing if less than this memory amount is available
+ private static long memStartShrink = 2000000; // a limit for the node cache to start with shrinking if less than this memory amount is available
+
// values that are only present at run-time
protected String filename; // the database's file name
private kelondroIOChunks entryFile; // the database file
@@ -148,7 +155,6 @@ public class kelondroRecords { // caching buffer
private kelondroIntBytesMap cacheHeaders; // the cache; holds overhead values and key element
- private int cacheSize; // number of cache records
private int readHit, readMiss, writeUnique, writeDouble, cacheDelete, cacheFlush;
// optional logger
@@ -385,7 +391,7 @@ public class kelondroRecords { }
- public kelondroRecords(File file, long buffersize /* bytes */, long preloadTime,
+ public kelondroRecords(File file, boolean useNodeCache, long preloadTime,
short ohbytec, short ohhandlec,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth) throws IOException {
// opens an existing file or creates a new file
@@ -403,13 +409,13 @@ public class kelondroRecords { //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
//kelondroRA raf = new kelondroCachedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
- initExistingFile(raf, buffersize / 10);
+ initExistingFile(raf, useNodeCache);
} else {
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
// kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 1024, 100);
// kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
- initNewFile(raf, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10);
+ initNewFile(raf, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, useNodeCache);
}
assignRowdef(rowdef);
if (fileExisted) {
@@ -422,33 +428,35 @@ public class kelondroRecords { // create new file structure
writeOrderType();
}
- initCache(buffersize / 10 * 9, preloadTime);
+ initCache(useNodeCache, preloadTime);
+ if (useNodeCache) recordTracker.put(this.filename, this);
}
- public kelondroRecords(kelondroRA ra, long buffersize /* bytes */, long preloadTime,
+ public kelondroRecords(kelondroRA ra, String filename, boolean useCache, long preloadTime,
short ohbytec, short ohhandlec,
kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth,
boolean exitOnFail) {
// this always creates a new file
this.fileExisted = false;
- this.filename = null;
+ this.filename = filename;
try {
- initNewFile(ra, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, buffersize / 10);
+ initNewFile(ra, ohbytec, ohhandlec, rowdef, FHandles, txtProps, txtPropWidth, useCache);
} catch (IOException e) {
logFailure("cannot create / " + e.getMessage());
if (exitOnFail) System.exit(-1);
}
assignRowdef(rowdef);
writeOrderType();
- initCache(buffersize / 10 * 9, preloadTime);
+ initCache(useCache, preloadTime);
+ if (useCache) recordTracker.put(this.filename, this);
}
-
+
private void initNewFile(kelondroRA ra, short ohbytec, short ohhandlec,
- kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, long writeBufferSize) throws IOException {
+ kelondroRow rowdef, int FHandles, int txtProps, int txtPropWidth, boolean useCache) throws IOException {
// create new Chunked IO
if (useWriteBuffer) {
- this.entryFile = new kelondroBufferedIOChunks(ra, ra.name(), writeBufferSize, 30000 + random.nextLong() % 30000);
+ this.entryFile = new kelondroBufferedIOChunks(ra, ra.name(), 0, 30000 + random.nextLong() % 30000);
} else {
this.entryFile = new kelondroRAIOChunks(ra, ra.name());
}
@@ -558,18 +566,19 @@ public class kelondroRecords { this.theLogger.fine("KELONDRO DEBUG " + this.filename + ": " + message);
}
- public kelondroRecords(kelondroRA ra, long buffersize, long preloadTime) throws IOException{
+ public kelondroRecords(kelondroRA ra, String filename, boolean useNodeCache, long preloadTime) throws IOException{
this.fileExisted = false;
- this.filename = null;
- initExistingFile(ra, buffersize / 10);
+ this.filename = filename;
+ initExistingFile(ra, useNodeCache);
readOrderType();
- initCache(buffersize / 10 * 9, preloadTime);
+ initCache(useNodeCache, preloadTime);
+ if (useNodeCache) recordTracker.put(this.filename, this);
}
- private void initExistingFile(kelondroRA ra, long writeBufferSize) throws IOException {
+ private void initExistingFile(kelondroRA ra, boolean useNodeCache) throws IOException {
// read from Chunked IO
- if (useWriteBuffer) {
- this.entryFile = new kelondroBufferedIOChunks(ra, ra.name(), writeBufferSize, 30000 + random.nextLong() % 30000);
+ if ((useWriteBuffer) && (useNodeCache)) {
+ this.entryFile = new kelondroBufferedIOChunks(ra, ra.name(), 0, 30000 + random.nextLong() % 30000);
} else {
this.entryFile = new kelondroRAIOChunks(ra, ra.name());
}
@@ -641,13 +650,11 @@ public class kelondroRecords { return oo;
}
- private void initCache(long buffersize, long preloadTime) {
- if (buffersize <= 0) {
- this.cacheSize = 0;
- this.cacheHeaders = null;
- } else {
- this.cacheSize = (int) (buffersize / cacheNodeChunkSize());
+ private void initCache(boolean useNodeCache, long preloadTime) {
+ if (useNodeCache) {
this.cacheHeaders = new kelondroIntBytesMap(this.headchunksize, 0);
+ } else {
+ this.cacheHeaders = null;
}
this.readHit = 0;
this.readMiss = 0;
@@ -656,13 +663,13 @@ public class kelondroRecords { this.cacheDelete = 0;
this.cacheFlush = 0;
// pre-load node cache
- if ((preloadTime > 0) && (cacheSize > 0)) {
+ if ((preloadTime > 0) && (useNodeCache)) {
long stop = System.currentTimeMillis() + preloadTime;
int count = 0;
try {
Iterator i = contentNodes(preloadTime);
Node n;
- while ((System.currentTimeMillis() < stop) && (cacheHeaders.size() < cacheSize) && (i.hasNext())) {
+ while ((System.currentTimeMillis() < stop) && (cacheGrowStatus() == 2) && (i.hasNext())) {
n = (Node) i.next();
cacheHeaders.addb(n.handle.index, n.headChunk);
count++;
@@ -677,66 +684,65 @@ public class kelondroRecords { }
}
- public File file() {
- if (filename == null) return null;
- return new File(filename);
+ private int cacheGrowStatus() {
+ return cacheGrowStatus(memStopGrow, memStartShrink);
}
- public final int cacheObjectChunkSize() {
- // dummy method
- return -1;
+ public static final int cacheGrowStatus(long stopGrow, long startShrink) {
+ // returns either 0, 1 or 2:
+ // 0: cache is not allowed to grow, but shall shrink
+ // 1: cache is allowed to grow, but need not to shrink
+ // 2: cache is allowed to grow and must not shrink
+ long available = serverMemory.available();
+ if (available > stopGrow) return 2;
+ if (available > startShrink) return 1;
+ return 0;
}
- public long[] cacheObjectStatus() {
- // dummy method
- return null;
+ public static void setCacheGrowStati(long memStopGrowNew, long memStartShrinkNew) {
+ memStopGrow = memStopGrowNew;
+ memStartShrink = memStartShrinkNew;
}
- public final int cacheNodeChunkSize() {
- // returns the size that the node cache uses for a single entry
- return this.headchunksize + element_in_cache;
+ public static long getMemStopGrow() {
+ return memStopGrow ;
}
- public final int[] cacheNodeStatus() {
- // a collection of different node cache status values
- if (cacheHeaders == null) return new int[]{0,0,0,0,0,0,0,0,0,0};
- return new int[]{
- cacheSize,
- cacheHeaders.size(),
- 0, // not used
- 0, // not used
- readHit,
- readMiss,
- writeUnique,
- writeDouble,
- cacheDelete,
- cacheFlush
- };
+ public static long getMemStartShrink() {
+ return memStartShrink ;
}
- public final String cacheNodeStatusString() {
- return
- "cacheMaxSize=" + cacheSize +
- ", cacheCurrSize=" + ((cacheHeaders == null) ? 0 : cacheHeaders.size()) +
- ", readHit=" + readHit +
- ", readMiss=" + readMiss +
- ", writeUnique=" + writeUnique +
- ", writeDouble=" + writeDouble +
- ", cacheDelete=" + cacheDelete +
- ", cacheFlush=" + cacheFlush;
+ public String filename() {
+ return filename;
}
- private final static int[] cacheCombinedStatus(int[] a, int[] b) {
- int[] c = new int[a.length];
- for (int i = a.length - 1; i >= 0; i--) c[i] = a[i] + b[i];
- return c;
+ public static final Iterator filenames() {
+ // iterates string objects; all file names from record tracker
+ return recordTracker.keySet().iterator();
+ }
+
+ public static final Map memoryStats(String filename) {
+ // returns a map for each file in the tracker;
+ // the map represents properties for each record oobjects,
+ // i.e. for cache memory allocation
+ kelondroRecords theRecord = (kelondroRecords) recordTracker.get(filename);
+ return theRecord.memoryStats();
}
- public final static int[] cacheCombinedStatus(int[][] a, int l) {
- if ((a == null) || (a.length == 0) || (l == 0)) return null;
- if ((a.length >= 1) && (l == 1)) return a[0];
- if ((a.length >= 2) && (l == 2)) return cacheCombinedStatus(a[0], a[1]);
- return cacheCombinedStatus(cacheCombinedStatus(a, l - 1), a[l - 1]);
+ private final Map memoryStats() {
+ // returns statistical data about this object
+ if (cacheHeaders == null) return null;
+ HashMap map = new HashMap();
+ map.put("nodeChunkSize", Integer.toString(this.headchunksize + element_in_cache));
+ map.put("nodeCacheCount", Integer.toString(cacheHeaders.size()));
+ map.put("nodeCacheMem", Integer.toString(cacheHeaders.size() * (this.headchunksize + element_in_cache)));
+ map.put("nodeCacheReadHit", Integer.toString(readHit));
+ map.put("nodeCacheReadMiss", Integer.toString(readMiss));
+ map.put("nodeCacheWriteUnique", Integer.toString(writeUnique));
+ map.put("nodeCacheWriteDouble", Integer.toString(writeDouble));
+ map.put("nodeCacheDeletes", Integer.toString(cacheDelete));
+ map.put("nodeCacheFlushes", Integer.toString(cacheFlush));
+ return map;
}
public final byte[] bulkRead(int start, int end) throws IOException {
@@ -772,7 +778,7 @@ public class kelondroRecords { }
protected synchronized final void deleteNode(Handle handle) throws IOException {
- if ((cacheHeaders == null) || (cacheSize == 0)) {
+ if ((cacheHeaders == null) || (cacheHeaders.size() == 0)) {
USAGE.dispose(handle);
} else synchronized (cacheHeaders) {
cacheHeaders.removeb(handle.index);
@@ -901,7 +907,7 @@ public class kelondroRecords { // init the content
// create chunks; read them from file or cache
this.tailChunk = null;
- if (cacheSize == 0) {
+ if (cacheHeaders == null) {
if (fillTail) {
// read complete record
byte[] chunkbuffer = new byte[recordsize];
@@ -950,7 +956,7 @@ public class kelondroRecords { }
// if space left in cache, copy these value to the cache
- update2Cache(cp);
+ updateNodeCache(cp);
} else {
readHit++;
this.headChunk = cacheEntry;
@@ -1067,7 +1073,7 @@ public class kelondroRecords { if (this.headChanged) {
//System.out.println("WRITEH(" + filename + ", " + seekpos(this.handle) + ", " + this.headChunk.length + ")");
entryFile.write(seekpos(this.handle), (this.headChunk == null) ? new byte[headchunksize] : this.headChunk);
- update2Cache(cachePriority);
+ updateNodeCache(cachePriority);
this.headChanged = false;
}
@@ -1117,32 +1123,29 @@ public class kelondroRecords { return s;
}
- private void update2Cache(int forPriority) {
- if (cacheSpace()) updateNodeCache(forPriority);
- }
-
private boolean cacheSpace() {
// check for space in cache
// should be only called within a synchronized(cacheHeaders) environment
// returns true if it is allowed to add another entry to the cache
// returns false if the cache is considered to be full
- if (cacheSize == 0) return false; // no caching
+ if (cacheHeaders == null) return false; // no caching
if (cacheHeaders.size() == 0) return true; // nothing there to flush
- if ((cacheHeaders.size() < cacheSize) && (serverMemory.available() >= memBlock)) return true; // no need to flush cache space
+ if (cacheGrowStatus() == 2) return true; // no need to flush cache space
// just delete any of the entries
- cacheHeaders.removeoneb();
- cacheFlush++;
- return true;
+ if (cacheGrowStatus() <= 1) {
+ cacheHeaders.removeoneb();
+ cacheFlush++;
+ }
+ return cacheGrowStatus() > 0;
}
private void updateNodeCache(int priority) {
if (this.handle == null) return; // wrong access
if (this.headChunk == null) return; // nothing there to cache
if (priority == CP_NONE) return; // it is not wanted that this shall be cached
- if (cacheSize == 0) return; // we do not use the cache
- if (cacheHeaders == null) return; // no cache
- if (cacheHeaders.size() >= cacheSize) return; // no cache update if cache is full
+ if (cacheHeaders == null) return; // we do not use the cache
+ if (!(cacheSpace())) return;
synchronized (cacheHeaders) {
// generate cache entry
@@ -1162,7 +1165,7 @@ public class kelondroRecords { }
protected void printCache() {
- if (cacheSize == 0) {
+ if (cacheHeaders == null) {
System.out.println("### file report: " + size() + " entries");
for (int i = 0; i < USAGE.allCount(); i++) {
// print from file to compare
@@ -1467,12 +1470,24 @@ public class kelondroRecords { }
- public void close() throws IOException {
+ public synchronized void close() throws IOException {
+ if (cacheHeaders == null) {
+ if (recordTracker.get(this.filename) != null) {
+ theLogger.severe("close(): file '" + this.filename + "' was tracked with record tracker, but it should not.");
+ }
+ } else {
+ if (recordTracker.remove(this.filename) == null) {
+ theLogger.severe("close(): file '" + this.filename + "' was not tracked with record tracker.");
+ }
+ }
if (entryFile == null) {
- theLogger.severe("File was closed before close was called. File = " + filename);
+ theLogger.severe("close(): file '" + this.filename + "' was closed before close was called.");
} else {
USAGE.writeused(true);
this.entryFile.close();
+ this.entryFile = null;
+ this.cacheHeaders = null;
+ theLogger.fine("close(): file '" + this.filename + "' closed.");
}
}
diff --git a/source/de/anomic/kelondro/kelondroRowCollection.java b/source/de/anomic/kelondro/kelondroRowCollection.java index 7cc552c4c..2a40061c8 100644 --- a/source/de/anomic/kelondro/kelondroRowCollection.java +++ b/source/de/anomic/kelondro/kelondroRowCollection.java @@ -36,6 +36,8 @@ import de.anomic.server.logging.serverLog; public class kelondroRowCollection { + public static final double growfactor = 1.4; + protected byte[] chunkcache; protected int chunkcount; protected long lastTimeRead, lastTimeWrote; @@ -160,7 +162,7 @@ public class kelondroRowCollection { private final void ensureSize(int elements) { int needed = elements * rowdef.objectsize(); if (chunkcache.length >= needed) return; - byte[] newChunkcache = new byte[needed * 2]; // increase space + byte[] newChunkcache = new byte[(int) (needed * growfactor)]; // increase space System.arraycopy(chunkcache, 0, newChunkcache, 0, chunkcache.length); chunkcache = newChunkcache; newChunkcache = null; diff --git a/source/de/anomic/kelondro/kelondroRowSet.java b/source/de/anomic/kelondro/kelondroRowSet.java index 69de6e4a3..90cc8c792 100644 --- a/source/de/anomic/kelondro/kelondroRowSet.java +++ b/source/de/anomic/kelondro/kelondroRowSet.java @@ -549,4 +549,8 @@ public class kelondroRowSet extends kelondroRowCollection implements kelondroInd public static byte[] randomHash(Random r) {
return randomHash(r.nextLong(), r.nextLong());
}
+
+ public String filename() {
+ return null;
+ }
}
diff --git a/source/de/anomic/kelondro/kelondroSplittedTree.java b/source/de/anomic/kelondro/kelondroSplittedTree.java index ce2576294..49dd9495f 100644 --- a/source/de/anomic/kelondro/kelondroSplittedTree.java +++ b/source/de/anomic/kelondro/kelondroSplittedTree.java @@ -57,6 +57,7 @@ public class kelondroSplittedTree implements kelondroIndex { private kelondroTree[] ktfs; private kelondroOrder order; private int ff; + private String filename; private static File dbFile(File path, String filenameStub, int forkfactor, int columns, int number) { String ns = Integer.toHexString(number).toUpperCase(); @@ -67,24 +68,20 @@ public class kelondroSplittedTree implements kelondroIndex { while (co.length() < 2) co = "0" + co; return new File(path, filenameStub + "." + ff + "." + co + "." + ns + ".ktc"); } - - /* - private static boolean existsAll(File pathToFiles, String filenameStub, int forkfactor, int columns){ - for (int i = 0; i < forkfactor; i++) { - if (!(dbFile(pathToFiles, filenameStub, forkfactor, columns, i)).exists()) return false; - } - return true; - } - */ public kelondroSplittedTree(File pathToFiles, String filenameStub, kelondroOrder objectOrder, - long buffersize, long preloadTime, + long preloadTime, int forkfactor, kelondroRow rowdef, int txtProps, int txtPropsWidth) { + try { + this.filename = new File(pathToFiles, filenameStub).getCanonicalPath(); + } catch (IOException e) { + this.filename = null; + } ktfs = new kelondroTree[forkfactor]; File f; for (int i = 0; i < forkfactor; i++) { f = dbFile(pathToFiles, filenameStub, forkfactor, rowdef.columns(), i); - ktfs[i] = kelondroTree.open(f, buffersize/forkfactor, preloadTime / forkfactor, rowdef, txtProps, txtPropsWidth); + ktfs[i] = kelondroTree.open(f, true, preloadTime / forkfactor, rowdef, txtProps, txtPropsWidth); } this.order = objectOrder; ff = forkfactor; @@ -279,4 +276,8 @@ public class kelondroSplittedTree implements kelondroIndex { return new int[]{0,0,0,0,0,0,0,0,0,0}; } + public String filename() { + return this.filename; + } + } diff --git a/source/de/anomic/kelondro/kelondroStack.java b/source/de/anomic/kelondro/kelondroStack.java index 10e7b43d9..f103904d4 100644 --- a/source/de/anomic/kelondro/kelondroStack.java +++ b/source/de/anomic/kelondro/kelondroStack.java @@ -69,7 +69,7 @@ public final class kelondroStack extends kelondroRecords { public kelondroStack(File file, kelondroRow rowdef) throws IOException {
// this creates a new stack
- super(file, 0, 0, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
+ super(file, false, 0, thisOHBytes, thisOHHandles, rowdef, thisFHandles, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
if (super.fileExisted) {
//if ((getHandle(root) == null) && (getHandle(toor) == null)) clear();
} else {
diff --git a/source/de/anomic/kelondro/kelondroTree.java b/source/de/anomic/kelondro/kelondroTree.java index 67bee31a3..57f2dfa9e 100644 --- a/source/de/anomic/kelondro/kelondroTree.java +++ b/source/de/anomic/kelondro/kelondroTree.java @@ -91,13 +91,13 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { protected int readAheadChunkSize = 100;
protected long lastIteratorCount = readAheadChunkSize;
- public kelondroTree(File file, long buffersize, long preloadTime, kelondroRow rowdef) throws IOException {
- this(file, buffersize, preloadTime, rowdef, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
+ public kelondroTree(File file, boolean useNodeCache, long preloadTime, kelondroRow rowdef) throws IOException {
+ this(file, useNodeCache, preloadTime, rowdef, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
}
- public kelondroTree(File file, long buffersize, long preloadTime, kelondroRow rowdef, int txtProps, int txtPropsWidth) throws IOException {
+ public kelondroTree(File file, boolean useNodeCache, long preloadTime, kelondroRow rowdef, int txtProps, int txtPropsWidth) throws IOException {
// opens an existing tree file or creates a new tree file
- super(file, buffersize, preloadTime,
+ super(file, useNodeCache, preloadTime,
thisOHBytes, thisOHHandles, rowdef,
thisFHandles, txtProps, txtPropsWidth);
if (!fileExisted) {
@@ -108,19 +108,19 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { }
- public static final kelondroTree open(File file, long buffersize, long preloadTime, kelondroRow rowdef) {
- return open(file, buffersize, preloadTime, rowdef, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
+ public static final kelondroTree open(File file, boolean useNodeCache, long preloadTime, kelondroRow rowdef) {
+ return open(file, useNodeCache, preloadTime, rowdef, rowdef.columns() /* txtProps */, 80 /* txtPropWidth */);
}
- public static final kelondroTree open(File file, long buffersize, long preloadTime, kelondroRow rowdef, int txtProps, int txtPropsWidth) {
+ public static final kelondroTree open(File file, boolean useNodeCache, long preloadTime, kelondroRow rowdef, int txtProps, int txtPropsWidth) {
// opens new or existing file; in case that any error occur the file is deleted again and it is tried to create the file again
// if that fails, the method returns null
try {
- return new kelondroTree(file, buffersize, preloadTime, rowdef, txtProps, txtPropsWidth);
+ return new kelondroTree(file, useNodeCache, preloadTime, rowdef, txtProps, txtPropsWidth);
} catch (IOException e) {
file.delete();
try {
- return new kelondroTree(file, buffersize, preloadTime, rowdef, txtProps, txtPropsWidth);
+ return new kelondroTree(file, useNodeCache, preloadTime, rowdef, txtProps, txtPropsWidth);
} catch (IOException ee) {
log.severe("cannot open or create file " + file.toString());
e.printStackTrace();
@@ -130,14 +130,14 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { }
}
- public kelondroTree(kelondroRA ra, long buffersize, long preloadTime, kelondroRow rowdef, boolean exitOnFail) {
+ public kelondroTree(kelondroRA ra, String filename, boolean useNodeCache, long preloadTime, kelondroRow rowdef, boolean exitOnFail) {
// this creates a new tree within a kelondroRA
- this(ra, buffersize, preloadTime, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
+ this(ra, filename, useNodeCache, preloadTime, rowdef, new kelondroNaturalOrder(true), rowdef.columns() /* txtProps */, 80 /* txtPropWidth */, exitOnFail);
}
- public kelondroTree(kelondroRA ra, long buffersize, long preloadTime, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
+ public kelondroTree(kelondroRA ra, String filename, boolean useNodeCache, long preloadTime, kelondroRow rowdef, kelondroOrder objectOrder, int txtProps, int txtPropsWidth, boolean exitOnFail) {
// this creates a new tree within a kelondroRA
- super(ra, buffersize, preloadTime,
+ super(ra, filename, useNodeCache, preloadTime,
thisOHBytes, thisOHHandles, rowdef,
thisFHandles, txtProps, txtPropsWidth, exitOnFail);
try {
@@ -150,9 +150,9 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { super.setLogger(log);
}
- public kelondroTree(kelondroRA ra, long buffersize, long preloadTime) throws IOException {
+ public kelondroTree(kelondroRA ra, String filename, boolean useNodeCache, long preloadTime) throws IOException {
// this opens a file with an existing tree in a kelondroRA
- super(ra, buffersize, preloadTime);
+ super(ra, filename, useNodeCache, preloadTime);
super.setLogger(log);
}
@@ -1220,7 +1220,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { // test script
File testFile = new File("test.db");
while (testFile.exists()) testFile.delete();
- kelondroTree fm = new kelondroTree(testFile, 0x100000, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
+ kelondroTree fm = new kelondroTree(testFile, true, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
byte[] dummy = "".getBytes();
fm.put("abc0".getBytes(), dummy); fm.put("bcd0".getBytes(), dummy);
fm.put("def0".getBytes(), dummy); fm.put("bab0".getBytes(), dummy);
@@ -1242,7 +1242,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { ret = null;
}
} else if (args.length == 2) {
- kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
+ kelondroTree fm = new kelondroTree(new File(args[1]), true, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
if (args[0].equals("-v")) {
fm.print();
ret = null;
@@ -1250,11 +1250,11 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { fm.close();
} else if (args.length == 3) {
if (args[0].equals("-d")) {
- kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
+ kelondroTree fm = new kelondroTree(new File(args[1]), true, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
fm.remove(args[2].getBytes());
fm.close();
} else if (args[0].equals("-i")) {
- kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
+ kelondroTree fm = new kelondroTree(new File(args[1]), true, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
int i = fm.imp(new File(args[1]),";");
fm.close();
ret = (i + " records imported").getBytes();
@@ -1277,12 +1277,12 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { if (f != null) try {f.close();}catch(Exception e){}
}
} else if (args[0].equals("-g")) {
- kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
+ kelondroTree fm = new kelondroTree(new File(args[1]), true, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
kelondroRow.Entry ret2 = fm.get(args[2].getBytes());
ret = ((ret2 == null) ? null : ret2.getColBytes(1));
fm.close();
} else if (args[0].equals("-n")) {
- kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
+ kelondroTree fm = new kelondroTree(new File(args[1]), true, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
//byte[][] keys = fm.getSequentialKeys(args[2].getBytes(), 500, true);
Iterator rowIt = fm.rows(true, false, (args[2].length() == 0) ? null : args[2].getBytes());
Vector v = new Vector();
@@ -1296,10 +1296,10 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { File f = new File(args[3]);
if (f.exists()) f.delete();
kelondroRow lens = new kelondroRow("byte[] key-" + Integer.parseInt(args[1]) + ", byte[] value-" + Integer.parseInt(args[2]), kelondroNaturalOrder.naturalOrder, 0);
- kelondroTree fm = new kelondroTree(f, 0x100000, 10, lens);
+ kelondroTree fm = new kelondroTree(f, true, 10, lens);
fm.close();
} else if (args[0].equals("-u")) {
- kelondroTree fm = new kelondroTree(new File(args[1]), 0x100000, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
+ kelondroTree fm = new kelondroTree(new File(args[1]), true, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
ret = fm.put(args[1].getBytes(), args[2].getBytes());
fm.close();
}
@@ -1358,7 +1358,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { int steps = 0;
while (true) {
if (testFile.exists()) testFile.delete();
- tt = new kelondroTree(testFile, 200, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
+ tt = new kelondroTree(testFile, true, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
steps = 10 + ((int) System.currentTimeMillis() % 7) * (((int) System.currentTimeMillis() + 17) % 11);
t = s;
d = "";
@@ -1424,7 +1424,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { File f = new File("test.db");
if (f.exists()) f.delete();
try {
- kelondroTree tt = new kelondroTree(f, 1000, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
+ kelondroTree tt = new kelondroTree(f, true, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
byte[] b;
b = testWord('B'); tt.put(b, b); //tt.print();
b = testWord('C'); tt.put(b, b); //tt.print();
@@ -1489,7 +1489,7 @@ public class kelondroTree extends kelondroRecords implements kelondroIndex { public static kelondroTree testTree(File f, String testentities) throws IOException {
if (f.exists()) f.delete();
- kelondroTree tt = new kelondroTree(f, 0, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
+ kelondroTree tt = new kelondroTree(f, false, 10, new kelondroRow("byte[] a-4, byte[] b-4", kelondroNaturalOrder.naturalOrder, 0));
byte[] b;
for (int i = 0; i < testentities.length(); i++) {
b = testWord(testentities.charAt(i));
diff --git a/source/de/anomic/plasma/dbImport/AssortmentImporter.java b/source/de/anomic/plasma/dbImport/AssortmentImporter.java index 013e4c321..2ad5daaa1 100644 --- a/source/de/anomic/plasma/dbImport/AssortmentImporter.java +++ b/source/de/anomic/plasma/dbImport/AssortmentImporter.java @@ -58,7 +58,7 @@ public class AssortmentImporter extends AbstractImporter implements dbImporter{ // initializing the import assortment db
this.log.logInfo("Initializing source assortment file " + theImportAssortmentFile);
- this.assortmentFile = new plasmaWordIndexAssortment(importAssortmentPath, assortmentNr, this.cacheSize/1024, preloadTime, this.log);
+ this.assortmentFile = new plasmaWordIndexAssortment(importAssortmentPath, assortmentNr, preloadTime, this.log);
this.importStartSize = this.assortmentFile.size();
}
diff --git a/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java b/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java index 7e7683424..721656559 100644 --- a/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java +++ b/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java @@ -89,13 +89,13 @@ public class plasmaCrawlNURLImporter extends AbstractImporter implements dbImpor // init noticeUrlDB
this.log.logInfo("Initializing the source noticeUrlDB");
- this.importNurlDB = new plasmaCrawlNURL(this.importPath, ((this.cacheSize*3)/4)/1024, preloadTime);
+ this.importNurlDB = new plasmaCrawlNURL(this.importPath, preloadTime);
this.importStartSize = this.importNurlDB.size();
//int stackSize = this.importNurlDB.stackSize();
// init profile DB
this.log.logInfo("Initializing the source profileDB");
- this.importProfileDB = new plasmaCrawlProfile(profileDbFile, ((this.cacheSize*1)/4)/1024, 300);
+ this.importProfileDB = new plasmaCrawlProfile(profileDbFile, 300);
}
public void run() {
diff --git a/source/de/anomic/plasma/dbImport/plasmaDbImporter.java b/source/de/anomic/plasma/dbImport/plasmaDbImporter.java index 8a3b08d58..64564b7e3 100644 --- a/source/de/anomic/plasma/dbImport/plasmaDbImporter.java +++ b/source/de/anomic/plasma/dbImport/plasmaDbImporter.java @@ -64,7 +64,7 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter { }
this.log.logFine("Initializing source word index db.");
- this.importWordIndex = new plasmaWordIndex(this.importPath, this.cacheSize/2, this.cacheSize/2, preloadTime / 2, this.log);
+ this.importWordIndex = new plasmaWordIndex(this.importPath, preloadTime / 2, this.log);
this.importStartSize = this.importWordIndex.size();
}
diff --git a/source/de/anomic/plasma/plasmaCrawlEURL.java b/source/de/anomic/plasma/plasmaCrawlEURL.java index ad6416463..46af6a40e 100644 --- a/source/de/anomic/plasma/plasmaCrawlEURL.java +++ b/source/de/anomic/plasma/plasmaCrawlEURL.java @@ -56,11 +56,9 @@ import java.util.LinkedList; import de.anomic.plasma.plasmaURL;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBitfield;
-import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroRow;
-import de.anomic.kelondro.kelondroTree;
import de.anomic.net.URL;
import de.anomic.yacy.yacySeedDB;
@@ -152,12 +150,12 @@ public class plasmaCrawlEURL { // the class object
private kelondroIndex urlIndexFile = null;
- public plasmaCrawlEURL(File cachePath, int bufferkb, long preloadTime) {
+ public plasmaCrawlEURL(File cachePath, long preloadTime) {
super();
String newCacheName = "urlErr3.table";
cachePath.mkdirs();
try {
- urlIndexFile = new kelondroFlexTable(cachePath, newCacheName, bufferkb * 0x400, preloadTime, rowdef);
+ urlIndexFile = new kelondroFlexTable(cachePath, newCacheName, preloadTime, rowdef);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
@@ -182,34 +180,6 @@ public class plasmaCrawlEURL { urlIndexFile = null;
}
}
-
- public int cacheNodeChunkSize() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheNodeChunkSize();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheNodeChunkSize();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheNodeChunkSize();
- return 0;
- }
-
- public int[] cacheNodeStatus() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheNodeStatus();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheNodeStatus();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheNodeStatus();
- return new int[]{0,0,0,0,0,0,0,0,0,0};
- }
-
- public int cacheObjectChunkSize() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheObjectChunkSize();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheObjectChunkSize();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheObjectChunkSize();
- return 0;
- }
-
- public long[] cacheObjectStatus() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheObjectStatus();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheObjectStatus();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheObjectStatus();
- return new long[]{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
- }
public synchronized Entry newEntry(URL url, String referrer, String initiator, String executor,
String name, String failreason, kelondroBitfield flags) {
diff --git a/source/de/anomic/plasma/plasmaCrawlLURL.java b/source/de/anomic/plasma/plasmaCrawlLURL.java index 9a6d9289f..de9e38e58 100644 --- a/source/de/anomic/plasma/plasmaCrawlLURL.java +++ b/source/de/anomic/plasma/plasmaCrawlLURL.java @@ -7,7 +7,7 @@ //
// $LastChangedDate$
// $LastChangedRevision$
-// $LastChangedBy$
+// $LastChangedBy: $
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
@@ -68,10 +68,8 @@ import de.anomic.index.indexURLEntryNew; import de.anomic.kelondro.kelondroBitfield;
import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroFlexSplitTable;
-import de.anomic.kelondro.kelondroFlexTable;
import de.anomic.kelondro.kelondroIndex;
import de.anomic.kelondro.kelondroRow;
-import de.anomic.kelondro.kelondroTree;
import de.anomic.net.URL;
import de.anomic.plasma.urlPattern.plasmaURLPattern;
import de.anomic.server.serverCodings;
@@ -93,11 +91,11 @@ public final class plasmaCrawlLURL { // the class object
private kelondroIndex urlIndexFile;
- public plasmaCrawlLURL(File indexPath, long buffer, long preloadTime) {
+ public plasmaCrawlLURL(File indexPath, long preloadTime) {
super();
try {
- urlIndexFile = new kelondroFlexSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", buffer, preloadTime, indexURLEntryNew.rowdef);
+ urlIndexFile = new kelondroFlexSplitTable(new File(indexPath, "PUBLIC/TEXT"), "urls", preloadTime, indexURLEntryNew.rowdef);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
@@ -127,34 +125,6 @@ public final class plasmaCrawlLURL { }
}
- public int cacheNodeChunkSize() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheNodeChunkSize();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheNodeChunkSize();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheNodeChunkSize();
- return 0;
- }
-
- public int[] cacheNodeStatus() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheNodeStatus();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheNodeStatus();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheNodeStatus();
- return new int[]{0,0,0,0,0,0,0,0,0,0};
- }
-
- public int cacheObjectChunkSize() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheObjectChunkSize();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheObjectChunkSize();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheObjectChunkSize();
- return 0;
- }
-
- public long[] cacheObjectStatus() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheObjectStatus();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheObjectStatus();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheObjectStatus();
- return new long[]{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
- }
-
public synchronized void stack(indexURLEntry e, String initiatorHash, String executorHash, int stackType) {
if (e == null) { return; }
try {
@@ -590,7 +560,7 @@ public final class plasmaCrawlLURL { } catch (MalformedURLException e) {}
if (args[0].equals("-l")) try {
// arg 1 is path to URLCache
- final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[2]), 1, 0);
+ final plasmaCrawlLURL urls = new plasmaCrawlLURL(new File(args[2]), 0);
final Iterator enu = urls.entries(true, false, null);
while (enu.hasNext()) {
System.out.println(((indexURLEntry) enu.next()).toString());
diff --git a/source/de/anomic/plasma/plasmaCrawlNURL.java b/source/de/anomic/plasma/plasmaCrawlNURL.java index c58223ff7..db20f5cd7 100644 --- a/source/de/anomic/plasma/plasmaCrawlNURL.java +++ b/source/de/anomic/plasma/plasmaCrawlNURL.java @@ -60,7 +60,6 @@ import de.anomic.kelondro.kelondroIndex; import de.anomic.kelondro.kelondroRecords;
import de.anomic.kelondro.kelondroRow;
import de.anomic.kelondro.kelondroStack;
-import de.anomic.kelondro.kelondroTree;
import de.anomic.net.URL;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacySeedDB;
@@ -109,17 +108,15 @@ public class plasmaCrawlNURL { private final HashSet imageStackIndex, movieStackIndex, musicStackIndex; // to find out if a specific link is already on any stack
private File cacheStacksPath;
- private int bufferkb;
private long preloadTime;
initStackIndex initThead;
// the class object
private kelondroIndex urlIndexFile = null;
- public plasmaCrawlNURL(File cachePath, int bufferkb, long preloadTime) {
+ public plasmaCrawlNURL(File cachePath, long preloadTime) {
super();
this.cacheStacksPath = cachePath;
- this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
// create a stack for newly entered entries
@@ -157,34 +154,6 @@ public class plasmaCrawlNURL { }
}
- public int cacheNodeChunkSize() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheNodeChunkSize();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheNodeChunkSize();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheNodeChunkSize();
- return 0;
- }
-
- public int[] cacheNodeStatus() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheNodeStatus();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheNodeStatus();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheNodeStatus();
- return new int[]{0,0,0,0,0,0,0,0,0,0};
- }
-
- public int cacheObjectChunkSize() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheObjectChunkSize();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheObjectChunkSize();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheObjectChunkSize();
- return 0;
- }
-
- public long[] cacheObjectStatus() {
- if (urlIndexFile instanceof kelondroTree) return ((kelondroTree) urlIndexFile).cacheObjectStatus();
- if (urlIndexFile instanceof kelondroCache) return ((kelondroCache) urlIndexFile).cacheObjectStatus();
- if (urlIndexFile instanceof kelondroFlexTable) return ((kelondroFlexTable) urlIndexFile).cacheObjectStatus();
- return new long[]{0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
- }
-
public void waitOnInitThread() {
try {
if (this.initThead != null) {
@@ -203,7 +172,7 @@ public class plasmaCrawlNURL { String newCacheName = "urlNotice5.table";
cacheStacksPath.mkdirs();
try {
- urlIndexFile = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, bufferkb / 2 * 0x400, preloadTime, rowdef), bufferkb / 2 * 0x400, true, false);
+ urlIndexFile = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, rowdef), true, false);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
diff --git a/source/de/anomic/plasma/plasmaCrawlProfile.java b/source/de/anomic/plasma/plasmaCrawlProfile.java index 54c071d44..d2d0a1207 100644 --- a/source/de/anomic/plasma/plasmaCrawlProfile.java +++ b/source/de/anomic/plasma/plasmaCrawlProfile.java @@ -59,43 +59,25 @@ public class plasmaCrawlProfile { private kelondroMapObjects profileTable;
private HashMap domsCache;
private File profileTableFile;
- private int bufferkb;
private long preloadTime;
public static final int crawlProfileHandleLength = 4; // name of the prefetch profile
- public plasmaCrawlProfile(File file, int bufferkb, long preloadTime) {
+ public plasmaCrawlProfile(File file, long preloadTime) {
this.profileTableFile = file;
- this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
profileTableFile.getParentFile().mkdirs();
- kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#', true, false);
+ kelondroDyn dyn = kelondroDyn.open(profileTableFile, true, true, preloadTime, crawlProfileHandleLength, 2000, '#', true, false);
profileTable = new kelondroMapObjects(dyn, 500);
domsCache = new HashMap();
}
- public int cacheNodeChunkSize() {
- return profileTable.cacheNodeChunkSize();
- }
-
- public int cacheObjectChunkSize() {
- return profileTable.cacheObjectChunkSize();
- }
-
- public int[] cacheNodeStatus() {
- return profileTable.cacheNodeStatus();
- }
-
- public long[] cacheObjectStatus() {
- return profileTable.cacheObjectStatus();
- }
-
private void resetDatabase() {
// deletes the profile database and creates a new one
if (profileTable != null) try { profileTable.close(); } catch (IOException e) {}
if (!(profileTableFile.delete())) throw new RuntimeException("cannot delete crawl profile database");
profileTableFile.getParentFile().mkdirs();
- kelondroDyn dyn = kelondroDyn.open(profileTableFile, bufferkb * 1024, preloadTime, crawlProfileHandleLength, 2000, '#', true, false);
+ kelondroDyn dyn = kelondroDyn.open(profileTableFile, true, true, preloadTime, crawlProfileHandleLength, 2000, '#', true, false);
profileTable = new kelondroMapObjects(dyn, 500);
}
diff --git a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java index 4e95efa85..54cd5a8a4 100644 --- a/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java +++ b/source/de/anomic/plasma/plasmaCrawlRobotsTxt.java @@ -65,31 +65,13 @@ public class plasmaCrawlRobotsTxt { kelondroMapObjects robotsTable;
private final File robotsTableFile;
- private int bufferkb;
private long preloadTime;
- public plasmaCrawlRobotsTxt(File robotsTableFile, int bufferkb, long preloadTime) {
+ public plasmaCrawlRobotsTxt(File robotsTableFile, long preloadTime) {
this.robotsTableFile = robotsTableFile;
- this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
robotsTableFile.getParentFile().mkdirs();
- robotsTable = new kelondroMapObjects(kelondroDyn.open(robotsTableFile, bufferkb * 1024, preloadTime, 256, 512, '_', true, false), 100);
- }
-
- public int cacheNodeChunkSize() {
- return robotsTable.cacheNodeChunkSize();
- }
-
- public int cacheObjectChunkSize() {
- return robotsTable.cacheObjectChunkSize();
- }
-
- public int[] cacheNodeStatus() {
- return robotsTable.cacheNodeStatus();
- }
-
- public long[] cacheObjectStatus() {
- return robotsTable.cacheObjectStatus();
+ robotsTable = new kelondroMapObjects(kelondroDyn.open(robotsTableFile, true, true, preloadTime, 256, 512, '_', true, false), 100);
}
private void resetDatabase() {
@@ -99,7 +81,7 @@ public class plasmaCrawlRobotsTxt { } catch (IOException e) {}
if (!(robotsTableFile.delete())) throw new RuntimeException("cannot delete robots.txt database");
robotsTableFile.getParentFile().mkdirs();
- robotsTable = new kelondroMapObjects(kelondroDyn.open(robotsTableFile, this.bufferkb, preloadTime, 256, 512, '_', true, false), 100);
+ robotsTable = new kelondroMapObjects(kelondroDyn.open(robotsTableFile, true, true, preloadTime, 256, 512, '_', true, false), 100);
}
public void close() {
diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index 0b48dc758..d5af2aaa6 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -92,10 +92,10 @@ public final class plasmaCrawlStacker { //private boolean stopped = false;
private stackCrawlQueue queue;
- public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, int dbCacheSize, long preloadTime, int dbtype) {
+ public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, long preloadTime, int dbtype) {
this.sb = sb;
- this.queue = new stackCrawlQueue(dbPath, dbCacheSize, preloadTime, dbtype);
+ this.queue = new stackCrawlQueue(dbPath, preloadTime, dbtype);
this.log.logInfo(this.queue.size() + " entries in the stackCrawl queue.");
this.log.logInfo("STACKCRAWL thread initialized.");
@@ -166,22 +166,6 @@ public final class plasmaCrawlStacker { public int size() {
return this.queue.size();
}
-
- public int cacheNodeChunkSize() {
- return this.queue.cacheNodeChunkSize();
- }
-
- public int cacheObjectChunkSize() {
- return this.queue.cacheObjectChunkSize();
- }
-
- public int[] cacheNodeStatus() {
- return this.queue.cacheNodeStatus();
- }
-
- public long[] cacheObjectStatus() {
- return this.queue.cacheObjectStatus();
- }
public void job() {
try {
@@ -614,11 +598,10 @@ public final class plasmaCrawlStacker { private final LinkedList urlEntryHashCache;
private kelondroIndex urlEntryCache;
private File cacheStacksPath;
- private int bufferkb;
private long preloadTime;
private int dbtype;
- public stackCrawlQueue(File cacheStacksPath, int bufferkb, long preloadTime, int dbtype) {
+ public stackCrawlQueue(File cacheStacksPath, long preloadTime, int dbtype) {
// init the read semaphore
this.readSync = new serverSemaphore (0);
@@ -630,7 +613,6 @@ public final class plasmaCrawlStacker { // create a stack for newly entered entries
this.cacheStacksPath = cacheStacksPath;
- this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
this.dbtype = dbtype;
@@ -691,7 +673,7 @@ public final class plasmaCrawlStacker { String newCacheName = "urlPreNotice2.table";
cacheStacksPath.mkdirs();
try {
- this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, bufferkb / 2 * 0x400, preloadTime, plasmaCrawlNURL.rowdef), bufferkb / 2 * 0x400, true, false);
+ this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlNURL.rowdef), true, false);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
@@ -701,29 +683,13 @@ public final class plasmaCrawlStacker { File cacheFile = new File(cacheStacksPath, "urlPreNotice.db");
cacheFile.getParentFile().mkdirs();
try {
- this.urlEntryCache = new kelondroCache(kelondroTree.open(cacheFile, bufferkb / 2 * 0x400, preloadTime, plasmaCrawlNURL.rowdef), bufferkb / 2 * 0x400, true, true);
+ this.urlEntryCache = new kelondroCache(kelondroTree.open(cacheFile, true, preloadTime, plasmaCrawlNURL.rowdef), true, true);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
}
}
}
-
- public int cacheNodeChunkSize() {
- return urlEntryCache.cacheNodeChunkSize();
- }
-
- public int[] cacheNodeStatus() {
- return urlEntryCache.cacheNodeStatus();
- }
-
- public int cacheObjectChunkSize() {
- return urlEntryCache.cacheObjectChunkSize();
- }
-
- public long[] cacheObjectStatus() {
- return urlEntryCache.cacheObjectStatus();
- }
public void close() throws IOException {
// closing the db
diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 4d96c61f7..c014a63c6 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -111,7 +111,7 @@ public final class plasmaHTCache { private ResourceInfoFactory objFactory;
private serverThread cacheScanThread;
- public plasmaHTCache(File htCachePath, long maxCacheSize, int bufferkb, long preloadTime, String cacheLayout, boolean cacheMigration) {
+ public plasmaHTCache(File htCachePath, long maxCacheSize, long preloadTime, String cacheLayout, boolean cacheMigration) {
// this.switchboard = switchboard;
this.log = new serverLog("HTCACHE");
@@ -174,7 +174,7 @@ public final class plasmaHTCache { // open the response header database
File dbfile = new File(this.cachePath, "responseHeader.db");
try {
- this.responseHeaderDB = new kelondroMapObjects(new kelondroDyn(dbfile, bufferkb * 0x400, preloadTime, yacySeedDB.commonHashLength, 150, '#', true, false), 500);
+ this.responseHeaderDB = new kelondroMapObjects(new kelondroDyn(dbfile, true, true, preloadTime, yacySeedDB.commonHashLength, 150, '#', true, false), 500);
} catch (IOException e) {
this.log.logSevere("the request header database could not be opened: " + e.getMessage());
System.exit(0);
@@ -218,22 +218,6 @@ public final class plasmaHTCache { public int dbSize() {
return this.responseHeaderDB.size();
}
-
- public int cacheNodeChunkSize() {
- return this.responseHeaderDB.cacheNodeChunkSize();
- }
-
- public int cacheObjectChunkSize() {
- return this.responseHeaderDB.cacheObjectChunkSize();
- }
-
- public int[] cacheNodeStatus() {
- return this.responseHeaderDB.cacheNodeStatus();
- }
-
- public long[] cacheObjectStatus() {
- return this.responseHeaderDB.cacheObjectStatus();
- }
public void push(Entry entry) {
synchronized (this.cacheStack) {
diff --git a/source/de/anomic/plasma/plasmaRankingCRProcess.java b/source/de/anomic/plasma/plasmaRankingCRProcess.java index 042401b54..ba86e57ff 100644 --- a/source/de/anomic/plasma/plasmaRankingCRProcess.java +++ b/source/de/anomic/plasma/plasmaRankingCRProcess.java @@ -261,8 +261,8 @@ public class plasmaRankingCRProcess { kelondroCollectionIndex newseq = null; if (newdb) { File path = to_file.getParentFile(); // path to storage place - newacc = new kelondroFlexTable(path, CRG_accname, 128 * 1024 * 1024, -1, CRG_accrow); - newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, 2, 9, CRG_colrow); + newacc = new kelondroFlexTable(path, CRG_accname, -1, CRG_accrow); + newseq = new kelondroCollectionIndex(path, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, -1, 2, 9, CRG_colrow); } else { if (!(to_file.exists())) { acc = new kelondroAttrSeq("Global Ranking Accumulator File", @@ -393,8 +393,8 @@ public class plasmaRankingCRProcess { public static int genrcix(File cr_path_in, File rci_path_out) throws IOException { //kelondroFlexTable acc = new kelondroFlexTable(cr_path_in, CRG_accname, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, CRG_accrow, true); - kelondroCollectionIndex seq = new kelondroCollectionIndex(cr_path_in, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, 2, 9, CRG_colrow); - kelondroCollectionIndex rci = new kelondroCollectionIndex(rci_path_out, RCI_colname, 6, kelondroBase64Order.enhancedCoder, 128 * 1024 * 1024, -1, 2, 9, RCI_coli); + kelondroCollectionIndex seq = new kelondroCollectionIndex(cr_path_in, CRG_seqname, 12, kelondroBase64Order.enhancedCoder, -1, 2, 9, CRG_colrow); + kelondroCollectionIndex rci = new kelondroCollectionIndex(rci_path_out, RCI_colname, 6, kelondroBase64Order.enhancedCoder, -1, 2, 9, RCI_coli); // loop over all referees int count = 0; diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index 2ae5b9a16..de88b3288 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -140,10 +140,12 @@ import de.anomic.plasma.plasmaURL; import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroBitfield;
+import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMSetTools;
import de.anomic.kelondro.kelondroMapTable;
import de.anomic.kelondro.kelondroNaturalOrder;
+import de.anomic.kelondro.kelondroRecords;
import de.anomic.net.URL;
import de.anomic.plasma.dbImport.dbImportManager;
import de.anomic.plasma.parser.ParserException;
@@ -153,7 +155,6 @@ import de.anomic.server.serverAbstractSwitch; import de.anomic.server.serverDate;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverInstantThread;
-import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSemaphore;
import de.anomic.server.serverSwitch;
@@ -411,67 +412,56 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser * <p><code>public static final String <strong>RAM_CACHE_LURL</strong> = "ramCacheLURL"</code></p>
* <p>Name of the setting how much memory in bytes should be assigned to the Loaded URLs DB for caching purposes</p>
*/
- public static final String RAM_CACHE_LURL = "ramCacheLURL";
public static final String RAM_CACHE_LURL_TIME = "ramCacheLURL_time";
/**
* <p><code>public static final String <strong>RAM_CACHE_NURL</strong> = "ramCacheNURL"</code></p>
* <p>Name of the setting how much memory in bytes should be assigned to the Noticed URLs DB for caching purposes</p>
*/
- public static final String RAM_CACHE_NURL = "ramCacheNURL";
public static final String RAM_CACHE_NURL_TIME = "ramCacheNURL_time";
/**
* <p><code>public static final String <strong>RAM_CACHE_EURL</strong> = "ramCacheEURL"</code></p>
* <p>Name of the setting how much memory in bytes should be assigned to the Erroneous URLs DB for caching purposes</p>
*/
- public static final String RAM_CACHE_EURL = "ramCacheEURL";
public static final String RAM_CACHE_EURL_TIME = "ramCacheEURL_time";
/**
* <p><code>public static final String <strong>RAM_CACHE_RWI</strong> = "ramCacheRWI"</code></p>
* <p>Name of the setting how much memory in bytes should be assigned to the RWIs DB for caching purposes</p>
*/
- public static final String RAM_CACHE_RWI = "ramCacheRWI";
public static final String RAM_CACHE_RWI_TIME = "ramCacheRWI_time";
/**
* <p><code>public static final String <strong>RAM_CACHE_HTTP</strong> = "ramCacheHTTP"</code></p>
* <p>Name of the setting how much memory in bytes should be assigned to the HTTP Headers DB for caching purposes</p>
*/
- public static final String RAM_CACHE_HTTP = "ramCacheHTTP";
public static final String RAM_CACHE_HTTP_TIME = "ramCacheHTTP_time";
/**
* <p><code>public static final String <strong>RAM_CACHE_MESSAGE</strong> = "ramCacheMessage"</code></p>
* <p>Name of the setting how much memory in bytes should be assigned to the Message DB for caching purposes</p>
*/
- public static final String RAM_CACHE_MESSAGE = "ramCacheMessage";
public static final String RAM_CACHE_MESSAGE_TIME = "ramCacheMessage_time";
/**
* <p><code>public static final String <strong>RAM_CACHE_ROBOTS</strong> = "ramCacheRobots"</code></p>
* <p>Name of the setting how much memory in bytes should be assigned to the robots.txts DB for caching purposes</p>
*/
- public static final String RAM_CACHE_ROBOTS = "ramCacheRobots";
public static final String RAM_CACHE_ROBOTS_TIME = "ramCacheRobots_time";
/**
* <p><code>public static final String <strong>RAM_CACHE_PROFILES</strong> = "ramCacheProfiles"</code></p>
* <p>Name of the setting how much memory in bytes should be assigned to the Crawl Profiles DB for caching purposes</p>
*/
- public static final String RAM_CACHE_PROFILES = "ramCacheProfiles";
public static final String RAM_CACHE_PROFILES_TIME = "ramCacheProfiles_time";
/**
* <p><code>public static final String <strong>RAM_CACHE_PRE_NURL</strong> = "ramCachePreNURL"</code></p>
* <p>Name of the setting how much memory in bytes should be assigned to the Pre-Noticed URLs DB for caching purposes</p>
*/
- public static final String RAM_CACHE_PRE_NURL = "ramCachePreNURL";
public static final String RAM_CACHE_PRE_NURL_TIME = "ramCachePreNURL_time";
/**
* <p><code>public static final String <strong>RAM_CACHE_WIKI</strong> = "ramCacheWiki"</code></p>
* <p>Name of the setting how much memory in bytes should be assigned to the Wiki DB for caching purposes</p>
*/
- public static final String RAM_CACHE_WIKI = "ramCacheWiki";
public static final String RAM_CACHE_WIKI_TIME = "ramCacheWiki_time";
/**
* <p><code>public static final String <strong>RAM_CACHE_BLOG</strong> = "ramCacheBlog"</code></p>
* <p>Name of the setting how much memory in bytes should be assigned to the Blog DB for caching purposes</p>
*/
- public static final String RAM_CACHE_BLOG = "ramCacheBlog";
public static final String RAM_CACHE_BLOG_TIME = "ramCacheBlog_time";
//////////////////////////////////////////////////////////////////////////////////////////////
@@ -952,54 +942,33 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser }
// read memory amount
- int ramLURL = (int) getConfigLong(RAM_CACHE_LURL, 1024) / 1024;
long ramLURL_time = getConfigLong(RAM_CACHE_LURL_TIME, 1000);
- ramLURL = Math.max((int) (serverMemory.available() / 2), ramLURL);
- setConfig("ramCacheLURL", ramLURL);
- int ramNURL = (int) getConfigLong(RAM_CACHE_NURL, 1024) / 1024;
long ramNURL_time = getConfigLong(RAM_CACHE_NURL_TIME, 1000);
- ramNURL = Math.max((int) (serverMemory.available() / 10 / 1024), ramNURL);
- setConfig("ramCacheNURL", ramNURL * 1024);
- int ramEURL = (int) getConfigLong(RAM_CACHE_EURL, 1024) / 1024;
long ramEURL_time = getConfigLong(RAM_CACHE_EURL_TIME, 1000);
- ramEURL = Math.max((int) (serverMemory.available() / 20 / 1024), ramEURL);
- setConfig("ramCacheEURL", ramEURL * 1024);
- int ramRWI = (int) getConfigLong(RAM_CACHE_RWI, 1024) / 1024;
long ramRWI_time = getConfigLong(RAM_CACHE_RWI_TIME, 1000);
- ramRWI = Math.max((int) (serverMemory.available() / 4), ramRWI);
- setConfig("ramCacheRWI", ramRWI);
- int ramHTTP = (int) getConfigLong(RAM_CACHE_HTTP, 1024) / 1024;
long ramHTTP_time = getConfigLong(RAM_CACHE_HTTP_TIME, 1000);
- int ramMessage = (int) getConfigLong(RAM_CACHE_MESSAGE, 1024) / 1024;
long ramMessage_time = getConfigLong(RAM_CACHE_MESSAGE_TIME, 1000);
- int ramRobots = (int) getConfigLong(RAM_CACHE_ROBOTS, 1024) / 1024;
long ramRobots_time = getConfigLong(RAM_CACHE_ROBOTS_TIME, 1000);
- int ramProfiles = (int) getConfigLong(RAM_CACHE_PROFILES, 1024) / 1024;
long ramProfiles_time= getConfigLong(RAM_CACHE_PROFILES_TIME, 1000);
- int ramPreNURL = (int) getConfigLong(RAM_CACHE_PRE_NURL, 1024) / 1024;
long ramPreNURL_time = getConfigLong(RAM_CACHE_PRE_NURL_TIME, 1000);
- ramPreNURL = Math.max((int) (serverMemory.available() / 10 / 1024), ramPreNURL);
- setConfig("ramCachePreNURL", ramPreNURL * 1024);
- int ramWiki = (int) getConfigLong(RAM_CACHE_WIKI, 1024) / 1024;
long ramWiki_time = getConfigLong(RAM_CACHE_WIKI_TIME, 1000);
- int ramBlog = (int) getConfigLong(RAM_CACHE_BLOG, 1024) / 1024;
long ramBlog_time = getConfigLong(RAM_CACHE_BLOG_TIME, 1000);
- this.log.logConfig("LURL Cache memory = " + ppRamString(ramLURL) + ", preloadTime = " + ramLURL_time);
- this.log.logConfig("NURL Cache memory = " + ppRamString(ramNURL) + ", preloadTime = " + ramNURL_time);
- this.log.logConfig("EURL Cache memory = " + ppRamString(ramEURL) + ", preloadTime = " + ramEURL_time);
- this.log.logConfig("RWI Cache memory = " + ppRamString(ramRWI) + ", preloadTime = " + ramRWI_time);
- this.log.logConfig("HTTP Cache memory = " + ppRamString(ramHTTP) + ", preloadTime = " + ramHTTP_time);
- this.log.logConfig("Message Cache memory = " + ppRamString(ramMessage) + ", preloadTime = " + ramMessage_time);
- this.log.logConfig("Wiki Cache memory = " + ppRamString(ramWiki) + ", preloadTime = " + ramWiki_time);
- this.log.logConfig("Blog Cache memory = " + ppRamString(ramBlog) + ", preloadTime = " + ramBlog_time);
- this.log.logConfig("Robots Cache memory = " + ppRamString(ramRobots) + ", preloadTime = " + ramRobots_time);
- this.log.logConfig("Profiles Cache memory = " + ppRamString(ramProfiles) + ", preloadTime = " + ramProfiles_time);
- this.log.logConfig("PreNURL Cache memory = " + ppRamString(ramPreNURL) + ", preloadTime = " + ramPreNURL_time);
+ this.log.logConfig("LURL preloadTime = " + ramLURL_time);
+ this.log.logConfig("NURL preloadTime = " + ramNURL_time);
+ this.log.logConfig("EURL preloadTime = " + ramEURL_time);
+ this.log.logConfig("RWI preloadTime = " + ramRWI_time);
+ this.log.logConfig("HTTP preloadTime = " + ramHTTP_time);
+ this.log.logConfig("Message preloadTime = " + ramMessage_time);
+ this.log.logConfig("Wiki preloadTime = " + ramWiki_time);
+ this.log.logConfig("Blog preloadTime = " + ramBlog_time);
+ this.log.logConfig("Robots preloadTime = " + ramRobots_time);
+ this.log.logConfig("Profiles preloadTime = " + ramProfiles_time);
+ this.log.logConfig("PreNURL preloadTime = " + ramPreNURL_time);
// make crawl profiles database and default profiles
this.log.logConfig("Initializing Crawl Profiles");
File profilesFile = new File(this.plasmaPath, DBFILE_CRAWL_PROFILES);
- this.profiles = new plasmaCrawlProfile(profilesFile, ramProfiles, ramProfiles_time);
+ this.profiles = new plasmaCrawlProfile(profilesFile, ramProfiles_time);
initProfiles();
log.logConfig("Loaded profiles from file " + profilesFile.getName() +
", " + this.profiles.size() + " entries" +
@@ -1008,16 +977,16 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // loading the robots.txt db
this.log.logConfig("Initializing robots.txt DB");
File robotsDBFile = new File(this.plasmaPath, DBFILE_CRAWL_ROBOTS);
- robots = new plasmaCrawlRobotsTxt(robotsDBFile, ramRobots, ramRobots_time);
+ robots = new plasmaCrawlRobotsTxt(robotsDBFile, ramRobots_time);
this.log.logConfig("Loaded robots.txt DB from file " + robotsDBFile.getName() +
", " + robots.size() + " entries" +
", " + ppRamString(robotsDBFile.length()/1024));
// start indexing management
log.logConfig("Starting Indexing Management");
- wordIndex = new plasmaWordIndex(indexPath, ramRWI, ramLURL, ramRWI_time, log);
- noticeURL = new plasmaCrawlNURL(plasmaPath, ramNURL, -1);
- errorURL = new plasmaCrawlEURL(plasmaPath, ramEURL, -1);
+ wordIndex = new plasmaWordIndex(indexPath, ramRWI_time, log);
+ noticeURL = new plasmaCrawlNURL(plasmaPath, -1);
+ errorURL = new plasmaCrawlEURL(plasmaPath, -1);
// set a high maximum cache size to current size; this is adopted later automatically
int wordCacheMaxCount = Math.max((int) getConfigLong(WORDCACHE_INIT_COUNT, 30000),
@@ -1030,7 +999,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser wordIndex.setWordFlushSize((int) getConfigLong("wordFlushSize", 1000));
// set a minimum amount of memory for the indexer thread
- setConfig(INDEXER_MEMPREREQ, Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem()));
+ long memprereq = Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem());
+ setConfig(INDEXER_MEMPREREQ, memprereq);
+ kelondroRecords.setCacheGrowStati(memprereq + (memprereq / 8) + 2 * 1024 * 1024, memprereq);
+ kelondroCache.setCacheGrowStati(memprereq + (memprereq / 8) + 2 * 1024 * 1024, memprereq);
// start a cache manager
log.logConfig("Starting HT Cache Manager");
@@ -1048,7 +1020,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser long maxCacheSize = 1024 * 1024 * Long.parseLong(getConfig(PROXY_CACHE_SIZE, "2")); // this is megabyte
String cacheLayout = getConfig(PROXY_CACHE_LAYOUT, PROXY_CACHE_LAYOUT_TREE);
boolean cacheMigration = getConfigBool(PROXY_CACHE_MIGRATION, true);
- this.cacheManager = new plasmaHTCache(htCachePath, maxCacheSize, ramHTTP, ramHTTP_time, cacheLayout, cacheMigration);
+ this.cacheManager = new plasmaHTCache(htCachePath, maxCacheSize, ramHTTP_time, cacheLayout, cacheMigration);
// make parser
log.logConfig("Starting Parser");
@@ -1117,18 +1089,18 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser Boolean.valueOf(getConfig(CRAWLJOB_GLOBAL_CRAWL_TRIGGER + "_isPaused", "false"))});
// starting board
- initMessages(ramMessage, ramMessage_time);
+ initMessages(ramMessage_time);
// starting wiki
- initWiki(ramWiki, ramWiki_time);
+ initWiki(ramWiki_time);
//starting blog
- initBlog(ramBlog, ramBlog_time);
+ initBlog(ramBlog_time);
// Init User DB
this.log.logConfig("Loading User DB");
File userDbFile = new File(getRootPath(), DBFILE_USER);
- this.userDB = new userDB(userDbFile, 512, 500);
+ this.userDB = new userDB(userDbFile, 2000);
this.log.logConfig("Loaded User DB from file " + userDbFile.getName() +
", " + this.userDB.size() + " entries" +
", " + ppRamString(userDbFile.length()/1024));
@@ -1218,7 +1190,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser serverInstantThread.oneTimeJob(yc, "loadSeeds", yacyCore.log, 3000);
// initializing the stackCrawlThread
- this.sbStackCrawlThread = new plasmaCrawlStacker(this, this.plasmaPath, ramPreNURL, ramPreNURL_time, (int) getConfigLong("tableTypeForPreNURL", 0));
+ this.sbStackCrawlThread = new plasmaCrawlStacker(this, this.plasmaPath, ramPreNURL_time, (int) getConfigLong("tableTypeForPreNURL", 0));
//this.sbStackCrawlThread = new plasmaStackCrawlThread(this,this.plasmaPath,ramPreNURL);
//this.sbStackCrawlThread.start();
@@ -1284,34 +1256,34 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser }
- public void initMessages(int ramMessage, long ramMessage_time) {
+ public void initMessages(long ramMessage_time) {
this.log.logConfig("Starting Message Board");
File messageDbFile = new File(workPath, DBFILE_MESSAGE);
- this.messageDB = new messageBoard(messageDbFile, ramMessage, ramMessage_time);
+ this.messageDB = new messageBoard(messageDbFile, ramMessage_time);
this.log.logConfig("Loaded Message Board DB from file " + messageDbFile.getName() +
", " + this.messageDB.size() + " entries" +
", " + ppRamString(messageDbFile.length()/1024));
}
- public void initWiki(int ramWiki, long ramWiki_time) {
+ public void initWiki(long ramWiki_time) {
this.log.logConfig("Starting Wiki Board");
File wikiDbFile = new File(workPath, DBFILE_WIKI);
- this.wikiDB = new wikiBoard(wikiDbFile, new File(workPath, DBFILE_WIKI_BKP), ramWiki, ramWiki_time);
+ this.wikiDB = new wikiBoard(wikiDbFile, new File(workPath, DBFILE_WIKI_BKP), ramWiki_time);
this.log.logConfig("Loaded Wiki Board DB from file " + wikiDbFile.getName() +
", " + this.wikiDB.size() + " entries" +
", " + ppRamString(wikiDbFile.length()/1024));
}
- public void initBlog(int ramBlog, long ramBlog_time) {
+ public void initBlog(long ramBlog_time) {
this.log.logConfig("Starting Blog");
File blogDbFile = new File(workPath, DBFILE_BLOG);
- this.blogDB = new blogBoard(blogDbFile, ramBlog, ramBlog_time);
+ this.blogDB = new blogBoard(blogDbFile, ramBlog_time);
this.log.logConfig("Loaded Blog DB from file " + blogDbFile.getName() +
", " + this.blogDB.size() + " entries" +
", " + ppRamString(blogDbFile.length()/1024));
File blogCommentDbFile = new File(workPath, DBFILE_BLOGCOMMENTS);
- this.blogCommentDB = new blogBoardComments(blogCommentDbFile, ramBlog, ramBlog_time);
+ this.blogCommentDB = new blogBoardComments(blogCommentDbFile, ramBlog_time);
this.log.logConfig("Loaded Blog-Comment DB from file " + blogCommentDbFile.getName() +
", " + this.blogCommentDB.size() + " entries" +
", " + ppRamString(blogCommentDbFile.length()/1024));
@@ -1321,7 +1293,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser File bookmarksFile = new File(workPath, DBFILE_BOOKMARKS);
File tagsFile = new File(workPath, DBFILE_BOOKMARKS_TAGS);
File datesFile = new File(workPath, DBFILE_BOOKMARKS_DATES);
- this.bookmarksDB = new bookmarksDB(bookmarksFile, tagsFile, datesFile, 512, 500);
+ this.bookmarksDB = new bookmarksDB(bookmarksFile, tagsFile, datesFile, 2000);
this.log.logConfig("Loaded Bookmarks DB from files "+ bookmarksFile.getName()+ ", "+tagsFile.getName());
this.log.logConfig(this.bookmarksDB.tagsSize()+" Tag, "+this.bookmarksDB.bookmarksSize()+" Bookmarks");
}
@@ -1430,9 +1402,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser private void resetProfiles() {
final File pdb = new File(plasmaPath, DBFILE_CRAWL_PROFILES);
if (pdb.exists()) pdb.delete();
- int ramProfiles = (int) getConfigLong(RAM_CACHE_PROFILES, 1024) / 1024;
long ramProfiles_time = getConfigLong(RAM_CACHE_PROFILES_TIME, 1000);
- profiles = new plasmaCrawlProfile(pdb, ramProfiles, ramProfiles_time);
+ profiles = new plasmaCrawlProfile(pdb, ramProfiles_time);
initProfiles();
}
@@ -1797,12 +1768,14 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser } catch (IOException e) {}
// set new memory limit for indexer thread
- long memprereq = Math.max(getConfigLong("80_indexing_memprereq", 0), wordIndex.minMem());
+ long memprereq = Math.max(getConfigLong(INDEXER_MEMPREREQ, 0), wordIndex.minMem());
setConfig(INDEXER_MEMPREREQ, memprereq);
setThreadPerformance(INDEXER,
getConfigLong(INDEXER_IDLESLEEP, 0),
getConfigLong(INDEXER_BUSYSLEEP, 0),
memprereq);
+ kelondroRecords.setCacheGrowStati(memprereq + (memprereq / 8) + 2 * 1024 * 1024, memprereq);
+ kelondroCache.setCacheGrowStati(memprereq + (memprereq / 8) + 2 * 1024 * 1024, memprereq);
return hasDoneSomething;
} catch (InterruptedException e) {
diff --git a/source/de/anomic/plasma/plasmaWordConnotation.java b/source/de/anomic/plasma/plasmaWordConnotation.java index 0b1555c8b..beb64ef83 100644 --- a/source/de/anomic/plasma/plasmaWordConnotation.java +++ b/source/de/anomic/plasma/plasmaWordConnotation.java @@ -59,8 +59,8 @@ public class plasmaWordConnotation { private static final int nodesize = 4048;
private kelondroDynTree refDB;
- public plasmaWordConnotation(File refDBfile, int bufferkb, long preloadTime, char fillChar) throws IOException {
- refDB = new kelondroDynTree(refDBfile, bufferkb * 0x400, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength, kelondroNaturalOrder.naturalOrder, 0), fillChar);
+ public plasmaWordConnotation(File refDBfile, long preloadTime, char fillChar) throws IOException {
+ refDB = new kelondroDynTree(refDBfile, preloadTime, wordlength, nodesize, new kelondroRow("byte[] word-" + wordlength + ", Cardinal count-" + countlength, kelondroNaturalOrder.naturalOrder, 0), fillChar);
}
private void addSingleRef(String word, String reference) throws IOException {
diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 906028272..b8e08639a 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -67,7 +67,7 @@ public final class plasmaWordIndex implements indexRI { private int flushsize;
public final plasmaCrawlLURL loadedURL;
- public plasmaWordIndex(File indexRoot, long rwibuffer, long lurlbuffer, long preloadTime, serverLog log) {
+ public plasmaWordIndex(File indexRoot, long preloadTime, serverLog log) {
File textindexcache = new File(indexRoot, "PUBLIC/TEXT/RICACHE");
if (!(textindexcache.exists())) textindexcache.mkdirs();
this.dhtOutCache = new indexRAMRI(textindexcache, indexRWIEntryNew.urlEntryRow, wCacheMaxChunk, wCacheMaxAge, "dump1.array", log);
@@ -76,10 +76,10 @@ public final class plasmaWordIndex implements indexRI { // create collections storage path
File textindexcollections = new File(indexRoot, "PUBLIC/TEXT/RICOLLECTION");
if (!(textindexcollections.exists())) textindexcollections.mkdirs();
- this.collections = new indexCollectionRI(textindexcollections, "collection", rwibuffer, preloadTime, maxCollectionPartition, indexRWIEntryNew.urlEntryRow);
+ this.collections = new indexCollectionRI(textindexcollections, "collection", preloadTime, maxCollectionPartition, indexRWIEntryNew.urlEntryRow);
// create LURL-db
- loadedURL = new plasmaCrawlLURL(indexRoot, lurlbuffer, preloadTime);
+ loadedURL = new plasmaCrawlLURL(indexRoot, preloadTime);
// performance settings
busyCacheFlush = false;
diff --git a/source/de/anomic/plasma/plasmaWordIndexAssortment.java b/source/de/anomic/plasma/plasmaWordIndexAssortment.java index 16ae84283..78ac109c9 100644 --- a/source/de/anomic/plasma/plasmaWordIndexAssortment.java +++ b/source/de/anomic/plasma/plasmaWordIndexAssortment.java @@ -76,7 +76,6 @@ public final class plasmaWordIndexAssortment { private File assortmentFile;
private serverLog log;
private kelondroTree assortment;
- private long bufferSize;
private static String intx(int x) {
String s = Integer.toString(x);
@@ -98,22 +97,19 @@ public final class plasmaWordIndexAssortment { return (rowsize - yacySeedDB.commonHashLength - 12) / indexRWIEntryOld.urlEntryRow.objectsize();
}
- public plasmaWordIndexAssortment(File storagePath, int assortmentLength, int bufferkb, long preloadTime, serverLog log) {
+ public plasmaWordIndexAssortment(File storagePath, int assortmentLength, long preloadTime, serverLog log) {
if (!(storagePath.exists())) storagePath.mkdirs();
this.assortmentFile = new File(storagePath, assortmentFileName + intx(assortmentLength) + ".db");
//this.bufferStructureLength = 3 + 2 * assortmentLength;
- this.bufferSize = bufferkb * 1024;
this.log = log;
// open assortment tree file
long start = System.currentTimeMillis();
- assortment = kelondroTree.open(assortmentFile, bufferSize / 2, preloadTime, bufferStructure(assortmentLength));
+ assortment = kelondroTree.open(assortmentFile, true, preloadTime, bufferStructure(assortmentLength));
long stop = System.currentTimeMillis();
if (log != null) log.logConfig("Opened Assortment, " +
assortment.size() + " entries, width " +
- assortmentLength + ", " + bufferkb + "kb buffer, " +
preloadTime + " ms preloadTime, " +
- (stop - start) + " ms effective, " +
- assortment.cacheNodeStatus()[1] + " preloaded");
+ (stop - start) + " ms effective");
}
diff --git a/source/de/anomic/plasma/plasmaWordIndexFile.java b/source/de/anomic/plasma/plasmaWordIndexFile.java index 89263bf93..1f81d5ca8 100644 --- a/source/de/anomic/plasma/plasmaWordIndexFile.java +++ b/source/de/anomic/plasma/plasmaWordIndexFile.java @@ -87,9 +87,7 @@ public final class plasmaWordIndexFile { theLocation = wordHash2path(databaseRoot, wordHash);
File fp = theLocation.getParentFile();
if (fp != null) fp.mkdirs();
- long cacheSize = theLocation.length();
- if (cacheSize > 1048576) cacheSize = 1048576;
- return kelondroTree.open(theLocation, cacheSize, 0,
+ return kelondroTree.open(theLocation, true, 0,
new kelondroRow("byte[] urlhash-" + yacySeedDB.commonHashLength + ", byte[] ba-" + (indexRWIEntryOld.urlEntryRow.objectsize() - yacySeedDB.commonHashLength),
kelondroBase64Order.enhancedCoder, 0));
}
@@ -191,7 +189,7 @@ public final class plasmaWordIndexFile { i = theIndex.contentRows(-1);
} catch (kelondroException e) {
e.printStackTrace();
- theIndex.file().delete();
+ new File(theIndex.filename()).delete();
i = null;
}
}
diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index 179a5eaa9..e788142a8 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -195,22 +195,18 @@ public class yacyCore { if (!yacyDBPath.exists()) { yacyDBPath.mkdir(); }
// create or init seed cache
- int memDHT = Integer.parseInt(switchboard.getConfig("ramCacheDHT", "1024")) / 1024;
long memDHT_time = Long.parseLong(switchboard.getConfig("ramCacheDHT_time", "1000"));
- log.logConfig("DHT Cache memory = " + memDHT + " KB");
seedDB = new yacySeedDB(
sb,
new File(yacyDBPath, "seed0.new.db"),
new File(yacyDBPath, "seed0.old.db"),
new File(yacyDBPath, "seed0.pot.db"),
- memDHT, memDHT_time);
+ memDHT_time);
// create or init news database
- int memNews = Integer.parseInt(switchboard.getConfig("ramCacheNews", "1024")) / 1024;
long memNews_time = Long.parseLong(switchboard.getConfig("ramCacheNews_time", "1000"));
- log.logConfig("News Cache memory = " + memNews + " KB");
try {
- newsPool = new yacyNewsPool(yacyDBPath, memNews, memNews_time);
+ newsPool = new yacyNewsPool(yacyDBPath, memNews_time);
} catch (IOException e) {
e.printStackTrace();
System.exit(-1);
diff --git a/source/de/anomic/yacy/yacyNewsDB.java b/source/de/anomic/yacy/yacyNewsDB.java index b9894e1e9..9c3cc0505 100644 --- a/source/de/anomic/yacy/yacyNewsDB.java +++ b/source/de/anomic/yacy/yacyNewsDB.java @@ -61,37 +61,19 @@ import de.anomic.server.serverDate; public class yacyNewsDB {
private File path;
- private int bufferkb;
private long preloadTime;
protected kelondroIndex news;
- public yacyNewsDB(File path, int bufferkb, long preloadTime) throws IOException {
+ public yacyNewsDB(File path, long preloadTime) throws IOException {
this.path = path;
- this.bufferkb = bufferkb;
this.preloadTime = preloadTime;
- this.news = new kelondroCache(kelondroTree.open(path, bufferkb / 2 * 0x400, preloadTime, yacyNewsRecord.rowdef), bufferkb / 2 * 0x400, true, false);
+ this.news = new kelondroCache(kelondroTree.open(path, true, preloadTime, yacyNewsRecord.rowdef), true, false);
}
private void resetDB() throws IOException {
try {close();} catch (Exception e) {}
if (path.exists()) path.delete();
- this.news = new kelondroCache(kelondroTree.open(path, bufferkb / 2 * 0x400, preloadTime, yacyNewsRecord.rowdef), bufferkb / 2 * 0x400, true, false);
- }
-
- public int cacheNodeChunkSize() {
- return news.cacheNodeChunkSize();
- }
-
- public int cacheObjectChunkSize() {
- return news.cacheObjectChunkSize();
- }
-
- public int[] cacheNodeStatus() {
- return news.cacheNodeStatus();
- }
-
- public long[] cacheObjectStatus() {
- return news.cacheObjectStatus();
+ this.news = new kelondroCache(kelondroTree.open(path, true, preloadTime, yacyNewsRecord.rowdef), true, false);
}
public void close() {
diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index 56bc2a389..abce9b5fa 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -91,8 +91,8 @@ public class yacyNewsPool { private int maxDistribution;
- public yacyNewsPool(File yacyDBPath, int bufferkb, long preloadTime) throws IOException {
- newsDB = new yacyNewsDB(new File(yacyDBPath, "news1.db"), bufferkb, preloadTime);
+ public yacyNewsPool(File yacyDBPath, long preloadTime) throws IOException {
+ newsDB = new yacyNewsDB(new File(yacyDBPath, "news1.db"), preloadTime);
outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut1.stack"), newsDB);
publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished1.stack"), newsDB);
incomingNews = new yacyNewsQueue(new File(yacyDBPath, "newsIn1.stack"), newsDB);
@@ -109,22 +109,6 @@ public class yacyNewsPool { }
}
- public int cacheNodeChunkSize() {
- return newsDB.cacheNodeChunkSize();
- }
-
- public int cacheObjectChunkSize() {
- return newsDB.cacheObjectChunkSize();
- }
-
- public int[] cacheNodeStatus() {
- return newsDB.cacheNodeStatus();
- }
-
- public long[] cacheObjectStatus() {
- return newsDB.cacheObjectStatus();
- }
-
public void publishMyNews(yacyNewsRecord record) {
// this shall be called if our peer generated a new news record and wants to publish it
try {
diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index a38b8f7d6..2103d5b1a 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -61,12 +61,10 @@ import java.util.Map; import de.anomic.http.httpHeader;
import de.anomic.http.httpc;
import de.anomic.http.httpd;
-import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects;
-import de.anomic.kelondro.kelondroRecords;
import de.anomic.net.URL;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
@@ -96,7 +94,6 @@ public final class yacySeedDB { protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
protected kelondroMapObjects seedActiveDB, seedPassiveDB, seedPotentialDB;
- private int seedDBBufferKB;
private long preloadTime;
public final plasmaSwitchboard sb;
@@ -109,9 +106,8 @@ public final class yacySeedDB { File seedActiveDBFile,
File seedPassiveDBFile,
File seedPotentialDBFile,
- int bufferkb, long preloadTime) {
+ long preloadTime) {
- this.seedDBBufferKB = bufferkb;
this.seedActiveDBFile = seedActiveDBFile;
this.seedPassiveDBFile = seedPassiveDBFile;
this.seedPotentialDBFile = seedPotentialDBFile;
@@ -176,35 +172,6 @@ public final class yacySeedDB { } catch (IOException e) {}
}
- public int cacheNodeChunkSize() {
- int ac = seedActiveDB.cacheNodeChunkSize();
- int pa = seedPassiveDB.cacheNodeChunkSize();
- int po = seedPotentialDB.cacheNodeChunkSize();
-
- return (ac+ pa + po) / 3;
- }
- public int cacheObjectChunkSize() {
- int ac = seedActiveDB.cacheObjectChunkSize();
- int pa = seedPassiveDB.cacheObjectChunkSize();
- int po = seedPotentialDB.cacheObjectChunkSize();
-
- return (ac+ pa + po) / 3;
- }
-
- public int[] cacheNodeStatus() {
- int[] ac = seedActiveDB.cacheNodeStatus();
- int[] pa = seedPassiveDB.cacheNodeStatus();
- int[] po = seedPotentialDB.cacheNodeStatus();
- return kelondroRecords.cacheCombinedStatus(new int[][]{ac, pa, po}, 3);
- }
-
- public long[] cacheObjectStatus() {
- return kelondroCache.combinedStatus(new long[][] {
- seedActiveDB.cacheObjectStatus(),
- seedPassiveDB.cacheObjectStatus(),
- seedPotentialDB.cacheObjectStatus() }, 3);
- }
-
private synchronized kelondroMapObjects openSeedTable(File seedDBFile) {
new File(seedDBFile.getParent()).mkdirs();
Class[] args;
@@ -225,11 +192,11 @@ public final class yacySeedDB { initializeHandlerMethod = null;
}
try {
- return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
+ return new kelondroMapObjects(kelondroDyn.open(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
} catch (Exception e) {
seedDBFile.delete();
// try again
- return new kelondroMapObjects(kelondroDyn.open(seedDBFile, (seedDBBufferKB * 0x400) / 3, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
+ return new kelondroMapObjects(kelondroDyn.open(seedDBFile, true, true, preloadTime / 3, commonHashLength, 480, '#', false, false), 500, sortFields, longaccFields, doubleaccFields, initializeHandlerMethod, this);
}
}
diff --git a/source/migration.java b/source/migration.java index aaedb5e67..b40c4297e 100644 --- a/source/migration.java +++ b/source/migration.java @@ -163,7 +163,7 @@ public class migration { file.delete(); } catch (IOException e) {} } - sb.initWiki((int) sb.getConfigLong("ramCacheWiki", 1024) / 1024, sb.getConfigLong("ramCacheWiki_time", 1000)); + sb.initWiki(sb.getConfigLong("ramCacheWiki_time", 1000)); } @@ -176,7 +176,7 @@ public class migration { serverFileUtils.copy(file, file2); file.delete(); } catch (IOException e) {} - sb.initMessages((int) sb.getConfigLong("ramCacheMessage", 1024) / 1024, sb.getConfigLong("ramCacheMessage_time", 1000)); + sb.initMessages(sb.getConfigLong("ramCacheMessage_time", 1000)); } } diff --git a/source/yacy.java b/source/yacy.java index 533800947..10ea41dbc 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -635,7 +635,7 @@ public final class yacy { File indexRoot = new File(new File(homePath), "DATA/INDEX");
serverLog log = new serverLog("WORDMIGRATION");
log.logInfo("STARTING MIGRATION");
- plasmaWordIndex wordIndexCache = new plasmaWordIndex(indexRoot, 60000000, 60000000, 10000, log);
+ plasmaWordIndex wordIndexCache = new plasmaWordIndex(indexRoot, 10000, log);
enumerateFiles words = new enumerateFiles(new File(dbroot, "WORDS"), true, false, true, true);
String wordhash;
File wordfile;
@@ -713,7 +713,7 @@ public final class yacy { serverLog log = new serverLog("ASSORTMENTMIGRATION");
File aclusterroot = new File(new File(homePath), "DATA/PLASMADB/ACLUSTER");
File indexRoot = new File(new File(homePath), "DATA/INDEX");
- plasmaWordIndex wordIndexCache = new plasmaWordIndex(indexRoot, 60000000, 60000000, 10000, log);
+ plasmaWordIndex wordIndexCache = new plasmaWordIndex(indexRoot, 10000, log);
log.logInfo("STARTING MIGRATION");
String[] a = aclusterroot.list();
AssortmentImporter importer = new AssortmentImporter(wordIndexCache);
@@ -732,7 +732,7 @@ public final class yacy { * @param homePath path to the YaCy directory
* @param dbcache cache size in MB
*/
- public static void minimizeUrlDB(String homePath, int dbcache) {
+ public static void minimizeUrlDB(String homePath) {
// run with "java -classpath classes yacy -minimizeUrlDB"
try {serverLog.configureLogging(new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {}
File indexRoot = new File(new File(homePath), "DATA/INDEX");
@@ -742,18 +742,16 @@ public final class yacy { log.logInfo("STARTING URL CLEANUP");
// db containing all currently loades urls
- int cache = dbcache * 1024; // in KB
- log.logFine("URLDB-Caches: "+cache+" bytes");
- plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexRoot, cache, 10000);
+ plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexRoot, 10000);
// db used to hold all neede urls
- plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(indexRoot2, cache, 10000);
+ plasmaCrawlLURL minimizedUrlDB = new plasmaCrawlLURL(indexRoot2, 10000);
Runtime rt = Runtime.getRuntime();
int cacheMem = (int)(serverMemory.max-rt.totalMemory());
if (cacheMem < 2048000) throw new OutOfMemoryError("Not enough memory available to start clean up.");
- plasmaWordIndex wordIndex = new plasmaWordIndex(indexRoot, cacheMem, cacheMem, 10000, log);
+ plasmaWordIndex wordIndex = new plasmaWordIndex(indexRoot, 10000, log);
Iterator indexContainerIterator = wordIndex.wordContainers("AAAAAAAAAAAA", false, false);
long urlCounter = 0, wordCounter = 0;
@@ -1163,7 +1161,7 @@ public final class yacy { final plasmaSwitchboard sb = new plasmaSwitchboard(homePath, "yacy.init", "DATA/SETTINGS/httpProxy.conf");
kelondroTree oldindex = null;
try {
- oldindex = new kelondroTree(urlHash, 1000, -1, indexURLEntryOld.rowdef);
+ oldindex = new kelondroTree(urlHash, true, -1, indexURLEntryOld.rowdef);
} catch (IOException e) {
System.out.println("ERROR: CANNOT OPEN OLD INDEX: " + e.getMessage());
}
@@ -1239,7 +1237,7 @@ public final class yacy { serverLog log = new serverLog("URLDBCLEANUP");
try {serverLog.configureLogging(new File(homePath, "DATA/LOG/yacy.logging"));} catch (Exception e) {}
try {
- plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexroot, 4194304, 10000);
+ plasmaCrawlLURL currentUrlDB = new plasmaCrawlLURL(indexroot, 10000);
currentUrlDB.urldbcleanup();
currentUrlDB.close();
} catch (IOException e) {
@@ -1259,11 +1257,11 @@ public final class yacy { try {
Iterator indexContainerIterator = null;
if (resource.equals("all")) {
- WordIndex = new plasmaWordIndex(indexRoot, 8*1024*1024, 8*1024*1024, 3000, log);
+ WordIndex = new plasmaWordIndex(indexRoot, 3000, log);
indexContainerIterator = WordIndex.wordContainers(wordChunkStartHash, false, false);
} else if (resource.startsWith("assortment")) {
int a = Integer.parseInt(resource.substring(10));
- plasmaWordIndexAssortment assortment = new plasmaWordIndexAssortment(new File(homeDBroot, "ACLUSTER"), a, 8*1024*1024, 3000, null);
+ plasmaWordIndexAssortment assortment = new plasmaWordIndexAssortment(new File(homeDBroot, "ACLUSTER"), a, 3000, null);
indexContainerIterator = assortment.wordContainers();
} else if (resource.equals("words")) {
plasmaWordIndexFileCluster fileDB = new plasmaWordIndexFileCluster(homeDBroot);
@@ -1327,7 +1325,7 @@ public final class yacy { String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
File dbFile = new File(yacyDBPath,dbFileNames[i]);
- kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, (1024 * 0x400) / 3, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null);
+ kelondroMapObjects db = new kelondroMapObjects(new kelondroDyn(dbFile, true, true, 3000, yacySeedDB.commonHashLength, 480, '#', true, false), 500, yacySeedDB.sortFields, yacySeedDB.longaccFields, yacySeedDB.doubleaccFields, null, null);
kelondroMapObjects.mapIterator it;
it = db.maps(true, false);
@@ -1406,13 +1404,11 @@ public final class yacy { } else if ((args.length >= 1) && (args[0].toLowerCase().equals("-minimizeurldb"))) {
// migrate words from DATA/PLASMADB/WORDS path to assortment cache, if possible
// attention: this may run long and should not be interrupted!
- int dbcache = 4;
if (args.length >= 3 && args[1].toLowerCase().equals("-cache")) {
- dbcache = Integer.parseInt(args[2]);
args = shift(args, 1, 2);
}
if (args.length == 2) applicationRoot= args[1];
- minimizeUrlDB(applicationRoot, dbcache);
+ minimizeUrlDB(applicationRoot);
} else if ((args.length >= 1) && (args[0].toLowerCase().equals("-testpeerdb"))) {
if (args.length == 2) {
applicationRoot= args[1];
@@ -516,55 +516,42 @@ xpstopw=true # ram cache for database files
# ram cache for assortment cache cluster (for all 64 files)
-ramCacheRWI = 4194304
ramCacheRWI_time = 30000
# ram cache for responseHeader.db
-ramCacheHTTP = 1048576
ramCacheHTTP_time = 1000
# ram cache for urlHash.db
-ramCacheLURL = 4194304
ramCacheLURL_time = 10000
# ram cache for stack crawl thread db
-ramCachePreNURL = 1048576
ramCachePreNURL_time = 3000
# ram cache for urlNotice.db
-ramCacheNURL = 1048576
ramCacheNURL_time = 2000
# ram cache for urlErr.db
-ramCacheEURL = 8192
ramCacheEURL_time = 1000
# ram cache for seedDBs
-ramCacheDHT = 131072
ramCacheDHT_time = 1000
# ram cache for message.db
-ramCacheMessage = 8192
ramCacheMessage_time = 500
# ram cache for wiki.db
-ramCacheWiki = 8192
ramCacheWiki_time = 500
# ram cache for blog.db
-ramCacheBlog = 2048
ramCacheBlog_time = 500
# ram cache for news1.db
-ramCacheNews = 1048576
ramCacheNews_time = 1000
# ram cache for robotsTxt.db
-ramCacheRobots = 2097152
-ramCacheRobots_time = 3000
+ramCacheRobots_time = 0
# ram cache for crawlProfile.db
-ramCacheProfiles = 8192
ramCacheProfiles_time= 500
# default memory settings for startup of yacy
|