diff options
| -rw-r--r-- | htroot/Performance_p.html | 62 | ||||
| -rw-r--r-- | htroot/Performance_p.java | 23 | ||||
| -rw-r--r-- | source/de/anomic/http/httpdSwitchboard.java | 10 | ||||
| -rw-r--r-- | source/de/anomic/kelondro/kelondroBufferedRA.java | 31 | ||||
| -rw-r--r-- | source/de/anomic/kelondro/kelondroRecords.java | 28 | ||||
| -rw-r--r-- | source/de/anomic/plasma/plasmaHTCache.java | 2 | ||||
| -rw-r--r-- | source/de/anomic/plasma/plasmaSwitchboard.java | 50 | ||||
| -rw-r--r-- | source/de/anomic/plasma/plasmaWordIndex.java | 14 | ||||
| -rw-r--r-- | source/de/anomic/plasma/plasmaWordIndexRAMCache.java | 24 | ||||
| -rw-r--r-- | source/de/anomic/server/serverAbstractSwitch.java | 2 | ||||
| -rw-r--r-- | source/de/anomic/server/serverSwitch.java | 2 | ||||
| -rw-r--r-- | yacy.init | 52 |
12 files changed, 196 insertions, 104 deletions
diff --git a/htroot/Performance_p.html b/htroot/Performance_p.html index b63860fe8..0b565dad5 100644 --- a/htroot/Performance_p.html +++ b/htroot/Performance_p.html @@ -9,7 +9,7 @@ <br><br>
<h2>Performance</h2>
<p>
-You can manipulate some performance attributes here.<br>
+<div class=small><b>Scheduled tasks overview and waiting time settings:</b></div>
<form action="Performance_p.html" method="post" enctype="multipart/form-data">
<table border="0" cellpadding="2" cellspacing="1">
<tr class="TableHeader" valign="bottom">
@@ -47,10 +47,66 @@ You can manipulate some performance attributes here.<br> <td class="small" align="left">#[longdescr]#</td>
</tr>
#{/table}#
-</table><br>
+<tr class="TableCellLight">
+<td class="small" align="left" colspan="16">
<input type="submit" name="delaysubmit" value="Submit New Delay Values">
-Changes take effect immidiately
+Changes take effect immidiately</td>
+</tr>
+</table>
</p></form>
+
+<p>
+<div class=small><b>Indexing Cache Settings:</b></div>
+<table border="0" cellpadding="5" cellspacing="0" width="100%">
+<form action="Performance_p.html" method="post" enctype="multipart/form-data">
+ <tr valign="top" class="TableCellDark">
+ <td width="120"></td>
+ <td></td>
+ <td></td>
+ </tr>
+ <tr valign="top" class="TableCellDark">
+ <td class=small>Words in RAM Cache:</td>
+ <td class=small>#[wordCacheRAMSize]#</td>
+ <td class=small>
+ This is the current size of the word cache.
+ The smaller this number, the faster will be the shut-down procedure.
+ The maximum of this cache can be set below.
+ </td>
+ </tr>
+ <tr valign="top" class="TableCellDark">
+ <td class=small>Maximum URLs currently assigned<br>to one cached word:</td>
+ <td class=small>#[maxURLinWordCache]#</td>
+ <td class=small>
+ This is the maximum size of URLs assigned to a single word cache entry.
+ If this is a big number, it shows that the caching works efficiently.
+ </td>
+ </tr>
+ <tr valign="top" class="TableCellDark">
+ <td class=small>Maximum number of Word Caches:</td>
+ <td class=small><input name="wordCacheMax" type="text" size="20" maxlength="100" value="#[wordCacheMax]#"></td>
+ <td class=small>
+ This is is the number of word indexes that shall be held in the
+ ram cache during indexing. When YaCy is shut down, this cache must be
+ flushed to disc; this may last some minutes.
+ </td>
+ </tr>
+ <tr valign="top" class="TableCellDark">
+ <td class=small>Maximum waitingtime to flush word cache<br>during shut-down (seconds):</td>
+ <td class=small><input name="maxWaitingWordFlush" type="text" size="6" maxlength="6" value="#[maxWaitingWordFlush]#"></td>
+ <td class=small>
+ This gives the number of seconds that the shutdown
+ may last for the final word flush. Words that cannot be flushed within this
+ time are lost.
+ </td>
+ </tr>
+ <tr valign="top" class="TableCellLight">
+ <td class=small colspan="3"><input type="submit" name="cacheSizeSubmit" value="Enter New Cache Size">
+ Changes take effect immidiately</td>
+ </td>
+ </tr>
+</form>
+</table>
+</p>
#[footer]#
</body>
</html>
diff --git a/htroot/Performance_p.java b/htroot/Performance_p.java index 9ca8d1552..f79777696 100644 --- a/htroot/Performance_p.java +++ b/htroot/Performance_p.java @@ -109,11 +109,7 @@ public class Performance_p { prop.put("table_" + c + "_sleeppercycle", ((idleCycles + busyCycles) == 0) ? "-" : ("" + (sleeptime / (idleCycles + busyCycles))));
prop.put("table_" + c + "_execpercycle", (busyCycles == 0) ? "-" : ("" + (exectime / busyCycles)));
- if (post == null) {
- // load with old values
- idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000"));
- busysleep = Long.parseLong(switchboard.getConfig(threadName + "_busysleep", "1000"));
- } else {
+ if ((post != null) && (post.containsKey("delaysubmit"))) {
// load with new values
idlesleep = Long.parseLong((String) post.get(threadName + "_idlesleep", "1")) * 1000;
busysleep = Long.parseLong((String) post.get(threadName + "_busysleep", "1")) * 1000;
@@ -122,6 +118,10 @@ public class Performance_p { switchboard.setThreadSleep(threadName, idlesleep, busysleep);
switchboard.setConfig(threadName + "_idlesleep", idlesleep);
switchboard.setConfig(threadName + "_busysleep", busysleep);
+ } else {
+ // load with old values
+ idlesleep = Long.parseLong(switchboard.getConfig(threadName + "_idlesleep" , "1000"));
+ busysleep = Long.parseLong(switchboard.getConfig(threadName + "_busysleep", "1000"));
}
prop.put("table_" + c + "_idlesleep", idlesleep / 1000);
prop.put("table_" + c + "_busysleep", busysleep / 1000);
@@ -130,6 +130,19 @@ public class Performance_p { }
prop.put("table", c);
+ if ((post != null) && (post.containsKey("cacheSizeSubmit"))) {
+ int wordCacheMax = Integer.parseInt((String) post.get("wordCacheMax", "10000"));
+ switchboard.setConfig("wordCacheMax", "" + wordCacheMax);
+ switchboard.wordIndex.setMaxWords(wordCacheMax);
+ int maxWaitingWordFlush = Integer.parseInt((String) post.get("maxWaitingWordFlush", "180"));
+ switchboard.setConfig("maxWaitingWordFlush", "" + maxWaitingWordFlush);
+ }
+ // table cache settings
+ prop.put("wordCacheRAMSize", switchboard.wordIndex.wordCacheRAMSize());
+ prop.put("maxURLinWordCache", "" + switchboard.wordIndex.maxURLinWordCache());
+ prop.put("maxWaitingWordFlush", switchboard.getConfig("maxWaitingWordFlush", "180"));
+ prop.put("wordCacheMax", switchboard.getConfig("wordCacheMax", "10000"));
+
// return rewrite values for templates
return prop;
}
diff --git a/source/de/anomic/http/httpdSwitchboard.java b/source/de/anomic/http/httpdSwitchboard.java index 55f20cae9..1e9396431 100644 --- a/source/de/anomic/http/httpdSwitchboard.java +++ b/source/de/anomic/http/httpdSwitchboard.java @@ -61,8 +61,14 @@ public final class httpdSwitchboard extends serverAbstractSwitch implements serv cacheStack.addLast(job);
}
- public void deQueue() {
- System.out.println("Process: " + cacheStack.removeFirst().toString());
+ public boolean deQueue() {
+ if (cacheStack.size() > 0) {
+ System.out.println("Process: " + cacheStack.removeFirst().toString());
+ return true;
+ } else {
+ System.out.println("Process: queue is empty");
+ return false;
+ }
}
public serverObjects action(String actionName, serverObjects actionInput) {
diff --git a/source/de/anomic/kelondro/kelondroBufferedRA.java b/source/de/anomic/kelondro/kelondroBufferedRA.java index 83439e0c7..42f19b093 100644 --- a/source/de/anomic/kelondro/kelondroBufferedRA.java +++ b/source/de/anomic/kelondro/kelondroBufferedRA.java @@ -79,16 +79,10 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA // delete elements in buffer if buffer too big
Iterator it = bufferScore.scores(true);
Integer element = (Integer) it.next();
+ writeBuffer((byte[]) bufferMemory.get(element), element.intValue());
bufferMemory.remove(element);
int age = bufferScore.deleteScore(element);
- int minclean = bufferMaxElements / 8;
- int cleaned = 1;
- while ((cleaned++ < minclean) && (it.hasNext())) {
- element = (Integer) it.next();
- bufferMemory.remove(element);
- age = bufferScore.deleteScore(element);
- }
- de.anomic.server.serverLog.logDebug("CACHE: " + name, "GC; cleaned=" + (cleaned - 1) + ", age=" + ((((int) (0xFFFFFFFFL & System.currentTimeMillis())) - age) / 1000));
+ de.anomic.server.serverLog.logDebug("CACHE: " + name, "GC; age=" + ((((int) (0xFFFFFFFFL & System.currentTimeMillis())) - age) / 1000));
}
// add new element
buffer = new byte[bufferElementSize];
@@ -101,14 +95,6 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA return buffer;
}
- /*
- private static int log2i(int x) {
- int log = 0;
- while (x != 0) {x >>= 1; log++;}
- return log;
- }
- */
-
private void writeBuffer(byte[] buffer, int bufferNr) throws IOException {
if (buffer == null) return;
Integer bufferNrI = new Integer(bufferNr);
@@ -132,7 +118,7 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA byte[] buffer = readBuffer(bn);
seekpos++;
buffer[offset] = (byte) b;
- writeBuffer(buffer, bn);
+ //writeBuffer(buffer, bn);
}
public int read(byte[] b, int off, int len) throws IOException {
@@ -165,13 +151,13 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA // simple case
System.arraycopy(b, off, buffer, offset, len);
seekpos += len;
- writeBuffer(buffer, bn1);
+ //writeBuffer(buffer, bn1);
} else {
// do recursively
int thislen = bufferElementSize - offset;
System.arraycopy(b, off, buffer, offset, thislen);
seekpos += thislen;
- writeBuffer(buffer, bn1);
+ //writeBuffer(buffer, bn1);
write(b, thislen, len - thislen);
}
}
@@ -181,6 +167,13 @@ public class kelondroBufferedRA extends kelondroAbstractRA implements kelondroRA }
public void close() throws IOException {
+ // write all unwritten buffers
+ Iterator it = bufferScore.scores(true);
+ while (it.hasNext()) {
+ Integer element = (Integer) it.next();
+ writeBuffer((byte[]) bufferMemory.get(element), element.intValue());
+ bufferMemory.remove(element);
+ }
ra.close();
bufferScore = null;
bufferMemory = null;
diff --git a/source/de/anomic/kelondro/kelondroRecords.java b/source/de/anomic/kelondro/kelondroRecords.java index aff2261a4..2070b16c3 100644 --- a/source/de/anomic/kelondro/kelondroRecords.java +++ b/source/de/anomic/kelondro/kelondroRecords.java @@ -142,6 +142,7 @@ public class kelondroRecords { throw new IOException("kelondroRecords: tree file " + file + " already exist");
this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
+ //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, false, 10000);
init(raf, ohbytec, ohhandlec, columns, FHandles, txtProps, txtPropWidth);
this.cachesize = (int) (buffersize / ((long) (overhead + recordsize)));
@@ -240,6 +241,7 @@ public class kelondroRecords { this.filename = file.getCanonicalPath();
kelondroRA raf = new kelondroFileRA(this.filename);
+ //kelondroRA raf = new kelondroBufferedRA(new kelondroFileRA(this.filename), 5000000, 1000);
//kelondroRA raf = new kelondroNIOFileRA(this.filename, (file.length() < 4000000), 10000);
init(raf);
this.cachesize = (int) (buffersize / ((long) (overhead + recordsize)));
@@ -353,32 +355,6 @@ public class kelondroRecords { // we simply clear the cache
this.cacheScore = new kelondroMScoreCluster();
this.cache = new HashMap();
-
- /*
- java.util.NoSuchElementException
- at java.util.TreeMap.key(TreeMap.java:431)
- at java.util.TreeMap.firstKey(TreeMap.java:286)
- at de.anomic.kelondro.kelondroMScoreCluster$scoreIterator.internalNext(kelondroMScoreCluster.java:235)
- at de.anomic.kelondro.kelondroMScoreCluster$scoreIterator.<init>(kelondroMScoreCluster.java:224)
- at de.anomic.kelondro.kelondroMScoreCluster.scores(kelondroMScoreCluster.java:209)
- at de.anomic.kelondro.kelondroMScoreCluster.getScores(kelondroMScoreCluster.java:191)
- at de.anomic.kelondro.kelondroMScoreCluster.getScores(kelondroMScoreCluster.java:185)
- at de.anomic.kelondro.kelondroMScoreCluster.getMinObject(kelondroMScoreCluster.java:181)
- at de.anomic.kelondro.kelondroRecords.checkCacheSpace(kelondroRecords.java:344)
- at de.anomic.kelondro.kelondroRecords.access$1500(kelondroRecords.java:74)
- at de.anomic.kelondro.kelondroRecords$Node.updateNode(kelondroRecords.java:624)
- at de.anomic.kelondro.kelondroRecords$Node.<init>(kelondroRecords.java:399)
- at de.anomic.kelondro.kelondroRecords$Node.<init>(kelondroRecords.java:350)
- at de.anomic.kelondro.kelondroRecords.getNode(kelondroRecords.java:321)
- at de.anomic.kelondro.kelondroTree$Search.searchproc(kelondroTree.java:265)
- at de.anomic.kelondro.kelondroTree$Search.<init>(kelondroTree.java:246)
- at de.anomic.kelondro.kelondroTree.get(kelondroTree.java:317)
- at de.anomic.plasma.plasmaURL.exists(plasmaURL.java:106)
- at de.anomic.plasma.plasmaSwitchboard.stackCrawl(plasmaSwitchboard.java:389)
- at de.anomic.plasma.plasmaSwitchboard.processResourceStack(plasmaSwitchboard.java:304)
- at de.anomic.plasma.plasmaSwitchboard.deQueue(plasmaSwitchboard.java:262)
- at yacyProxy.main(yacyProxy.java:228)
- */
}
}
}
diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 57c5ec71b..65dba7449 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -257,7 +257,7 @@ public final class plasmaHTCache { entry.cacheFile.getParentFile().mkdirs();
log.logInfo("WRITE FILE (" + entry.cacheArray.length + " bytes) " + entry.cacheFile);
serverFileUtils.write(entry.cacheArray, entry.cacheFile);
- log.logInfo("AFTER WRITE cacheArray = " + entry.cacheFile + ": " + ((entry.cacheArray == null) ? "empty" : "full"));
+ log.logDebug("AFTER WRITE cacheArray = " + entry.cacheFile + ": " + ((entry.cacheArray == null) ? "empty" : "full"));
//entry.cacheArray = null;
} catch (FileNotFoundException e) {
// this is the case of a "(Not a directory)" error, which should be prohibited
diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index d91b983e3..d4ffd40b9 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -190,19 +190,20 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi }
// read memory amount
- int ramCacheKB = Integer.parseInt(getConfig("ramCacheSize", "1")) * 0x400;
- int ramLURL = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentLURL", "1")) / 100;
- int ramPURL = ramLURL / 2;
- int ramEURL = ramLURL / 2;
- int ramRWI = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentRWI", "1")) / 100;
- int ramHTTP = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentHTTP", "1")) / 100;
- int ramMessage = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentMessage", "1")) / 100;
- int ramWiki = ramCacheKB * Integer.parseInt(getConfig("ramCachePercentWiki", "1")) / 100;
- log.logSystem("LURL Cache memory = " + ramLURL + " KB");
- log.logSystem("RWI Cache memory = " + ramRWI + " KB");
- log.logSystem("HTTP Cache memory = " + ramHTTP + " KB");
- log.logSystem("Message Cache memory = " + ramMessage + " KB");
- log.logSystem("Wiki Cache memory = " + ramWiki + " KB");
+ int ramLURL = Integer.parseInt(getConfig("ramCacheLURL", "1024")) / 1024;
+ int ramNURL = Integer.parseInt(getConfig("ramCacheNURL", "1024")) / 1024;
+ int ramEURL = Integer.parseInt(getConfig("ramCacheEURL", "1024")) / 1024;
+ int ramRWI = Integer.parseInt(getConfig("ramCacheRWI", "1024")) / 1024;
+ int ramHTTP = Integer.parseInt(getConfig("ramCacheHTTP", "1024")) / 1024;
+ int ramMessage = Integer.parseInt(getConfig("ramCacheMessage", "1024")) / 1024;
+ int ramWiki = Integer.parseInt(getConfig("ramCacheWiki", "1024")) / 1024;
+ log.logSystem("LURL Cache memory = " + ppRamString(ramLURL));
+ log.logSystem("NURL Cache memory = " + ppRamString(ramNURL));
+ log.logSystem("EURL Cache memory = " + ppRamString(ramEURL));
+ log.logSystem("RWI Cache memory = " + ppRamString(ramRWI));
+ log.logSystem("HTTP Cache memory = " + ppRamString(ramHTTP));
+ log.logSystem("Message Cache memory = " + ppRamString(ramMessage));
+ log.logSystem("Wiki Cache memory = " + ppRamString(ramWiki));
// make crawl profiles database and default profiles
profiles = new plasmaCrawlProfile(new File(plasmaPath, "crawlProfiles0.db"));
@@ -213,10 +214,10 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi // start indexing management
loadedURL = new plasmaCrawlLURL(new File(plasmaPath, "urlHash.db"), ramLURL);
- noticeURL = new plasmaCrawlNURL(plasmaPath, ramPURL);
+ noticeURL = new plasmaCrawlNURL(plasmaPath, ramNURL);
errorURL = new plasmaCrawlEURL(new File(plasmaPath, "urlErr0.db"), ramEURL);
- //indexCache = new plasmaWordIndexRAMCache(plasmaPath, 2000, ramRWI);
- wordIndex = new plasmaWordIndex(plasmaPath, ramRWI);
+ wordIndex = new plasmaWordIndex(plasmaPath, ramRWI);
+ wordIndex.setMaxWords(10000);
searchManager = new plasmaSearch(loadedURL, wordIndex);
// start a cache manager
@@ -290,6 +291,15 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi new serverInstantThread(indexDistribution, "job", null), log, 120000);
}
+ private static String ppRamString(int bytes) {
+ if (bytes < 1024) return bytes + " KByte";
+ bytes = bytes / 1024;
+ if (bytes < 1024) return bytes + " MByte";
+ bytes = bytes / 1024;
+ if (bytes < 1024) return bytes + " GByte";
+ return (bytes / 1024) + "TByte";
+ }
+
public void handleBusyState(int jobs) {
this.serverJobs = jobs;
}
@@ -359,7 +369,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi log.logSystem("SWITCHBOARD SHUTDOWN STEP 1: sending termination signal to managed threads:");
terminateAllThreads(true);
log.logSystem("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing (stand by..)");
- int waitingBoundSeconds = Integer.parseInt(getConfig("shutdownWaiting", "120"));
+ int waitingBoundSeconds = Integer.parseInt(getConfig("maxWaitingWordFlush", "120"));
wordIndex.close(waitingBoundSeconds);
log.logSystem("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal to database manager");
try {
@@ -397,16 +407,18 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi processStack.addLast(entry);
}
- public synchronized void deQueue() {
+ public synchronized boolean deQueue() {
if (serverJobs < 6) {
if (processStack.size() > 0) {
processResourceStack((plasmaHTCache.Entry) processStack.removeFirst());
+ return true;
}
} else {
//if (processStack.size() > 0) {
log.logDebug("DEQUEUE: serverJobs=" + serverJobs + " 'busy' - no dequeueing (processStack=" + processStack.size() + ", localStackSize=" + noticeURL.localStackSize() + ", remoteStackSize=" + noticeURL.remoteStackSize() + ")");
//}
}
+ return false;
}
public int cleanupJobSize() {
@@ -444,7 +456,7 @@ public class plasmaSwitchboard extends serverAbstractSwitch implements serverSwi }
public boolean localCrawlJob() {
- if ((serverJobs < 2) &&
+ if ((serverJobs < 6) &&
(processStack.size() < crawlSlots) &&
(noticeURL.localStackSize() > 0) &&
(cacheLoader.size() < crawlSlots)) {
diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 2cb9c431f..a7256ee8f 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -57,10 +57,22 @@ public class plasmaWordIndex { public plasmaWordIndex(File databaseRoot, int bufferkb) throws IOException {
this.databaseRoot = databaseRoot;
- this.ramCache = new plasmaWordIndexRAMCache(databaseRoot, 2000, bufferkb);
+ this.ramCache = new plasmaWordIndexRAMCache(databaseRoot, bufferkb);
ramCache.start();
}
+ public int maxURLinWordCache() {
+ return ramCache.maxURLinWordCache();
+ }
+
+ public int wordCacheRAMSize() {
+ return ramCache.wordCacheRAMSize();
+ }
+
+ public void setMaxWords(int maxWords) {
+ ramCache.setMaxWords(maxWords);
+ }
+
public synchronized int addEntry(String wordHash, plasmaWordIndexEntry entry) throws IOException {
return ramCache.addEntryToIndexMem(wordHash, entry);
}
diff --git a/source/de/anomic/plasma/plasmaWordIndexRAMCache.java b/source/de/anomic/plasma/plasmaWordIndexRAMCache.java index 26890a893..fc3d595c8 100644 --- a/source/de/anomic/plasma/plasmaWordIndexRAMCache.java +++ b/source/de/anomic/plasma/plasmaWordIndexRAMCache.java @@ -70,14 +70,26 @@ public class plasmaWordIndexRAMCache extends Thread { for (int i = 0; i < yacySeedDB.commonHashLength; i++) maxKey += '-';
}
- public plasmaWordIndexRAMCache(File databaseRoot, int maxWords, int bufferkb) throws IOException {
+ public plasmaWordIndexRAMCache(File databaseRoot, int bufferkb) throws IOException {
this.pic = new plasmaWordIndexFileCache(databaseRoot, bufferkb);
this.cache = new TreeMap();
this.hashScore = new kelondroMScoreCluster();
- this.maxWords = maxWords;
+ this.maxWords = 1000;
this.terminate = false;
}
+ public int maxURLinWordCache() {
+ return hashScore.getScore(hashScore.getMaxObject());
+ }
+
+ public int wordCacheRAMSize() {
+ return cache.size();
+ }
+
+ public void setMaxWords(int maxWords) {
+ this.maxWords = maxWords;
+ }
+
public void run() {
serverLog.logSystem("PLASMA INDEXING", "started word cache management");
int check;
@@ -86,9 +98,9 @@ public class plasmaWordIndexRAMCache extends Thread { if (hashScore.size() < 100) try {Thread.currentThread().sleep(10000);} catch (InterruptedException e) {}
while ((!(terminate)) && (cache != null) && (hashScore.size() > 0)) try {
//check = hashScore.size();
- flushSpecific(true);
+ flushSpecific(false);
//serverLog.logDebug("PLASMA INDEXING", "single flush. bevore=" + check + "; after=" + hashScore.size());
- try {Thread.currentThread().sleep(200 + (maxWords / (1 + hashScore.size())));} catch (InterruptedException e) {}
+ try {Thread.currentThread().sleep(10 + ((maxWords / 10) / (1 + hashScore.size())));} catch (InterruptedException e) {}
} catch (IOException e) {
serverLog.logError("PLASMA INDEXING", "PANIK! exception in main cache loop: " + e.getMessage());
e.printStackTrace();
@@ -158,7 +170,7 @@ public class plasmaWordIndexRAMCache extends Thread { String key = (String) ((greatest) ? hashScore.getMaxObject() : hashScore.getMinObject());
return flushKey(key, "flushSpecific");
}
-
+
private synchronized int flushKey(String key, String caller) throws IOException {
Vector v = (Vector) cache.get(key);
if (v == null) {
@@ -197,7 +209,7 @@ public class plasmaWordIndexRAMCache extends Thread { // make space for new words
int flushc = 0;
//serverLog.logDebug("PLASMA INDEXING", "addEntryToIndexMem: cache.size=" + cache.size() + "; hashScore.size=" + hashScore.size());
- while (hashScore.size() > maxWords) flushc += flushSpecific(false);
+ while (hashScore.size() > maxWords) flushc += flushSpecific(true);
//if (flushc > 0) serverLog.logDebug("PLASMA INDEXING", "addEntryToIndexMem - flushed " + flushc + " entries");
// put new words into cache
diff --git a/source/de/anomic/server/serverAbstractSwitch.java b/source/de/anomic/server/serverAbstractSwitch.java index 25aeb2756..7530dae24 100644 --- a/source/de/anomic/server/serverAbstractSwitch.java +++ b/source/de/anomic/server/serverAbstractSwitch.java @@ -225,7 +225,7 @@ public abstract class serverAbstractSwitch implements serverSwitch { abstract public int queueSize();
abstract public void enQueue(Object job);
- abstract public void deQueue();
+ abstract public boolean deQueue();
// authentification routines:
diff --git a/source/de/anomic/server/serverSwitch.java b/source/de/anomic/server/serverSwitch.java index e926371fb..a2ce82963 100644 --- a/source/de/anomic/server/serverSwitch.java +++ b/source/de/anomic/server/serverSwitch.java @@ -74,7 +74,7 @@ public interface serverSwitch { // work off exactly only one job, not all
public int queueSize();
public void enQueue(Object job);
- public void deQueue();
+ public boolean deQueue(); // returns true if there had been dequeued anything
// authentification routines: sets and reads access attributes according to host addresses
public void setAuthentify(InetAddress host, String user, String rigth);
@@ -12,13 +12,6 @@ # port number of server
port = 8080
-# shutdown time limit
-# this is the time that a peer takes at most for shutdown
-# the shutdown-procedure is difficult since there are many
-# caches that must be flushed first
-# measured in seconds
-shutdownWaiting = 120
-
# time-out of client control socket in milliseconds
# since this applies only to the client-proxy connection,
# it can be rather short
@@ -94,19 +87,6 @@ parseableMime=application/xhtml+xml,text/html,text/plain # These files will be excluded from indexing
mediaExt=swf,wmv,jpg,jpeg,jpe,rm,mov,mpg,mpeg,mp3,asf,gif,png,avi,zip,rar,sit,hqx,img,dmg,tar,gz,ps,pdf,doc,xls,ppt,ram,bz2,arj,jar,deb,torrent,ogg,iso,bin,ace,tgz,rpm,css
-# the proxy's and indexing maximum ram cache size in megabytes
-ramCacheSize = 12
-# ram cache is partitioned into different separate caches
-# we define percentual shares of the complete ram cache
-# values here are computed from usage profiles and should not be changed
-# it is not computed if the complete sum of percentages computes to 100
-ramCachePercentRWI = 85
-ramCachePercentHTTP = 6
-ramCachePercentLURL = 4
-ramCachePercentDHT = 3
-ramCachePercentMessage = 1
-ramCachePercentWiki = 1
-
# Promotion Strings
# These strings appear in the Web Mask of the YACY search client
# Set these Strings to cusomize your peer and give any message to
@@ -401,3 +381,35 @@ xpstopw=true 80_dequeue_busysleep=1000
90_cleanup_idlesleep=300000
90_cleanup_busysleep=300000
+
+# ram cache for database files
+
+# ram cache for indexCache.db
+ramCacheRWI = 16777216
+
+# ram cache for responseHeader.db
+ramCacheHTTP = 1048576
+
+# ram cache for urlHash.db
+ramCacheLURL = 8388608
+
+# ram cache for urlNotice.db
+ramCacheNURL = 1048576
+
+# ram cache for urlErr.db
+ramCacheEURL = 131072
+
+# ram cache for message.db
+ramCacheMessage = 8192
+
+# ram cache for wiki.db
+ramCacheWiki = 8192
+
+# performance properties for the word index cache
+# wordCacheMax is the number of word indexes that shall be held in the
+# ram cache during indexing. When YaCy is shut down, this cache must be
+# flushed to disc; this may last some minutes.
+# maxWaitingWordFlush gives the number of seconds that the shutdown
+# may last for the word flush
+wordCacheMax = 10000
+maxWaitingWordFlush = 180
|