diff options
| author | orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542> | 2007-08-26 22:06:58 +0000 |
|---|---|---|
| committer | orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542> | 2007-08-26 22:06:58 +0000 |
| commit | d9472b6a3a3bd8d7cb9bdd9a247a111165b9b398 (patch) | |
| tree | 925dd96178e00aa2a8884a2d300ae98a333bb393 | |
| parent | e332b844b2c0d4d41bef0add8f3f5de140839cc7 (diff) | |
* fixed problem with watch crawler
* added new column to network table (remote crawl urls):
the new value for provided URLs will be used for new remote crawl method
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4061 6c8d7289-2bf4-0310-a012-ef5d649a1542
| -rw-r--r-- | htroot/Network.html | 20 | ||||
| -rw-r--r-- | htroot/Network.java | 34 | ||||
| -rw-r--r-- | htroot/Network.xml | 10 | ||||
| -rw-r--r-- | htroot/xml/queues_p.xml | 3 | ||||
| -rw-r--r-- | source/de/anomic/plasma/plasmaCrawlBalancer.java | 4 |
5 files changed, 42 insertions, 29 deletions
diff --git a/htroot/Network.html b/htroot/Network.html index c544249e9..da383219c 100644 --- a/htroot/Network.html +++ b/htroot/Network.html @@ -59,8 +59,9 @@ <td><strong>UTC</strong><br />Offset<br /> </td>
<td><strong>Location</strong><br /><br /> </td>
<td><strong>Uptime</strong><br /> <a href="/Network.html?page=#[page]#&sort=Uptime&order=up"><</a> <a href="/Network.html?page=#[page]#&sort=Uptime&order=down">></a></td>
- <td><strong>#Links</strong><br /> <a href="/Network.html?page=#[page]#&sort=LCount&order=up"><</a> <a href="/Network.html?page=#[page]#&sort=LCount&order=down">></a></td>
- <td><strong>#RWIs</strong><br /> <a href="/Network.html?page=#[page]#&sort=ICount&order=up"><</a> <a href="/Network.html?page=#[page]#&sort=ICount&order=down">></a></td>
+ <td><strong>Links</strong><br /> <a href="/Network.html?page=#[page]#&sort=LCount&order=up"><</a> <a href="/Network.html?page=#[page]#&sort=LCount&order=down">></a></td>
+ <td><strong>RWIs</strong><br /> <a href="/Network.html?page=#[page]#&sort=ICount&order=up"><</a> <a href="/Network.html?page=#[page]#&sort=ICount&order=down">></a></td>
+ <td><strong>URLs for<br />Remote<br />Crawl</strong><br /> </td>
<td><strong>Sent<br />Words</strong><br /> </td>
<td><strong>Sent<br />URLs</strong><br /> </td>
<td><strong>Received<br />Words</strong><br /> </td>
@@ -97,8 +98,9 @@ <td align="right">#[utc]#</td>
<td align="right">#[location]#</td>
<td align="right">#[uptime]#</td>
- <td align="right">#[links]#</td>
- <td align="right">#[words]#</td>
+ <td align="right">#[LCount]#</td>
+ <td align="right">#[ICount]#</td>
+ <td align="right">#[RCount]#</td>
<td align="right">#[sI]#</td>
<td align="right">#[sU]#</td>
<td align="right">#[rI]#</td>
@@ -238,8 +240,9 @@ <td>Version</td>
<td>UTC</td>
<td>Uptime</td>
- <td>#Links</td>
- <td>#RWIs</td>
+ <td>Links</td>
+ <td>RWIs</td>
+ <td>URLs for<br />Remote Crawl</td>
<td>Sent<br />Words</td>
<td>Sent<br />URLs</td>
<td>Received<br />Words</td>
@@ -257,8 +260,9 @@ <td align="right">#[my-version]#</td>
<td align="right">#[my-utc]#</td>
<td align="right">#[my-uptime]#</td>
- <td align="right">#[my-links]#</td>
- <td align="right">#[my-words]#</td>
+ <td align="right">#[my-LCount]#</td>
+ <td align="right">#[my-ICount]#</td>
+ <td align="right">#[my-RCount]#</td>
<td align="right">#[my-sI]#</td>
<td align="right">#[my-sU]#</td>
<td align="right">#[my-rI]#</td>
diff --git a/htroot/Network.java b/htroot/Network.java index 47be416d9..80d9ab644 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -110,12 +110,14 @@ public class Network { // update seed info
yacyCore.peerActions.updateMySeed();
- long links;
- long words;
+ long LCount;
+ long ICount;
+ long RCount;
try {
- links = Long.parseLong(seed.get(yacySeed.LCOUNT, "0"));
- words = Long.parseLong(seed.get(yacySeed.ICOUNT, "0"));
- } catch (Exception e) {links = 0; words = 0;}
+ LCount = Long.parseLong(seed.get(yacySeed.LCOUNT, "0"));
+ ICount = Long.parseLong(seed.get(yacySeed.ICOUNT, "0"));
+ RCount = Long.parseLong(seed.get(yacySeed.RCOUNT, "0"));
+ } catch (Exception e) {LCount = 0; ICount = 0; RCount = 0;}
// my-info
prop.put("table_my-name", seed.get(yacySeed.NAME, "-") );
@@ -124,16 +126,16 @@ public class Network { prop.put("table_my-info", 0);
} else if(yacyCore.seedDB.mySeed.isJunior()) {
prop.put("table_my-info", 1);
- accPotLinks += links;
- accPotWords += words;
+ accPotLinks += LCount;
+ accPotWords += ICount;
} else if(yacyCore.seedDB.mySeed.isSenior()) {
prop.put("table_my-info", 2);
- accActLinks += links;
- accActWords += words;
+ accActLinks += LCount;
+ accActWords += ICount;
} else if(yacyCore.seedDB.mySeed.isPrincipal()) {
prop.put("table_my-info", 3);
- accActLinks += links;
- accActWords += words;
+ accActLinks += LCount;
+ accActWords += ICount;
}
prop.put("table_my-acceptcrawl", seed.getFlagAcceptRemoteCrawl() ? 1 : 0);
prop.put("table_my-dhtreceive", seed.getFlagAcceptRemoteIndex() ? 1 : 0);
@@ -145,8 +147,9 @@ public class Network { prop.put("table_my-version", seed.get(yacySeed.VERSION, "-"));
prop.put("table_my-utc", seed.get(yacySeed.UTC, "-"));
prop.put("table_my-uptime", serverDate.intervalToString(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, ""))));
- prop.put("table_my-links", groupDigits(links));
- prop.put("table_my-words", groupDigits(words));
+ prop.put("table_my-LCount", groupDigits(LCount));
+ prop.put("table_my-ICount", groupDigits(ICount));
+ prop.put("table_my-RCount", groupDigits(RCount));
prop.put("table_my-sI", groupDigits(seed.get(yacySeed.INDEX_OUT, "0")));
prop.put("table_my-sU", groupDigits(seed.get(yacySeed.URL_OUT, "0")));
prop.put("table_my-rI", groupDigits(seed.get(yacySeed.INDEX_IN, "0")));
@@ -470,8 +473,9 @@ public class Network { prop.put(STR_TABLE_LIST + conCount + "_lastSeen", /*seed.getLastSeenString() + " " +*/ lastseen);
prop.put(STR_TABLE_LIST + conCount + "_utc", seed.get(yacySeed.UTC, "-"));
prop.put(STR_TABLE_LIST + conCount + "_uptime", serverDate.intervalToString(60000 * Long.parseLong(seed.get(yacySeed.UPTIME, "0"))));
- prop.put(STR_TABLE_LIST + conCount + "_links", groupDigits(seed.get(yacySeed.LCOUNT, "0")));
- prop.put(STR_TABLE_LIST + conCount + "_words", groupDigits(seed.get(yacySeed.ICOUNT, "0")));
+ prop.put(STR_TABLE_LIST + conCount + "_LCount", groupDigits(seed.get(yacySeed.LCOUNT, "0")));
+ prop.put(STR_TABLE_LIST + conCount + "_ICount", groupDigits(seed.get(yacySeed.ICOUNT, "0")));
+ prop.put(STR_TABLE_LIST + conCount + "_RCount", groupDigits(seed.get(yacySeed.RCOUNT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_sI", groupDigits(seed.get(yacySeed.INDEX_OUT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_sU", groupDigits(seed.get(yacySeed.URL_OUT, "0")));
prop.put(STR_TABLE_LIST + conCount + "_rI", groupDigits(seed.get(yacySeed.INDEX_IN, "0")));
diff --git a/htroot/Network.xml b/htroot/Network.xml index c5c961665..6d80113e3 100644 --- a/htroot/Network.xml +++ b/htroot/Network.xml @@ -9,8 +9,9 @@ <ppm>#[ppm]#</ppm>
<qph>#[qph]#</qph>
<uptime>#[uptime]#</uptime>
- <links>#[links]#</links>
- <words>#[words]#</words>
+ <links>#[LCount]#</links>
+ <words>#[ICount]#</words> + <rurls>#[RCount]#</rurls> <lastseen>#[lastSeen]#</lastseen>
<sendWords>#[sI]#</sendWords>
<receivedWords>#[rI]#</receivedWords>
@@ -60,8 +61,9 @@ <version>#[my-version]#</version>
<utc>#[my-utc]#</utc>
<uptime>#[my-uptime]#</uptime>
- <links>#[my-links]#</links>
- <words>#[my-words]#</words>
+ <links>#[my-LCount]#</links>
+ <words>#[my-ICount]#</words> + <words>#[my-RCount]#</words> <acceptcrawl>#[my-acceptcrawl]#</acceptcrawl>
<acceptindex>#[my-dhtreceive]#</acceptindex>
<acceptranking>#[my-rankingreceive]#</acceptranking>
diff --git a/htroot/xml/queues_p.xml b/htroot/xml/queues_p.xml index 41928d748..ed82dd877 100644 --- a/htroot/xml/queues_p.xml +++ b/htroot/xml/queues_p.xml @@ -1,4 +1,5 @@ <?xml version="1.0"?> +<queues> <dbsize> <urlpublictext>#[urlpublictextSize]#</urlpublictext> <rwipublictext>#[rwipublictextSize]#</rwipublictext> @@ -64,4 +65,4 @@ </entry> #{/list-remote}# </remotecrawlerqueue> -</xml> +</queues> diff --git a/source/de/anomic/plasma/plasmaCrawlBalancer.java b/source/de/anomic/plasma/plasmaCrawlBalancer.java index 428101792..177631ee3 100644 --- a/source/de/anomic/plasma/plasmaCrawlBalancer.java +++ b/source/de/anomic/plasma/plasmaCrawlBalancer.java @@ -467,7 +467,9 @@ public class plasmaCrawlBalancer { while ((urlRAMStack.size() <= dist) && (urlFileStack.size() > 0)) {
// flush some entries from disc to ram stack
try {
- urlRAMStack.add(new String(urlFileStack.pop().getColBytes(0)));
+ kelondroRow.Entry t = urlFileStack.pop();
+ if (t == null) break;
+ urlRAMStack.add(new String(t.getColBytes(0)));
} catch (IOException e) {
break;
}
|