diff options
| author | orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542> | 2008-05-14 21:36:02 +0000 |
|---|---|---|
| committer | orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542> | 2008-05-14 21:36:02 +0000 |
| commit | cfe6790498a1060b61e0ae7625c1a5c354e1acbe (patch) | |
| tree | e5de56128ea09dc8594d919ae8e1243d1b20b7c8 /htroot | |
| parent | 7429687601d13972bedc5ea5d015af36ac9085a0 (diff) | |
- added option to switch between yacy networks, especially between the two default networks (freeworld and intranet),
from the ConfigNetwork online interface
- to make this possible, a large refactoring and reorganisation of data structures was necessary
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@4803 6c8d7289-2bf4-0310-a012-ef5d649a1542
Diffstat (limited to 'htroot')
68 files changed, 591 insertions, 530 deletions
diff --git a/htroot/AccessTracker_p.java b/htroot/AccessTracker_p.java index f9d712040..0c43702c1 100644 --- a/htroot/AccessTracker_p.java +++ b/htroot/AccessTracker_p.java @@ -203,7 +203,7 @@ public class AccessTracker_p { prop.put("page_list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark; prop.putHTML("page_list_" + entCount + "_host", host); if (page == 5) { - yacySeed remotepeer = sb.wordIndex.seedDB.lookupByIP(natLib.getInetAddress(host), true, true, true); + yacySeed remotepeer = sb.webIndex.seedDB.lookupByIP(natLib.getInetAddress(host), true, true, true); prop.putHTML("page_list_" + entCount + "_peername", (remotepeer == null) ? "UNKNOWN" : remotepeer.getName()); } prop.putNum("page_list_" + entCount + "_count", handles.size()); diff --git a/htroot/Banner.java b/htroot/Banner.java index 400dced39..bb0ab1bde 100644 --- a/htroot/Banner.java +++ b/htroot/Banner.java @@ -84,14 +84,14 @@ public class Banner { double myqph = 0; String type = ""; String network = env.getConfig("network.unit.name", "unspecified").toUpperCase(); - int peers = sb.wordIndex.seedDB.sizeConnected() + 1; // the '+ 1': the own peer is not included in sizeConnected() - long nlinks = sb.wordIndex.seedDB.countActiveURL(); - long nwords = sb.wordIndex.seedDB.countActiveRWI(); - double nqpm = sb.wordIndex.seedDB.countActiveQPM(); - long nppm = sb.wordIndex.seedDB.countActivePPM(); + int peers = sb.webIndex.seedDB.sizeConnected() + 1; // the '+ 1': the own peer is not included in sizeConnected() + long nlinks = sb.webIndex.seedDB.countActiveURL(); + long nwords = sb.webIndex.seedDB.countActiveRWI(); + double nqpm = sb.webIndex.seedDB.countActiveQPM(); + long nppm = sb.webIndex.seedDB.countActivePPM(); double nqph = 0; - yacySeed seed = sb.wordIndex.seedDB.mySeed(); + yacySeed seed = sb.webIndex.seedDB.mySeed(); if (seed != null){ name = seed.get(yacySeed.NAME, "-").toUpperCase(); links = Long.parseLong(seed.get(yacySeed.LCOUNT, "0")); @@ -99,19 +99,19 @@ public class Banner { myppm = seed.getPPM(); myqph = 60d * seed.getQPM(); - if (sb.wordIndex.seedDB.mySeed().isVirgin()) { + if (sb.webIndex.seedDB.mySeed().isVirgin()) { type = "VIRGIN"; nqph = Math.round(6000d * nqpm) / 100d; - } else if(sb.wordIndex.seedDB.mySeed().isJunior()) { + } else if(sb.webIndex.seedDB.mySeed().isJunior()) { type = "JUNIOR"; nqph = Math.round(6000d * nqpm) / 100d; - } else if(sb.wordIndex.seedDB.mySeed().isSenior()) { + } else if(sb.webIndex.seedDB.mySeed().isSenior()) { type = "SENIOR"; nlinks = nlinks + links; nwords = nwords + words; nqph = Math.round(6000d * nqpm + 100d * myqph) / 100d; nppm = nppm + myppm; - } else if(sb.wordIndex.seedDB.mySeed().isPrincipal()) { + } else if(sb.webIndex.seedDB.mySeed().isPrincipal()) { type = "PRINCIPAL"; nlinks = nlinks + links; nwords = nwords + words; diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java index 0f548a7bc..a5fb66594 100644 --- a/htroot/Blacklist_p.java +++ b/htroot/Blacklist_p.java @@ -321,11 +321,11 @@ prop.putHTML("asd", "0"); // List known hosts for BlackList retrieval
- if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0) { // no nullpointer error
+ if (sb.webIndex.seedDB != null && sb.webIndex.seedDB.sizeConnected() > 0) { // no nullpointer error
int peerCount = 0;
try {
TreeMap<String, String> hostList = new TreeMap<String, String>();
- final Iterator<yacySeed> e = sb.wordIndex.seedDB.seedsConnected(true, false, null, (float) 0.0);
+ final Iterator<yacySeed> e = sb.webIndex.seedDB.seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
yacySeed seed = (yacySeed) e.next();
if (seed != null) hostList.put(seed.get(yacySeed.NAME, "nameless"),seed.hash);
diff --git a/htroot/Blog.java b/htroot/Blog.java index f276e6a99..924b5a642 100644 --- a/htroot/Blog.java +++ b/htroot/Blog.java @@ -87,7 +87,7 @@ public class Blog { final boolean xml = ((String)header.get(httpHeader.CONNECTION_PROP_PATH)).endsWith(".xml"); - final String address = sb.wordIndex.seedDB.mySeed().getPublicAddress(); + final String address = sb.webIndex.seedDB.mySeed().getPublicAddress(); if(hasRights) { prop.put("mode_admin", "1"); @@ -96,7 +96,7 @@ public class Blog { } if (post == null) { - prop.putHTML("peername", sb.wordIndex.seedDB.mySeed().getName()); + prop.putHTML("peername", sb.webIndex.seedDB.mySeed().getName()); prop.put("address", address); return putBlogDefault(prop, sb, address, 0, 10, hasRights, xml); } @@ -123,10 +123,10 @@ public class Blog { StrAuthor = sb.blogDB.guessAuthor(ip); if (StrAuthor == null || StrAuthor.length() == 0) { - if (sb.wordIndex.seedDB.mySeed() == null) { + if (sb.webIndex.seedDB.mySeed() == null) { StrAuthor = "anonymous"; } else { - StrAuthor = sb.wordIndex.seedDB.mySeed().get("Name", "anonymous"); + StrAuthor = sb.webIndex.seedDB.mySeed().get("Name", "anonymous"); } } } @@ -188,7 +188,7 @@ public class Blog { map.put("page", pagename); map.put("subject", StrSubject.replace(',', ' ')); map.put("author", StrAuthor.replace(',', ' ')); - sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BLOG_ADD, map)); + sb.webIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BLOG_ADD, map)); } page = sb.blogDB.readBlogEntry(pagename); //maybe "if(page == null)" @@ -265,7 +265,7 @@ public class Blog { if(pagename.equals(DEFAULT_PAGE)) { // XXX: where are "peername" and "address" used in the template? // XXX: "clientname" is already set to the peername, no need for a new setting - prop.putHTML("peername", sb.wordIndex.seedDB.mySeed().getName(), xml); + prop.putHTML("peername", sb.webIndex.seedDB.mySeed().getName(), xml); prop.put("address", address); //index all entries putBlogDefault(prop, sb, address, start, num, hasRights, xml); diff --git a/htroot/BlogComments.java b/htroot/BlogComments.java index ed75f3c0b..4b4b2169d 100644 --- a/htroot/BlogComments.java +++ b/htroot/BlogComments.java @@ -110,11 +110,11 @@ public class BlogComments { StrAuthor = sb.blogDB.guessAuthor(ip); if (StrAuthor == null || StrAuthor.length() == 0) { - if (sb.wordIndex.seedDB.mySeed() == null) { + if (sb.webIndex.seedDB.mySeed() == null) { StrAuthor = "anonymous"; } else { - StrAuthor = sb.wordIndex.seedDB.mySeed().get("Name", "anonymous"); + StrAuthor = sb.webIndex.seedDB.mySeed().get("Name", "anonymous"); } } } @@ -169,15 +169,15 @@ public class BlogComments { sb.messageDB.write(msgEntry = sb.messageDB.newEntry( "blogComment", StrAuthor, - sb.wordIndex.seedDB.mySeed().hash, - sb.wordIndex.seedDB.mySeed().getName(), sb.wordIndex.seedDB.mySeed().hash, + sb.webIndex.seedDB.mySeed().hash, + sb.webIndex.seedDB.mySeed().getName(), sb.webIndex.seedDB.mySeed().hash, "new blog comment: " + new String(blogEntry.getSubject(),"UTF-8"), content)); } catch (UnsupportedEncodingException e1) { sb.messageDB.write(msgEntry = sb.messageDB.newEntry( "blogComment", StrAuthor, - sb.wordIndex.seedDB.mySeed().hash, - sb.wordIndex.seedDB.mySeed().getName(), sb.wordIndex.seedDB.mySeed().hash, + sb.webIndex.seedDB.mySeed().hash, + sb.webIndex.seedDB.mySeed().getName(), sb.webIndex.seedDB.mySeed().hash, "new blog comment: " + new String(blogEntry.getSubject()), content)); } @@ -355,7 +355,7 @@ public class BlogComments { .append(sendMailTo) .append("\nFrom: ") .append("yacy@") - .append(sb.wordIndex.seedDB.mySeed().getName()) + .append(sb.webIndex.seedDB.mySeed().getName()) .append("\nSubject: [YaCy] ") .append(msgEntry.subject().replace('\n', ' ')) .append("\nDate: ") diff --git a/htroot/Bookmarks.java b/htroot/Bookmarks.java index 4fd5cd838..cb60c0e3f 100644 --- a/htroot/Bookmarks.java +++ b/htroot/Bookmarks.java @@ -111,7 +111,7 @@ public class Bookmarks { */ // set peer address - final String address = sb.wordIndex.seedDB.mySeed().getPublicAddress(); + final String address = sb.webIndex.seedDB.mySeed().getPublicAddress(); prop.put("address", address); //defaultvalues @@ -195,7 +195,7 @@ public class Bookmarks { bookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(urlHash); if (bookmark == null) { // try to get the bookmark from the LURL database - indexURLReference urlentry = sb.wordIndex.getURL(urlHash, null, 0); + indexURLReference urlentry = sb.webIndex.getURL(urlHash, null, 0); plasmaParserDocument document = null; if (urlentry != null) { indexURLReference.Components comp = urlentry.comp(); @@ -443,7 +443,7 @@ public class Bookmarks { map.put("title", title.replace(',', ' ')); map.put("description", description.replace(',', ' ')); map.put("tags", tagsString.replace(',', ' ')); - sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); + sb.webIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); } } diff --git a/htroot/ConfigBasic.java b/htroot/ConfigBasic.java index 732c3d901..c9d8dc436 100644 --- a/htroot/ConfigBasic.java +++ b/htroot/ConfigBasic.java @@ -90,7 +90,7 @@ public class ConfigBasic { // starting a peer ping //boolean doPeerPing = false; - if ((sb.wordIndex.seedDB.mySeed().isVirgin()) || (sb.wordIndex.seedDB.mySeed().isJunior())) { + if ((sb.webIndex.seedDB.mySeed().isVirgin()) || (sb.webIndex.seedDB.mySeed().isJunior())) { serverInstantBusyThread.oneTimeJob(sb.yc, "peerPing", null, 0); //doPeerPing = true; } @@ -125,7 +125,7 @@ public class ConfigBasic { } // check if peer name already exists - yacySeed oldSeed = sb.wordIndex.seedDB.lookupByName(peerName); + yacySeed oldSeed = sb.webIndex.seedDB.lookupByName(peerName); if ((oldSeed == null) && (!(env.getConfig("peerName", "").equals(peerName)))) { // the name is new boolean nameOK = Pattern.compile("[A-Za-z0-9\\-_]{3,80}").matcher(peerName).matches(); @@ -168,7 +168,7 @@ public class ConfigBasic { // check if values are proper boolean properPW = (env.getConfig("adminAccount", "").length() == 0) && (env.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0); boolean properName = (env.getConfig("peerName","").length() >= 3) && (!(yacySeed.isDefaultPeerName(env.getConfig("peerName","")))); - boolean properPort = (sb.wordIndex.seedDB.mySeed().isSenior()) || (sb.wordIndex.seedDB.mySeed().isPrincipal()); + boolean properPort = (sb.webIndex.seedDB.mySeed().isSenior()) || (sb.webIndex.seedDB.mySeed().isPrincipal()); if ((properPW) && (env.getConfig("defaultFiles", "").startsWith("ConfigBasic.html,"))) { env.setConfig("defaultFiles", env.getConfig("defaultFiles", "").substring(17)); diff --git a/htroot/ConfigNetwork_p.html b/htroot/ConfigNetwork_p.html index 00ce2c64d..a40de8785 100644 --- a/htroot/ConfigNetwork_p.html +++ b/htroot/ConfigNetwork_p.html @@ -41,13 +41,14 @@ #(commit)# ::<div class="commit">Accepted Changes.</div> ::<div class="error">Inapplicable Setting Combination:</div> - #(/commit)# - #(commitCrawlPlea)#::<div class="error">P2P operation can run without remote indexing, but runs better with remote indexing switched on. Please switch 'Accept Remote Crawl Requests' on.</div>#(/commitCrawlPlea)# - #(commitDHTIsRobinson)#::<div class="error">For P2P operation, at least DHT distribution or DHT receive (or both) must be set. You have thus defined a Robinson configuration.</div>#(/commitDHTIsRobinson)# - #(commitDHTNoGlobalSearch)#::<div class="error">Global Search in P2P configuration is only allowed, if both, index receive and distribution is switched on. You have a P2P configuration, but are not allowed to search other peers.</div>#(/commitDHTNoGlobalSearch)# - #(commitRobinson)#::<div class="commit">For Robinson Mode, index distribution and receive is switched off.</div>#(/commitRobinson)# - #(commitRobinsonWithRemoteIndexing)#::<div class="commit">This Robinson Mode switches remote indexing on, but limits targets to peers within the same cluster. Remote indexing requests from peers within the same cluster are accepted.</div>#(/commitRobinsonWithRemoteIndexing)# - #(commitRobinsonWithoutRemoteIndexing)#::<div class="commit">This Robinson Mode does not allow any remote indexing (neither requests remote indexing, nor accepts it).</div>#(/commitRobinsonWithoutRemoteIndexing)# + ::<div class="error">No changes were made!</div> + #(/commit)# + #(commitCrawlPlea)#::<div class="error">P2P operation can run without remote indexing, but runs better with remote indexing switched on. Please switch 'Accept Remote Crawl Requests' on.</div>#(/commitCrawlPlea)# + #(commitDHTIsRobinson)#::<div class="error">For P2P operation, at least DHT distribution or DHT receive (or both) must be set. You have thus defined a Robinson configuration.</div>#(/commitDHTIsRobinson)# + #(commitDHTNoGlobalSearch)#::<div class="error">Global Search in P2P configuration is only allowed, if both, index receive and distribution is switched on. You have a P2P configuration, but are not allowed to search other peers.</div>#(/commitDHTNoGlobalSearch)# + #(commitRobinson)#::<div class="commit">For Robinson Mode, index distribution and receive is switched off.</div>#(/commitRobinson)# + #(commitRobinsonWithRemoteIndexing)#::<div class="commit">This Robinson Mode switches remote indexing on, but limits targets to peers within the same cluster. Remote indexing requests from peers within the same cluster are accepted.</div>#(/commitRobinsonWithRemoteIndexing)# + #(commitRobinsonWithoutRemoteIndexing)#::<div class="commit">This Robinson Mode does not allow any remote indexing (neither requests remote indexing, nor accepts it).</div>#(/commitRobinsonWithoutRemoteIndexing)# <form name="ConfigForm" method="post" action="ConfigNetwork_p.html" enctype="multipart/form-data" accept-charset="UTF-8"> <fieldset> <legend> @@ -60,23 +61,33 @@ </p> <dl> <dt>Network Definition</dt> - <dd>#[network.unit.definition]#</dd> - <dt>Network Nick</dt> - <dd>#[network.unit.name]#</dd> - <dt>Long Description</dt> - <dd>#[network.unit.description]#</dd> - <dt>Indexing Domain</dt> - <dd>#[network.unit.domain]#</dd> - <dt>DHT</dt> - <dd>#[network.unit.dht]#</dd> + <dd> + <select id="networkDefinition" name="networkDefinition"> + <option selected="selected">#[network.unit.definition]#</option> + #{networks}# + <option>#[network]#</option> + #{/networks}# + </select> + + </dd> + <dt>Network Nick</dt> + <dd>#[network.unit.name]#</dd> + <dt>Long Description</dt> + <dd>#[network.unit.description]#</dd> + <dt>Indexing Domain</dt> + <dd>#[network.unit.domain]#</dd> + <dt>DHT</dt> + <dd>#[network.unit.dht]#</dd> + <dt> </dt> + <dd><input type="submit" name="changeNetwork" value="Change Network" /></dd> </dl> - </fieldset> - </form> + </fieldset> + </form> <fieldset> <legend> <label for="p2p">Distributed Computing Network for Domain</label> </legend> - + <p> You can configure if you want to participate at the global YaCy network or if you want to have your own separate search cluster with or without connection to the global network. You may also define @@ -89,7 +100,7 @@ <input type="radio" name="network" id="p2p" onclick="EnableP2P()" value="p2p"#(p2p.checked)#:: checked="checked"#(/p2p.checked)# /> <label for="p2p">Peer-to-Peer Mode</label> </legend> - <dl> + <dl> <dt> <label for="indexDistribute">Index Distribution</label> <input type="checkbox" id="indexDistribute" name="indexDistribute" onclick="CheckForRobinson()" #(indexDistributeChecked)#::checked="checked" #(/indexDistributeChecked)#/> @@ -112,18 +123,18 @@ name="indexDistributeWhileIndexing" #(indexDistributeWhileIndexing.off)#::checked="checked" #(/indexDistributeWhileIndexing.off)#/> <label for="indexDistributeWhileIndexingOff">disabled during indexing</label>. - </dd> - <dt> - <label for="indexReceive">Index Receive</label> - <input type="checkbox" id="indexReceive" name="indexReceive" onclick="CheckForRobinson()" #(indexReceiveChecked)#::checked="checked" #(/indexReceiveChecked)#/> - </dt> - <dd> - Accept remote Index Transmissions.<br /> - This works only if you have a senior peer. The DHT-rules do not work without this function.<br /> - <input type="radio" value="on" id="indexReceiveBlockBlacklistOn" - name="indexReceiveBlockBlacklist" - #(indexReceiveBlockBlacklistChecked.on)#::checked="checked" #(/indexReceiveBlockBlacklistChecked.on)#/> - <label for="indexReceiveBlockBlacklistOn">reject</label> / + </dd> + <dt> + <label for="indexReceive">Index Receive</label> + <input type="checkbox" id="indexReceive" name="indexReceive" onclick="CheckForRobinson()" #(indexReceiveChecked)#::checked="checked" #(/indexReceiveChecked)#/> + </dt> + <dd> + Accept remote Index Transmissions.<br /> + This works only if you have a senior peer. The DHT-rules do not work without this function.<br /> + <input type="radio" value="on" id="indexReceiveBlockBlacklistOn" + name="indexReceiveBlockBlacklist" + #(indexReceiveBlockBlacklistChecked.on)#::checked="checked" #(/indexReceiveBlockBlacklistChecked.on)#/> + <label for="indexReceiveBlockBlacklistOn">reject</label> / <input type="radio" value="off" id="indexReceiveBlockBlacklistOff" name="indexReceiveBlockBlacklist" #(indexReceiveBlockBlacklistChecked.off)#::checked="checked" #(/indexReceiveBlockBlacklistChecked.off)#/> @@ -133,11 +144,11 @@ <label for="crawlResponse">Accept Remote Crawl Requests</label> <input type="checkbox" id="crawlResponse" name="crawlResponse" #(crawlResponse)#::checked="checked" #(/crawlResponse)#/> </dt> - <dd> - Perform web indexing upon request of another peer.<br /> - This works only if you are a senior peer.<br /> - <label for="acceptCrawlLimit">Load with a maximum of</label> - <input id="acceptCrawlLimit" name="acceptCrawlLimit" type="text" size="3" maxlength="3" value="#[acceptCrawlLimit]#" /> pages per minute + <dd> + Perform web indexing upon request of another peer.<br /> + This works only if you are a senior peer.<br /> + <label for="acceptCrawlLimit">Load with a maximum of</label> + <input id="acceptCrawlLimit" name="acceptCrawlLimit" type="text" size="3" maxlength="3" value="#[acceptCrawlLimit]#" /> pages per minute </dd> </dl> </fieldset> diff --git a/htroot/ConfigNetwork_p.java b/htroot/ConfigNetwork_p.java index ed9abb674..ff6dc244e 100644 --- a/htroot/ConfigNetwork_p.java +++ b/htroot/ConfigNetwork_p.java @@ -25,118 +25,153 @@ // along with this program; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +import java.io.File; +import java.util.HashSet; + import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.plasma.plasmaWordIndex; import de.anomic.server.serverBusyThread; import de.anomic.server.serverCodings; +import de.anomic.server.serverFileUtils; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; public class ConfigNetwork_p { - public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) { + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) { plasmaSwitchboard sb = (plasmaSwitchboard) env; serverObjects prop = new serverObjects(); int commit = 0; - prop.put("network.unit.definition", sb.getConfig("network.unit.definition", "")); - prop.put("network.unit.name", sb.getConfig("network.unit.name", "")); - prop.put("network.unit.description", sb.getConfig("network.unit.description", "")); - prop.put("network.unit.domain", sb.getConfig("network.unit.domain", "")); - prop.put("network.unit.dht", sb.getConfig("network.unit.dht", "")); + // load all options for network definitions + File networkBootstrapLocationsFile = new File(new File(sb.getRootPath(), "defaults"), "yacy.networks"); + HashSet<String> networkBootstrapLocations = serverFileUtils.loadList(networkBootstrapLocationsFile); + if (post != null) { - - boolean crawlResponse = post.get("crawlResponse", "off").equals("on"); - - // DHT control - boolean indexDistribute = post.get("indexDistribute", "").equals("on"); - boolean indexReceive = post.get("indexReceive", "").equals("on"); - boolean robinsonmode = post.get("network", "").equals("robinson"); - String clustermode = post.get("cluster.mode", "publicpeer"); - if (robinsonmode) { - indexDistribute = false; - indexReceive = false; - if ((clustermode.equals("privatepeer")) || (clustermode.equals("publicpeer"))) { - prop.put("commitRobinsonWithoutRemoteIndexing", "1"); - crawlResponse = false; - } - if ((clustermode.equals("privatecluster")) || (clustermode.equals("publiccluster"))) { - prop.put("commitRobinsonWithRemoteIndexing", "1"); - crawlResponse = true; - } - commit = 1; - } else { - if (!indexDistribute && !indexReceive) { - prop.put("commitDHTIsRobinson", "1"); - commit = 2; - } else if (indexDistribute && indexReceive) { - commit = 1; - } else { - prop.put("commitDHTNoGlobalSearch", "1"); - commit = 1; - } - if (!crawlResponse) { - prop.put("commitCrawlPlea", "1"); - } - } - if (indexDistribute) { - sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true"); - } else { - sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "false"); - } - - if (post.get("indexDistributeWhileCrawling","").equals("on")) { - sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW_WHILE_CRAWLING, "true"); - } else { - sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW_WHILE_CRAWLING, "false"); - } - - if (post.get("indexDistributeWhileIndexing","").equals("on")) { - sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW_WHILE_INDEXING, "true"); - } else { - sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW_WHILE_INDEXING, "false"); - } - - if (indexReceive) { - sb.setConfig("allowReceiveIndex", "true"); - sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(true); - } else { - sb.setConfig("allowReceiveIndex", "false"); - sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(false); - } - - if (post.get("indexReceiveBlockBlacklist", "").equals("on")) { - sb.setConfig("indexReceiveBlockBlacklist", "true"); - } else { - sb.setConfig("indexReceiveBlockBlacklist", "false"); - } - - if (post.containsKey("peertags")) { - sb.wordIndex.seedDB.mySeed().setPeerTags(serverCodings.string2set(normalizedList((String) post.get("peertags")), ",")); + if (post.containsKey("changeNetwork")) { + String networkDefinition = post.get("networkDefinition", "defaults/yacy.network.freeworld.unit"); + if (networkDefinition.equals(sb.getConfig("network.unit.definition", ""))) { + // no change + commit = 3; + } else { + // shut down old network and index, start up new network and index + commit = 1; + // pause crawls + boolean lcp = sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); + if (!lcp) sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); + boolean rcp = sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); + if (!rcp) sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); + // trigger online caution + sb.proxyLastAccess = System.currentTimeMillis() + 60000; // at least 1 minute online caution to prevent unnecessary action on database meanwhile + // switch the networks + synchronized (sb.webIndex) { + sb.webIndex.close(); + sb.setConfig("network.unit.definition", networkDefinition); + plasmaSwitchboard.overwriteNetworkDefinition(sb); + File indexPrimaryPath = sb.getConfigPath(plasmaSwitchboard.INDEX_PRIMARY_PATH, plasmaSwitchboard.INDEX_PATH_DEFAULT); + File indexSecondaryPath = (sb.getConfig(plasmaSwitchboard.INDEX_SECONDARY_PATH, "").length() == 0) ? indexPrimaryPath : new File(sb.getConfig(plasmaSwitchboard.INDEX_SECONDARY_PATH, "")); + sb.webIndex = new plasmaWordIndex(sb.getConfig("network.unit.name", ""), sb.getLog(), indexPrimaryPath, indexSecondaryPath); + } + // start up crawl jobs again + if (lcp) sb.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); + if (rcp) sb.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); + } } - sb.setConfig("cluster.mode", post.get("cluster.mode", "publicpeer")); - - // read remote crawl request settings - sb.setConfig("crawlResponse", (crawlResponse) ? "true" : "false"); - int newppm = 1; - try { - newppm = Math.max(1, Integer.parseInt(post.get("acceptCrawlLimit", "1"))); - } catch (NumberFormatException e) {} - long newBusySleep = Math.max(100, 60000 / newppm); - serverBusyThread rct = sb.getThread(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); - rct.setBusySleep(newBusySleep); - sb.setConfig(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, Long.toString(newBusySleep)); - - sb.setConfig("cluster.peers.ipport", checkIPPortList(post.get("cluster.peers.ipport", ""))); - sb.setConfig("cluster.peers.yacydomain", checkYaCyDomainList(post.get("cluster.peers.yacydomain", ""))); - - // update the cluster hash set - sb.clusterhashes = sb.wordIndex.seedDB.clusterHashes(sb.getConfig("cluster.peers.yacydomain", "")); - + if (post.containsKey("save")) { + boolean crawlResponse = post.get("crawlResponse", "off").equals("on"); + + // DHT control + boolean indexDistribute = post.get("indexDistribute", "").equals("on"); + boolean indexReceive = post.get("indexReceive", "").equals("on"); + boolean robinsonmode = post.get("network", "").equals("robinson"); + String clustermode = post.get("cluster.mode", "publicpeer"); + if (robinsonmode) { + indexDistribute = false; + indexReceive = false; + if ((clustermode.equals("privatepeer")) || (clustermode.equals("publicpeer"))) { + prop.put("commitRobinsonWithoutRemoteIndexing", "1"); + crawlResponse = false; + } + if ((clustermode.equals("privatecluster")) || (clustermode.equals("publiccluster"))) { + prop.put("commitRobinsonWithRemoteIndexing", "1"); + crawlResponse = true; + } + commit = 1; + } else { + if (!indexDistribute && !indexReceive) { + prop.put("commitDHTIsRobinson", "1"); + commit = 2; + } else if (indexDistribute && indexReceive) { + commit = 1; + } else { + prop.put("commitDHTNoGlobalSearch", "1"); + commit = 1; + } + if (!crawlResponse) { + prop.put("commitCrawlPlea", "1"); + } + } + + if (indexDistribute) { + sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "true"); + } else { + sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW, "false"); + } + + if (post.get("indexDistributeWhileCrawling","").equals("on")) { + sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW_WHILE_CRAWLING, "true"); + } else { + sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW_WHILE_CRAWLING, "false"); + } + + if (post.get("indexDistributeWhileIndexing","").equals("on")) { + sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW_WHILE_INDEXING, "true"); + } else { + sb.setConfig(plasmaSwitchboard.INDEX_DIST_ALLOW_WHILE_INDEXING, "false"); + } + + if (indexReceive) { + sb.setConfig("allowReceiveIndex", "true"); + sb.webIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(true); + } else { + sb.setConfig("allowReceiveIndex", "false"); + sb.webIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(false); + } + + if (post.get("indexReceiveBlockBlacklist", "").equals("on")) { + sb.setConfig("indexReceiveBlockBlacklist", "true"); + } else { + sb.setConfig("indexReceiveBlockBlacklist", "false"); + } + + if (post.containsKey("peertags")) { + sb.webIndex.seedDB.mySeed().setPeerTags(serverCodings.string2set(normalizedList((String) post.get("peertags")), ",")); + } + + sb.setConfig("cluster.mode", post.get("cluster.mode", "publicpeer")); + + // read remote crawl request settings + sb.setConfig("crawlResponse", (crawlResponse) ? "true" : "false"); + int newppm = 1; + try { + newppm = Math.max(1, Integer.parseInt(post.get("acceptCrawlLimit", "1"))); + } catch (NumberFormatException e) {} + long newBusySleep = Math.max(100, 60000 / newppm); + serverBusyThread rct = sb.getThread(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); + rct.setBusySleep(newBusySleep); + sb.setConfig(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, Long.toString(newBusySleep)); + + sb.setConfig("cluster.peers.ipport", checkIPPortList(post.get("cluster.peers.ipport", ""))); + sb.setConfig("cluster.peers.yacydomain", checkYaCyDomainList(post.get("cluster.peers.yacydomain", ""))); + + // update the cluster hash set + sb.clusterhashes = sb.webIndex.seedDB.clusterHashes(sb.getConfig("cluster.peers.yacydomain", "")); + } } // write answer code @@ -161,11 +196,11 @@ public class ConfigNetwork_p { prop.put("indexReceiveChecked", (indexReceive) ? "1" : "0"); prop.put("indexReceiveBlockBlacklistChecked.on", (sb.getConfig("indexReceiveBlockBlacklist", "true").equals("true")) ? "1" : "0"); prop.put("indexReceiveBlockBlacklistChecked.off", (sb.getConfig("indexReceiveBlockBlacklist", "true").equals("true")) ? "0" : "1"); - prop.putHTML("peertags", serverCodings.set2string(sb.wordIndex.seedDB.mySeed().getPeerTags(), ",", false)); + prop.putHTML("peertags", serverCodings.set2string(sb.webIndex.seedDB.mySeed().getPeerTags(), ",", false)); // set seed information directly - sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteCrawl(sb.getConfigBool("crawlResponse", false)); - sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(indexReceive); + sb.webIndex.seedDB.mySeed().setFlagAcceptRemoteCrawl(sb.getConfigBool("crawlResponse", false)); + sb.webIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(indexReceive); // set p2p/robinson mode flags and values prop.put("p2p.checked", (indexDistribute || indexReceive) ? "1" : "0"); @@ -173,42 +208,54 @@ public class ConfigNetwork_p { prop.put("cluster.peers.ipport", sb.getConfig("cluster.peers.ipport", "")); prop.put("cluster.peers.yacydomain", sb.getConfig("cluster.peers.yacydomain", "")); prop.put("cluster.peers.yacydomain.hashes", (sb.clusterhashes.size() == 0) ? "" : sb.clusterhashes.toString()); + // set p2p mode flags prop.put("privatepeerChecked", (sb.getConfig("cluster.mode", "").equals("privatepeer")) ? "1" : "0"); prop.put("privateclusterChecked", (sb.getConfig("cluster.mode", "").equals("privatecluster")) ? "1" : "0"); prop.put("publicclusterChecked", (sb.getConfig("cluster.mode", "").equals("publiccluster")) ? "1" : "0"); prop.put("publicpeerChecked", (sb.getConfig("cluster.mode", "").equals("publicpeer")) ? "1" : "0"); + // set network configuration + prop.put("network.unit.definition", sb.getConfig("network.unit.definition", "")); + prop.put("network.unit.name", sb.getConfig("network.unit.name", "")); + prop.put("network.unit.description", sb.getConfig("network.unit.description", "")); + prop.put("network.unit.domain", sb.getConfig("network.unit.domain", "")); + prop.put("network.unit.dht", sb.getConfig("network.unit.dht", "")); + networkBootstrapLocations.remove(sb.getConfig("network.unit.definition", "")); + int c = 0; + for (String s: networkBootstrapLocations) prop.put("networks_" + c++ + "_network", s); + prop.put("networks", c); + return prop; - } - - public static String normalizedList(String input) { - input = input.replace(' ', ','); - input = input.replace(' ', ';'); - input = input.replaceAll(",,", ","); - if (input.startsWith(",")) input = input.substring(1); - if (input.endsWith(",")) input = input.substring(0, input.length() - 1); - return input; - } - - public static String checkYaCyDomainList(String input) { - input = normalizedList(input); - String[] s = input.split(","); - input = ""; - for (int i = 0; i < s.length; i++) { - if ((s[i].endsWith(".yacyh")) || (s[i].endsWith(".yacy")) || - (s[i].indexOf(".yacyh=") > 0) || (s[i].indexOf(".yacy=") > 0)) input += "," + s[i]; - } - if (input.length() == 0) return input; else return input.substring(1); - } - - public static String checkIPPortList(String input) { - input = normalizedList(input); - String[] s = input.split(","); - input = ""; - for (int i = 0; i < s.length; i++) { - if (s[i].indexOf(':') >= 9) input += "," + s[i]; - } - if (input.length() == 0) return input; else return input.substring(1); - } + } + + public static String normalizedList(String input) { + input = input.replace(' ', ','); + input = input.replace(' ', ';'); + input = input.replaceAll(",,", ","); + if (input.startsWith(",")) input = input.substring(1); + if (input.endsWith(",")) input = input.substring(0, input.length() - 1); + return input; + } + + public static String checkYaCyDomainList(String input) { + input = normalizedList(input); + String[] s = input.split(","); + input = ""; + for (int i = 0; i < s.length; i++) { + if ((s[i].endsWith(".yacyh")) || (s[i].endsWith(".yacy")) || + (s[i].indexOf(".yacyh=") > 0) || (s[i].indexOf(".yacy=") > 0)) input += "," + s[i]; + } + if (input.length() == 0) return input; else return input.substring(1); + } + + public static String checkIPPortList(String input) { + input = normalizedList(input); + String[] s = input.split(","); + input = ""; + for (int i = 0; i < s.length; i++) { + if (s[i].indexOf(':') >= 9) input += "," + s[i]; + } + if (input.length() == 0) return input; else return input.substring(1); + } } diff --git a/htroot/ConfigProfile_p.java b/htroot/ConfigProfile_p.java index 74a49b4d3..0b3e06481 100644 --- a/htroot/ConfigProfile_p.java +++ b/htroot/ConfigProfile_p.java @@ -113,7 +113,7 @@ public class ConfigProfile_p { // generate a news message
Properties news = profile;
news.remove("comment");
- sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_PROFILE_UPDATE, news));
+ sb.webIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_PROFILE_UPDATE, news));
//yacyCore.newsPool.publishMyNews(new yacyNewsRecord(yacyNewsRecord.CATEGORY_PROFILE_UPDATE, profile));
} catch(IOException e) {
} finally {
diff --git a/htroot/ConfigRobotsTxt_p.java b/htroot/ConfigRobotsTxt_p.java index 077e05142..21d43116e 100644 --- a/htroot/ConfigRobotsTxt_p.java +++ b/htroot/ConfigRobotsTxt_p.java @@ -62,7 +62,7 @@ public class ConfigRobotsTxt_p { final servletProperties prop = new servletProperties(); httpdRobotsTxtConfig rbc = ((plasmaSwitchboard)env).robotstxtConfig; - prop.put("clientname", sb.wordIndex.seedDB.mySeed().getPublicAddress()); + prop.put("clientname", sb.webIndex.seedDB.mySeed().getPublicAddress()); if (post != null) { if (post.containsKey("save")) { diff --git a/htroot/Connections_p.java b/htroot/Connections_p.java index b90bfb584..8b647cb02 100644 --- a/htroot/Connections_p.java +++ b/htroot/Connections_p.java @@ -176,14 +176,14 @@ public final class Connections_p { commandLine = urlRedir.getURL();
}
- if ((dest != null) && (dest.equals(virtualHost))) dest = sb.wordIndex.seedDB.mySeed().getName() + ".yacy";
+ if ((dest != null) && (dest.equals(virtualHost))) dest = sb.webIndex.seedDB.mySeed().getName() + ".yacy";
// determining if the source is a yacy host
yacySeed seed = null;
if (doNameLookup) {
- seed = sb.wordIndex.seedDB.lookupByIP(userAddress,true,false,false);
+ seed = sb.webIndex.seedDB.lookupByIP(userAddress,true,false,false);
if (seed != null) {
- if ((seed.hash.equals(sb.wordIndex.seedDB.mySeed().hash)) &&
+ if ((seed.hash.equals(sb.webIndex.seedDB.mySeed().hash)) &&
(!seed.get(yacySeed.PORT,"").equals(Integer.toString(userPort)))) {
seed = null;
}
diff --git a/htroot/CrawlProfileEditor_p.java b/htroot/CrawlProfileEditor_p.java index ad0b6777d..e9bc38e9a 100644 --- a/htroot/CrawlProfileEditor_p.java +++ b/htroot/CrawlProfileEditor_p.java @@ -32,6 +32,7 @@ import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.CrawlProfile.entry; import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.plasma.plasmaWordIndex; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.servletProperties; @@ -87,34 +88,34 @@ public class CrawlProfileEditor_p { if (post != null) { if (post.containsKey("terminate")) { // termination of a crawl: shift the crawl from active to passive - CrawlProfile.entry entry = sb.profilesActiveCrawls.getEntry(handle); - if (entry != null) sb.profilesPassiveCrawls.newEntry(entry.map()); - sb.profilesActiveCrawls.removeEntry(handle); + CrawlProfile.entry entry = sb.webIndex.profilesActiveCrawls.getEntry(handle); + if (entry != null) sb.webIndex.profilesPassiveCrawls.newEntry(entry.map()); + sb.webIndex.profilesActiveCrawls.removeEntry(handle); // delete all entries from the crawl queue that are deleted here sb.crawlQueues.noticeURL.removeByProfileHandle(handle, 10000); } if (post.containsKey("delete")) { // deletion of a terminated crawl profile - sb.profilesPassiveCrawls.removeEntry(handle); + sb.webIndex.profilesPassiveCrawls.removeEntry(handle); } if (post.containsKey("deleteTerminatedProfiles")) { - Iterator<CrawlProfile.entry> profiles = sb.profilesPassiveCrawls.profiles(false); + Iterator<CrawlProfile.entry> profiles = sb.webIndex.profilesPassiveCrawls.profiles(false); while (profiles.hasNext()) { profiles.next(); profiles.remove(); - profiles = sb.profilesPassiveCrawls.profiles(false); + profiles = sb.webIndex.profilesPassiveCrawls.profiles(false); } } } // generate handle list int count = 0; - Iterator<CrawlProfile.entry> it = sb.profilesActiveCrawls.profiles(true); + Iterator<CrawlProfile.entry> it = sb.webIndex.profilesActiveCrawls.profiles(true); entry selentry; while (it.hasNext()) { selentry = it.next(); - if (selentry.name().equals(plasmaSwitchboard.CRAWL_PROFILE_PROXY) || - selentry.name().equals(plasmaSwitchboard.CRAWL_PROFILE_REMOTE) /*|| + if (selentry.name().equals(plasmaWordIndex.CRAWL_PROFILE_PROXY) || + selentry.name().equals(plasmaWordIndex.CRAWL_PROFILE_REMOTE) /*|| selentry.name().equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_TEXT) || selentry.name().equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_MEDIA)*/) continue; @@ -125,7 +126,7 @@ public class CrawlProfileEditor_p { count++; } prop.put("profiles", count); - selentry = sb.profilesActiveCrawls.getEntry(handle); + selentry = sb.webIndex.profilesActiveCrawls.getEntry(handle); // read post for change submit if ((post != null) && (selentry != null)) { @@ -137,7 +138,7 @@ public class CrawlProfileEditor_p { tee = lit.next(); String cval = (String) selentry.map().get(tee.name); String val = (tee.type == eentry.BOOLEAN) ? Boolean.toString(post.containsKey(tee.name)) : post.get(tee.name, cval); - if (!cval.equals(val)) sb.profilesActiveCrawls.changeEntry(selentry, tee.name, val); + if (!cval.equals(val)) sb.webIndex.profilesActiveCrawls.changeEntry(selentry, tee.name, val); } } catch (IOException ex) { prop.put("error", "1"); @@ -152,7 +153,7 @@ public class CrawlProfileEditor_p { int domlistlength = (post == null) ? 160 : post.getInt("domlistlength", 160); CrawlProfile.entry profile; // put active crawls into list - it = sb.profilesActiveCrawls.profiles(true); + it = sb.webIndex.profilesActiveCrawls.profiles(true); while (it.hasNext()) { profile = it.next(); putProfileEntry(prop, profile, true, dark, count, domlistlength); @@ -161,7 +162,7 @@ public class CrawlProfileEditor_p { } // put passive crawls into list boolean existPassiveCrawls = false; - it = sb.profilesPassiveCrawls.profiles(true); + it = sb.webIndex.profilesPassiveCrawls.profiles(true); while (it.hasNext()) { profile = it.next(); putProfileEntry(prop, profile, false, dark, count, domlistlength); diff --git a/htroot/CrawlResults.java b/htroot/CrawlResults.java index c19cffb69..a03fdfcb0 100644 --- a/htroot/CrawlResults.java +++ b/htroot/CrawlResults.java @@ -102,7 +102,7 @@ public class CrawlResults { String hash = post.get("hash", null);
if (hash != null) {
// delete from database
- sb.wordIndex.removeURL(hash);
+ sb.webIndex.removeURL(hash);
}
}
if (post.containsKey("moreIndexed")) {
@@ -166,11 +166,11 @@ public class CrawlResults { urlHash = sb.crawlResults.getUrlHash(tabletype, i);
// serverLog.logFinest("PLASMA", "plasmaCrawlLURL/genTableProps urlHash=" + urlHash);
try {
- urle = sb.wordIndex.getURL(urlHash, null, 0);
+ urle = sb.webIndex.getURL(urlHash, null, 0);
indexURLReference.Components comp = urle.comp();
// serverLog.logFinest("PLASMA", "plasmaCrawlLURL/genTableProps urle=" + urle.toString());
- initiatorSeed = sb.wordIndex.seedDB.getConnected(initiatorHash);
- executorSeed = sb.wordIndex.seedDB.getConnected(executorHash);
+ initiatorSeed = sb.webIndex.seedDB.getConnected(initiatorHash);
+ executorSeed = sb.webIndex.seedDB.getConnected(executorHash);
urlstr = comp.url().toNormalform(false, true);
urltxt = nxTools.shortenURLString(urlstr, 72); // shorten the string text like a URL
diff --git a/htroot/CrawlStartSimple_p.java b/htroot/CrawlStartSimple_p.java index ac304eb4b..2333575bc 100644 --- a/htroot/CrawlStartSimple_p.java +++ b/htroot/CrawlStartSimple_p.java @@ -100,7 +100,7 @@ public class CrawlStartSimple_p { boolean dark = true; // create other peer crawl table using YaCyNews - Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true); + Iterator<yacyNewsRecord> recordIterator = sb.webIndex.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true); int showedCrawl = 0; yacyNewsRecord record; yacySeed peer; @@ -109,7 +109,7 @@ public class CrawlStartSimple_p { record = recordIterator.next(); if (record == null) continue; if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) { - peer = sb.wordIndex.seedDB.get(record.originator()); + peer = sb.webIndex.seedDB.get(record.originator()); if (peer == null) peername = record.originator(); else peername = peer.getName(); prop.put("otherCrawlStartInProgress_" + showedCrawl + "_dark", dark ? "1" : "0"); prop.put("otherCrawlStartInProgress_" + showedCrawl + "_cre", record.created().toString()); @@ -125,13 +125,13 @@ public class CrawlStartSimple_p { prop.put("otherCrawlStartInProgress", showedCrawl); // finished remote crawls - recordIterator = sb.wordIndex.newsPool.recordIterator(yacyNewsPool.PROCESSED_DB, true); + recordIterator = sb.webIndex.newsPool.recordIterator(yacyNewsPool.PROCESSED_DB, true); showedCrawl = 0; while (recordIterator.hasNext()) { record = (yacyNewsRecord) recordIterator.next(); if (record == null) continue; if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) { - peer = sb.wordIndex.seedDB.get(record.originator()); + peer = sb.webIndex.seedDB.get(record.originator()); if (peer == null) peername = record.originator(); else peername = peer.getName(); prop.put("otherCrawlStartFinished_" + showedCrawl + "_dark", dark ? "1" : "0"); prop.put("otherCrawlStartFinished_" + showedCrawl + "_cre", record.created().toString()); @@ -148,7 +148,7 @@ public class CrawlStartSimple_p { // remote crawl peers - if ((sb.wordIndex.seedDB == null) || (sb.wordIndex.seedDB.mySeed().isVirgin()) || (sb.wordIndex.seedDB.mySeed().isJunior())) { + if ((sb.webIndex.seedDB == null) || (sb.webIndex.seedDB.mySeed().isVirgin()) || (sb.webIndex.seedDB.mySeed().isJunior())) { prop.put("remoteCrawlPeers", "0"); } else { Iterator<yacySeed> crawlavail = yacyCore.peerActions.dhtAction.getAcceptRemoteCrawlSeeds(null, true); diff --git a/htroot/IndexCleaner_p.java b/htroot/IndexCleaner_p.java index 270390700..d25ac010a 100755 --- a/htroot/IndexCleaner_p.java +++ b/htroot/IndexCleaner_p.java @@ -62,7 +62,7 @@ public class IndexCleaner_p { //prop.putHTML("bla", "post!=null");
if (post.get("action").equals("ustart")) {
if (urldbCleanerThread==null || !urldbCleanerThread.isAlive()) {
- urldbCleanerThread = sb.wordIndex.getURLCleaner(plasmaSwitchboard.urlBlacklist);
+ urldbCleanerThread = sb.webIndex.getURLCleaner(plasmaSwitchboard.urlBlacklist);
urldbCleanerThread.start();
}
else {
@@ -77,7 +77,7 @@ public class IndexCleaner_p { }
else if (post.get("action").equals("rstart")) {
if (indexCleanerThread==null || !indexCleanerThread.isAlive()) {
- indexCleanerThread = sb.wordIndex.getReferenceCleaner(post.get("wordHash","AAAAAAAAAAAA"));
+ indexCleanerThread = sb.webIndex.getReferenceCleaner(post.get("wordHash","AAAAAAAAAAAA"));
indexCleanerThread.start();
}
else {
@@ -96,7 +96,7 @@ public class IndexCleaner_p { //prop.put("bla", "post==null");
if (urldbCleanerThread!=null) {
prop.put("urldb", "1");
- prop.putNum("urldb_percentUrls", ((double)urldbCleanerThread.totalSearchedUrls/sb.wordIndex.countURL())*100);
+ prop.putNum("urldb_percentUrls", ((double)urldbCleanerThread.totalSearchedUrls/sb.webIndex.countURL())*100);
prop.putNum("urldb_blacklisted", urldbCleanerThread.blacklistedUrls);
prop.putNum("urldb_total", urldbCleanerThread.totalSearchedUrls);
prop.putHTML("urldb_lastBlacklistedUrl", urldbCleanerThread.lastBlacklistedUrl);
@@ -113,7 +113,7 @@ public class IndexCleaner_p { prop.put("rwidb_threadAlive", indexCleanerThread.isAlive() + "");
prop.put("rwidb_threadToString", indexCleanerThread.toString());
prop.putNum("rwidb_RWIcountstart", indexCleanerThread.rwiCountAtStart);
- prop.putNum("rwidb_RWIcountnow", sb.wordIndex.size());
+ prop.putNum("rwidb_RWIcountnow", sb.webIndex.size());
prop.put("rwidb_wordHashNow", indexCleanerThread.wordHashNow);
prop.put("rwidb_lastWordHash", indexCleanerThread.lastWordHash);
prop.putNum("rwidb_lastDeletionCounter", indexCleanerThread.lastDeletionCounter);
diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index 4733cb2b8..a274bb6a2 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -110,7 +110,7 @@ public class IndexControlRWIs_p { if (delurl || delurlref) { // generate an urlx array indexContainer index = null; - index = sb.wordIndex.getContainer(keyhash, null); + index = sb.webIndex.getContainer(keyhash, null); Iterator<indexRWIRowEntry> en = index.entries(); int i = 0; urlx = new String[index.size()]; @@ -127,7 +127,7 @@ public class IndexControlRWIs_p { sb.urlRemove(urlx[i]); } } - sb.wordIndex.deleteContainer(keyhash); + sb.webIndex.deleteContainer(keyhash); post.remove("keyhashdeleteall"); post.put("urllist", "generated"); } @@ -144,7 +144,7 @@ public class IndexControlRWIs_p { } Set<String> urlHashes = new HashSet<String>(); for (int i = 0; i < urlx.length; i++) urlHashes.add(urlx[i]); - sb.wordIndex.removeEntries(keyhash, urlHashes); + sb.webIndex.removeEntries(keyhash, urlHashes); // this shall lead to a presentation of the list; so handle that the remaining program // thinks that it was called for a list presentation post.remove("keyhashdelete"); @@ -173,21 +173,21 @@ public class IndexControlRWIs_p { if (host.length() != 0) { if (host.length() == 12) { // the host string is a peer hash - seed = sb.wordIndex.seedDB.getConnected(host); + seed = sb.webIndex.seedDB.getConnected(host); } else { // the host string can be a host name - seed = sb.wordIndex.seedDB.lookupByName(host); + seed = sb.webIndex.seedDB.lookupByName(host); } } else { host = post.get("hostHash", ""); // if input field is empty, get from select box - seed = sb.wordIndex.seedDB.getConnected(host); + seed = sb.webIndex.seedDB.getConnected(host); } // prepare index indexContainer index; String result; long starttime = System.currentTimeMillis(); - index = sb.wordIndex.getContainer(keyhash, null); + index = sb.webIndex.getContainer(keyhash, null); // built urlCache Iterator<indexRWIRowEntry> urlIter = index.entries(); HashMap<String, indexURLReference> knownURLs = new HashMap<String, indexURLReference>(); @@ -196,7 +196,7 @@ public class IndexControlRWIs_p { indexURLReference lurl; while (urlIter.hasNext()) { iEntry = urlIter.next(); - lurl = sb.wordIndex.getURL(iEntry.urlHash(), null, 0); + lurl = sb.webIndex.getURL(iEntry.urlHash(), null, 0); if (lurl == null) { unknownURLEntries.add(iEntry.urlHash()); urlIter.remove(); @@ -209,7 +209,7 @@ public class IndexControlRWIs_p { String gzipBody = sb.getConfig("indexControl.gzipBody","false"); int timeout = (int) sb.getConfigLong("indexControl.timeout",60000); HashMap<String, Object> resultObj = yacyClient.transferIndex( - sb.wordIndex.seedDB, + sb.webIndex.seedDB, seed, new indexContainer[]{index}, knownURLs, @@ -222,7 +222,7 @@ public class IndexControlRWIs_p { // generate list if (post.containsKey("keyhashsimilar")) { - final Iterator<indexContainer> containerIt = sb.wordIndex.indexContainerSet(keyhash, false, true, 256).iterator(); + final Iterator<indexContainer> containerIt = sb.webIndex.indexContainerSet(keyhash, false, true, 256).iterator(); indexContainer container; int i = 0; int rows = 0, cols = 0; @@ -254,8 +254,8 @@ public class IndexControlRWIs_p { yacyURL url; for (int i=0; i<urlx.length; i++) { urlHashes.add(urlx[i]); - indexURLReference e = sb.wordIndex.getURL(urlx[i], null, 0); - sb.wordIndex.removeURL(urlx[i]); + indexURLReference e = sb.webIndex.getURL(urlx[i], null, 0); + sb.webIndex.removeURL(urlx[i]); if (e != null) { url = e.comp().url(); pw.println(url.getHost() + "/" + url.getFile()); @@ -282,8 +282,8 @@ public class IndexControlRWIs_p { yacyURL url; for (int i=0; i<urlx.length; i++) { urlHashes.add(urlx[i]); - indexURLReference e = sb.wordIndex.getURL(urlx[i], null, 0); - sb.wordIndex.removeURL(urlx[i]); + indexURLReference e = sb.webIndex.getURL(urlx[i], null, 0); + sb.webIndex.removeURL(urlx[i]); if (e != null) { url = e.comp().url(); pw.println(url.getHost() + "/.*"); @@ -300,7 +300,7 @@ public class IndexControlRWIs_p { } catch (IOException e) { } } - sb.wordIndex.removeEntries(keyhash, urlHashes); + sb.webIndex.removeEntries(keyhash, urlHashes); } if (prop.getInt("searchresult", 0) == 3) plasmaSearchAPI.listHosts(prop, keyhash); @@ -308,7 +308,7 @@ public class IndexControlRWIs_p { // insert constants - prop.putNum("wcount", sb.wordIndex.size()); + prop.putNum("wcount", sb.webIndex.size()); // return rewrite properties return prop; } diff --git a/htroot/IndexControlURLs_p.java b/htroot/IndexControlURLs_p.java index b04433c5b..bb57ceb0d 100644 --- a/htroot/IndexControlURLs_p.java +++ b/htroot/IndexControlURLs_p.java @@ -52,10 +52,10 @@ public class IndexControlURLs_p { prop.put("urlstring", ""); prop.put("urlhash", ""); prop.put("result", ""); - prop.put("ucount", Integer.toString(sb.wordIndex.countURL())); + prop.put("ucount", Integer.toString(sb.webIndex.countURL())); prop.put("otherHosts", ""); - indexRepositoryReference.Export export = sb.wordIndex.exportURL(); + indexRepositoryReference.Export export = sb.webIndex.exportURL(); if ((export != null) && (export.isAlive())) { // there is currently a running export prop.put("lurlexportfinished", 0); @@ -103,7 +103,7 @@ public class IndexControlURLs_p { } if (post.containsKey("urlhashdelete")) { - indexURLReference entry = sb.wordIndex.getURL(urlhash, null, 0); + indexURLReference entry = sb.webIndex.getURL(urlhash, null, 0); if (entry == null) { prop.put("result", "No Entry for URL hash " + urlhash + "; nothing deleted."); } else { @@ -135,7 +135,7 @@ public class IndexControlURLs_p { yacyURL url = new yacyURL(urlstring, null); urlhash = url.hash(); prop.put("urlhash", urlhash); - indexURLReference entry = sb.wordIndex.getURL(urlhash, null, 0); + indexURLReference entry = sb.webIndex.getURL(urlhash, null, 0); if (entry == null) { prop.putHTML("urlstring", "unknown url: " + urlstring); prop.put("urlhash", ""); @@ -150,7 +150,7 @@ public class IndexControlURLs_p { } if (post.containsKey("urlhashsearch")) { - indexURLReference entry = sb.wordIndex.getURL(urlhash, null, 0); + indexURLReference entry = sb.webIndex.getURL(urlhash, null, 0); if (entry == null) { prop.put("result", "No Entry for URL hash " + urlhash); } else { @@ -163,7 +163,7 @@ public class IndexControlURLs_p { // generate list if (post.containsKey("urlhashsimilar")) { try { - final Iterator<indexURLReference> entryIt = new kelondroRotateIterator<indexURLReference>(sb.wordIndex.entriesURL(true, urlhash), new String(kelondroBase64Order.zero(urlhash.length())), sb.wordIndex.size()); + final Iterator<indexURLReference> entryIt = new kelondroRotateIterator<indexURLReference>(sb.webIndex.entriesURL(true, urlhash), new String(kelondroBase64Order.zero(urlhash.length())), sb.webIndex.size()); StringBuffer result = new StringBuffer("Sequential List of URL-Hashes:<br />"); indexURLReference entry; int i = 0; @@ -208,7 +208,7 @@ public class IndexControlURLs_p { File f = new File(s); f.getParentFile().mkdirs(); String filter = post.get("exportfilter", ".*"); - indexRepositoryReference.Export running = sb.wordIndex.exportURL(f, filter, format, dom); + indexRepositoryReference.Export running = sb.webIndex.exportURL(f, filter, format, dom); prop.put("lurlexport_exportfile", s); prop.put("lurlexport_urlcount", running.count()); @@ -218,7 +218,7 @@ public class IndexControlURLs_p { } // insert constants - prop.putNum("ucount", sb.wordIndex.countURL()); + prop.putNum("ucount", sb.webIndex.countURL()); // return rewrite properties return prop; } @@ -231,7 +231,7 @@ public class IndexControlURLs_p { return prop; } indexURLReference.Components comp = entry.comp(); - indexURLReference le = ((entry.referrerHash() == null) || (entry.referrerHash().length() != yacySeedDB.commonHashLength)) ? null : switchboard.wordIndex.getURL(entry.referrerHash(), null, 0); + indexURLReference le = ((entry.referrerHash() == null) || (entry.referrerHash().length() != yacySeedDB.commonHashLength)) ? null : switchboard.webIndex.getURL(entry.referrerHash(), null, 0); if (comp.url() == null) { prop.put("genUrlProfile", "1"); prop.put("genUrlProfile_urlhash", urlhash); diff --git a/htroot/IndexCreateIndexingQueue_p.java b/htroot/IndexCreateIndexingQueue_p.java index c12821c7e..6e0c691ce 100644 --- a/htroot/IndexCreateIndexingQueue_p.java +++ b/htroot/IndexCreateIndexingQueue_p.java @@ -47,10 +47,10 @@ import java.util.ArrayList; import java.util.Iterator;
import de.anomic.crawler.ZURL;
+import de.anomic.crawler.IndexingStack;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.plasma.plasmaSwitchboardQueue;
import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
@@ -82,20 +82,20 @@ public class IndexCreateIndexingQueue_p { }
if (post.containsKey("clearIndexingQueue")) {
try {
- synchronized (sb.sbQueue) {
- plasmaSwitchboardQueue.QueueEntry entry = null;
- while ((entry = sb.sbQueue.pop()) != null) {
+ synchronized (sb.webIndex.queuePreStack) {
+ IndexingStack.QueueEntry entry = null;
+ while ((entry = sb.webIndex.queuePreStack.pop()) != null) {
if ((entry != null) && (entry.profile() != null) && (!(entry.profile().storeHTCache()))) {
plasmaHTCache.deleteURLfromCache(entry.url());
}
}
- sb.sbQueue.clear(); // reset file to clean up content completely
+ sb.webIndex.queuePreStack.clear(); // reset file to clean up content completely
}
} catch (Exception e) {}
} else if (post.containsKey("deleteEntry")) {
String urlHash = (String) post.get("deleteEntry");
try {
- sb.sbQueue.remove(urlHash);
+ sb.webIndex.queuePreStack.remove(urlHash);
} catch (Exception e) {}
prop.put("LOCATION","");
return prop;
@@ -105,24 +105,24 @@ public class IndexCreateIndexingQueue_p { yacySeed initiator;
boolean dark;
- if ((sb.sbQueue.size() == 0) && (sb.sbQueue.getActiveQueueSize() == 0)) {
+ if ((sb.webIndex.queuePreStack.size() == 0) && (sb.webIndex.queuePreStack.getActiveQueueSize() == 0)) {
prop.put("indexing-queue", "0"); //is empty
} else {
prop.put("indexing-queue", "1"); // there are entries in the queue or in process
dark = true;
- plasmaSwitchboardQueue.QueueEntry pcentry;
+ IndexingStack.QueueEntry pcentry;
int entryCount = 0, totalCount = 0;
long totalSize = 0;
// getting all entries that are currently in process
- ArrayList<plasmaSwitchboardQueue.QueueEntry> entryList = new ArrayList<plasmaSwitchboardQueue.QueueEntry>();
- entryList.addAll(sb.sbQueue.getActiveQueueEntries());
+ ArrayList<IndexingStack.QueueEntry> entryList = new ArrayList<IndexingStack.QueueEntry>();
+ entryList.addAll(sb.webIndex.queuePreStack.getActiveQueueEntries());
int inProcessCount = entryList.size();
// getting all enqueued entries
- if ((sb.sbQueue.size() > 0)) {
- Iterator<plasmaSwitchboardQueue.QueueEntry> i = sb.sbQueue.entryIterator(false);
+ if ((sb.webIndex.queuePreStack.size() > 0)) {
+ Iterator<IndexingStack.QueueEntry> i = sb.webIndex.queuePreStack.entryIterator(false);
while (i.hasNext()) entryList.add(i.next());
}
@@ -131,11 +131,11 @@ public class IndexCreateIndexingQueue_p { for (int i = 0; (i < count) && (entryCount < showLimit); i++) {
boolean inProcess = i < inProcessCount;
- pcentry = (plasmaSwitchboardQueue.QueueEntry) entryList.get(i);
+ pcentry = (IndexingStack.QueueEntry) entryList.get(i);
if ((pcentry != null)&&(pcentry.url() != null)) {
long entrySize = pcentry.size();
totalSize += entrySize;
- initiator = sb.wordIndex.seedDB.getConnected(pcentry.initiator());
+ initiator = sb.webIndex.seedDB.getConnected(pcentry.initiator());
prop.put("indexing-queue_list_"+entryCount+"_dark", inProcess ? "2" : (dark ? "1" : "0"));
prop.putHTML("indexing-queue_list_"+entryCount+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("indexing-queue_list_"+entryCount+"_depth", pcentry.depth());
@@ -182,8 +182,8 @@ public class IndexCreateIndexingQueue_p { initiatorHash = entry.initiator();
executorHash = entry.executor();
- initiatorSeed = sb.wordIndex.seedDB.getConnected(initiatorHash);
- executorSeed = sb.wordIndex.seedDB.getConnected(executorHash);
+ initiatorSeed = sb.webIndex.seedDB.getConnected(initiatorHash);
+ executorSeed = sb.webIndex.seedDB.getConnected(executorHash);
prop.putHTML("rejected_list_"+j+"_initiator", ((initiatorSeed == null) ? "proxy" : initiatorSeed.getName()));
prop.putHTML("rejected_list_"+j+"_executor", ((executorSeed == null) ? "proxy" : executorSeed.getName()));
prop.putHTML("rejected_list_"+j+"_url", url.toNormalform(false, true));
diff --git a/htroot/IndexCreateLoaderQueue_p.java b/htroot/IndexCreateLoaderQueue_p.java index 50ece5648..b989edb7c 100644 --- a/htroot/IndexCreateLoaderQueue_p.java +++ b/htroot/IndexCreateLoaderQueue_p.java @@ -69,7 +69,7 @@ public class IndexCreateLoaderQueue_p { for (int i = 0; i < w.length; i++) {
if (w[i] == null) continue;
- initiator = sb.wordIndex.seedDB.getConnected(w[i].initiator());
+ initiator = sb.webIndex.seedDB.getConnected(w[i].initiator());
prop.put("loader-set_list_"+count+"_dark", dark ? "1" : "0");
prop.putHTML("loader-set_list_"+count+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("loader-set_list_"+count+"_depth", w[i].depth());
diff --git a/htroot/IndexCreateWWWGlobalQueue_p.java b/htroot/IndexCreateWWWGlobalQueue_p.java index 031f87680..4fef0d421 100644 --- a/htroot/IndexCreateWWWGlobalQueue_p.java +++ b/htroot/IndexCreateWWWGlobalQueue_p.java @@ -114,9 +114,9 @@ public class IndexCreateWWWGlobalQueue_p { for (i = 0; (i < crawlerList.length) && (showNum < showLimit); i++) {
urle = crawlerList[i];
if ((urle != null)&&(urle.url()!=null)) {
- initiator = sb.wordIndex.seedDB.getConnected(urle.initiator());
+ initiator = sb.webIndex.seedDB.getConnected(urle.initiator());
profileHandle = urle.profileHandle();
- profileEntry = (profileHandle == null) ? null : sb.profilesActiveCrawls.getEntry(profileHandle);
+ profileEntry = (profileHandle == null) ? null : sb.webIndex.profilesActiveCrawls.getEntry(profileHandle);
prop.put("crawler-queue_list_"+showNum+"_dark", dark ? "1" : "0");
prop.putHTML("crawler-queue_list_"+showNum+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()) );
prop.put("crawler-queue_list_"+showNum+"_profile", ((profileEntry == null) ? "unknown" : profileEntry.name()));
diff --git a/htroot/IndexCreateWWWLocalQueue_p.java b/htroot/IndexCreateWWWLocalQueue_p.java index e9778fcd6..864a1565f 100644 --- a/htroot/IndexCreateWWWLocalQueue_p.java +++ b/htroot/IndexCreateWWWLocalQueue_p.java @@ -56,6 +56,7 @@ import de.anomic.crawler.CrawlProfile; import de.anomic.crawler.NoticedURL;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.plasma.plasmaWordIndex;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacySeed;
@@ -107,20 +108,20 @@ public class IndexCreateWWWLocalQueue_p { if (option == PROFILE) {
// search and delete the crawl profile (_much_ faster, independant of queue size)
// XXX: what to do about the annoying LOST PROFILE messages in the log?
- Iterator<CrawlProfile.entry> it = sb.profilesActiveCrawls.profiles(true);
+ Iterator<CrawlProfile.entry> it = sb.webIndex.profilesActiveCrawls.profiles(true);
CrawlProfile.entry entry;
while (it.hasNext()) {
entry = it.next();
final String name = entry.name();
- if (name.equals(plasmaSwitchboard.CRAWL_PROFILE_PROXY) ||
- name.equals(plasmaSwitchboard.CRAWL_PROFILE_REMOTE) ||
- name.equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_LOCAL_TEXT) ||
- name.equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT) ||
- name.equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA) ||
- name.equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA))
+ if (name.equals(plasmaWordIndex.CRAWL_PROFILE_PROXY) ||
+ name.equals(plasmaWordIndex.CRAWL_PROFILE_REMOTE) ||
+ name.equals(plasmaWordIndex.CRAWL_PROFILE_SNIPPET_LOCAL_TEXT) ||
+ name.equals(plasmaWordIndex.CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT) ||
+ name.equals(plasmaWordIndex.CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA) ||
+ name.equals(plasmaWordIndex.CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA))
continue;
if (compiledPattern.matcher(name).find()) {
- sb.profilesActiveCrawls.removeEntry(entry.handle());
+ sb.webIndex.profilesActiveCrawls.removeEntry(entry.handle());
}
}
} else {
@@ -181,9 +182,9 @@ public class IndexCreateWWWLocalQueue_p { for (i = 0; (i < crawlerList.length) && (showNum < showLimit); i++) {
urle = crawlerList[i];
if ((urle != null)&&(urle.url()!=null)) {
- initiator = sb.wordIndex.seedDB.getConnected(urle.initiator());
+ initiator = sb.webIndex.seedDB.getConnected(urle.initiator());
profileHandle = urle.profileHandle();
- profileEntry = (profileHandle == null) ? null : sb.profilesActiveCrawls.getEntry(profileHandle);
+ profileEntry = (profileHandle == null) ? null : sb.webIndex.profilesActiveCrawls.getEntry(profileHandle);
prop.put("crawler-queue_list_"+showNum+"_dark", dark ? "1" : "0");
prop.putHTML("crawler-queue_list_"+showNum+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()) );
prop.put("crawler-queue_list_"+showNum+"_profile", ((profileEntry == null) ? "unknown" : profileEntry.name()));
diff --git a/htroot/IndexCreateWWWRemoteQueue_p.java b/htroot/IndexCreateWWWRemoteQueue_p.java index d9cde963b..c83726709 100644 --- a/htroot/IndexCreateWWWRemoteQueue_p.java +++ b/htroot/IndexCreateWWWRemoteQueue_p.java @@ -114,9 +114,9 @@ public class IndexCreateWWWRemoteQueue_p { for (i = 0; (i < crawlerList.length) && (showNum < showLimit); i++) { urle = crawlerList[i]; if (urle != null && urle.url() != null) { - initiator = sb.wordIndex.seedDB.getConnected(urle.initiator()); + initiator = sb.webIndex.seedDB.getConnected(urle.initiator()); profileHandle = urle.profileHandle(); - profileEntry = (profileHandle == null) ? null : sb.profilesActiveCrawls.getEntry(profileHandle); + profileEntry = (profileHandle == null) ? null : sb.webIndex.profilesActiveCrawls.getEntry(profileHandle); prop.put("crawler-queue_list_" + showNum + "_dark", dark ? "1" : "0"); prop.putHTML("crawler-queue_list_" + showNum + "_initiator", ((initiator == null) ? "proxy" : initiator.getName())); prop.put("crawler-queue_list_" + showNum + "_profile", ((profileEntry == null) ? "unknown" : profileEntry.name())); diff --git a/htroot/IndexImport_p.java b/htroot/IndexImport_p.java index 46587de48..6022aeed6 100644 --- a/htroot/IndexImport_p.java +++ b/htroot/IndexImport_p.java @@ -74,7 +74,7 @@ public final class IndexImport_p { try {
boolean startImport = true;
if (startImport) {
- Importer importerThread = new NoticeURLImporter(switchboard.plasmaPath, switchboard.crawlQueues, switchboard.profilesActiveCrawls, switchboard.dbImportManager);
+ Importer importerThread = new NoticeURLImporter(switchboard.plasmaPath, switchboard.crawlQueues, switchboard.webIndex.profilesActiveCrawls, switchboard.dbImportManager);
if (importerThread != null) {
importerThread.setJobID(switchboard.dbImportManager.generateUniqueJobID());
@@ -125,8 +125,8 @@ public final class IndexImport_p { }
}
- prop.putNum("wcount", switchboard.wordIndex.size());
- prop.putNum("ucount", switchboard.wordIndex.countURL());
+ prop.putNum("wcount", switchboard.webIndex.size());
+ prop.putNum("ucount", switchboard.webIndex.countURL());
/*
* Loop over all currently running jobs
diff --git a/htroot/IndexShare_p.java b/htroot/IndexShare_p.java index cbb465f94..5795aff9a 100644 --- a/htroot/IndexShare_p.java +++ b/htroot/IndexShare_p.java @@ -65,8 +65,8 @@ public class IndexShare_p { prop.put("wordfreq", switchboard.getConfig("defaultWordReceiveFrequency","10"));
prop.put("dtable", "");
prop.put("rtable", "");
- prop.putNum("wcount", switchboard.wordIndex.size());
- prop.putNum("ucount", switchboard.wordIndex.countURL());
+ prop.putNum("wcount", switchboard.webIndex.size());
+ prop.putNum("ucount", switchboard.webIndex.countURL());
return prop; // be save
}
@@ -78,8 +78,8 @@ public class IndexShare_p { }
// insert constants
- prop.putNum("wcount", switchboard.wordIndex.size());
- prop.putNum("ucount", switchboard.wordIndex.countURL());
+ prop.putNum("wcount", switchboard.webIndex.size());
+ prop.putNum("ucount", switchboard.webIndex.countURL());
// return rewrite properties
return prop;
diff --git a/htroot/IndexTransfer_p.java b/htroot/IndexTransfer_p.java index 440f9bc98..3e91be9b9 100644 --- a/htroot/IndexTransfer_p.java +++ b/htroot/IndexTransfer_p.java @@ -70,7 +70,7 @@ public final class IndexTransfer_p { if (post != null) {
if (post.containsKey("startIndexTransfer")) {
- yacySeed seed = sb.wordIndex.seedDB.getConnected(post.get("hostHash", ""));
+ yacySeed seed = sb.webIndex.seedDB.getConnected(post.get("hostHash", ""));
if (seed == null) {
prop.put("running_status","Disconnected peer");
} else {
@@ -95,8 +95,8 @@ public final class IndexTransfer_p { }
// insert constants
- prop.putNum("wcount", sb.wordIndex.size());
- prop.putNum("ucount", sb.wordIndex.countURL());
+ prop.putNum("wcount", sb.webIndex.size());
+ prop.putNum("ucount", sb.webIndex.countURL());
prop.put("running",(sb.transferIdxThread==null) ? "0" : "1");
if (sb.transferIdxThread != null) {
String[] status = sb.transferIdxThread.getStatus();
@@ -130,7 +130,7 @@ public final class IndexTransfer_p { //List known hosts
yacySeed seed;
int hc = 0;
- if ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.sizeConnected() > 0)) {
+ if ((sb.webIndex.seedDB != null) && (sb.webIndex.seedDB.sizeConnected() > 0)) {
Iterator<yacySeed> e = yacyCore.peerActions.dhtAction.getAcceptRemoteIndexSeeds("------------");
TreeMap<String, String> hostList = new TreeMap<String, String>();
while (e.hasNext()) {
diff --git a/htroot/MessageSend_p.java b/htroot/MessageSend_p.java index f9299b313..bdbdb9c55 100644 --- a/htroot/MessageSend_p.java +++ b/htroot/MessageSend_p.java @@ -87,14 +87,14 @@ public class MessageSend_p { // open an editor page for the message
// first ask if the other peer is online, and also what kind of document it accepts
- HashMap<String, String> result = yacyClient.permissionMessage(sb.wordIndex.seedDB, hash);
+ HashMap<String, String> result = yacyClient.permissionMessage(sb.webIndex.seedDB, hash);
//System.out.println("DEBUG: permission request result = " + result.toString());
String peerName;
yacySeed targetPeer = null;
- if (hash.equals(sb.wordIndex.seedDB.mySeed().hash)) {
- peerName = sb.wordIndex.seedDB.mySeed().get(yacySeed.NAME,"nameless");
+ if (hash.equals(sb.webIndex.seedDB.mySeed().hash)) {
+ peerName = sb.webIndex.seedDB.mySeed().get(yacySeed.NAME,"nameless");
} else {
- targetPeer = sb.wordIndex.seedDB.getConnected(hash);
+ targetPeer = sb.webIndex.seedDB.getConnected(hash);
if (targetPeer == null)
peerName = "nameless";
else
@@ -151,7 +151,7 @@ public class MessageSend_p { } catch (UnsupportedEncodingException e) {
mb = message.getBytes();
}
- HashMap<String, String> result = yacyClient.postMessage(sb.wordIndex.seedDB, hash, subject, mb);
+ HashMap<String, String> result = yacyClient.postMessage(sb.webIndex.seedDB, hash, subject, mb);
//message has been sent
prop.put("mode_status_response", (String) result.get("response"));
diff --git a/htroot/Messages_p.java b/htroot/Messages_p.java index 236c2c177..c8039a156 100644 --- a/htroot/Messages_p.java +++ b/htroot/Messages_p.java @@ -74,18 +74,18 @@ public class Messages_p { serverObjects prop = new serverObjects();
// set peer address / name
- final String peerAddress = sb.wordIndex.seedDB.mySeed().getPublicAddress();
- final String peerName = sb.wordIndex.seedDB.mySeed().getName();
+ final String peerAddress = sb.webIndex.seedDB.mySeed().getPublicAddress();
+ final String peerName = sb.webIndex.seedDB.mySeed().getName();
prop.put("peerAddress", peerAddress);
prop.putHTML("peerName", peerName, true);
// List known hosts for message sending (from Blacklist_p.java)
- if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0) {
+ if (sb.webIndex.seedDB != null && sb.webIndex.seedDB.sizeConnected() > 0) {
prop.put("peersKnown", "1");
int peerCount = 0;
try {
TreeMap<String, String> hostList = new TreeMap<String, String>();
- final Iterator<yacySeed> e = sb.wordIndex.seedDB.seedsConnected(true, false, null, (float) 0.0);
+ final Iterator<yacySeed> e = sb.webIndex.seedDB.seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
yacySeed seed = e.next();
if (seed != null) hostList.put(seed.get(yacySeed.NAME, "nameless"),seed.hash);
diff --git a/htroot/Network.java b/htroot/Network.java index 2d14f5dc5..c7bcd5ebb 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -84,30 +84,30 @@ public class Network { prop.putHTML("page_networkName", sb.getConfig("network.unit.name", "unspecified"));
final boolean overview = (post == null) || (post.get("page", "0").equals("0"));
- final String mySeedType = sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
+ final String mySeedType = sb.webIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
final boolean iAmActive = (mySeedType.equals(yacySeed.PEERTYPE_SENIOR) || mySeedType.equals(yacySeed.PEERTYPE_PRINCIPAL));
if (overview) {
- long accActLinks = sb.wordIndex.seedDB.countActiveURL();
- long accActWords = sb.wordIndex.seedDB.countActiveRWI();
- final long accPassLinks = sb.wordIndex.seedDB.countPassiveURL();
- final long accPassWords = sb.wordIndex.seedDB.countPassiveRWI();
- long accPotLinks = sb.wordIndex.seedDB.countPotentialURL();
- long accPotWords = sb.wordIndex.seedDB.countPotentialRWI();
+ long accActLinks = sb.webIndex.seedDB.countActiveURL();
+ long accActWords = sb.webIndex.seedDB.countActiveRWI();
+ final long accPassLinks = sb.webIndex.seedDB.countPassiveURL();
+ final long accPassWords = sb.webIndex.seedDB.countPassiveRWI();
+ long accPotLinks = sb.webIndex.seedDB.countPotentialURL();
+ long accPotWords = sb.webIndex.seedDB.countPotentialRWI();
- int conCount = sb.wordIndex.seedDB.sizeConnected();
- final int disconCount = sb.wordIndex.seedDB.sizeDisconnected();
- int potCount = sb.wordIndex.seedDB.sizePotential();
+ int conCount = sb.webIndex.seedDB.sizeConnected();
+ final int disconCount = sb.webIndex.seedDB.sizeDisconnected();
+ int potCount = sb.webIndex.seedDB.sizePotential();
// final boolean complete = ((post == null) ? false : post.get("links", "false").equals("true"));
- final long otherppm = sb.wordIndex.seedDB.countActivePPM();
- final double otherqpm = sb.wordIndex.seedDB.countActiveQPM();
+ final long otherppm = sb.webIndex.seedDB.countActivePPM();
+ final double otherqpm = sb.webIndex.seedDB.countActiveQPM();
long myppm = 0;
double myqph = 0d;
// create own peer info
- yacySeed seed = sb.wordIndex.seedDB.mySeed();
- if (sb.wordIndex.seedDB.mySeed() != null){ //our Peer
+ yacySeed seed = sb.webIndex.seedDB.mySeed();
+ if (sb.webIndex.seedDB.mySeed() != null){ //our Peer
// update seed info
yacyCore.peerActions.updateMySeed();
@@ -123,17 +123,17 @@ public class Network { // my-info
prop.putHTML("table_my-name", seed.get(yacySeed.NAME, "-") );
prop.put("table_my-hash", seed.hash );
- if (sb.wordIndex.seedDB.mySeed().isVirgin()) {
+ if (sb.webIndex.seedDB.mySeed().isVirgin()) {
prop.put("table_my-info", 0);
- } else if(sb.wordIndex.seedDB.mySeed().isJunior()) {
+ } else if(sb.webIndex.seedDB.mySeed().isJunior()) {
prop.put("table_my-info", 1);
accPotLinks += LCount;
accPotWords += ICount;
- } else if(sb.wordIndex.seedDB.mySeed().isSenior()) {
+ } else if(sb.webIndex.seedDB.mySeed().isSenior()) {
prop.put("table_my-info", 2);
accActLinks += LCount;
accActWords += ICount;
- } else if(sb.wordIndex.seedDB.mySeed().isPrincipal()) {
+ } else if(sb.webIndex.seedDB.mySeed().isPrincipal()) {
prop.put("table_my-info", 3);
accActLinks += LCount;
accActWords += ICount;
@@ -214,13 +214,13 @@ public class Network { yacySeed peer = new yacySeed(post.get("peerHash"),map);
yacyCore.peerActions.updateMySeed();
- final int added = yacyClient.publishMySeed(sb.wordIndex.seedDB.mySeed(), peer.getPublicAddress(), peer.hash);
+ final int added = yacyClient.publishMySeed(sb.webIndex.seedDB.mySeed(), peer.getPublicAddress(), peer.hash);
if (added <= 0) {
prop.put("table_comment",1);
prop.putHTML("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getPublicAddress());
} else {
- peer = sb.wordIndex.seedDB.getConnected(peer.hash);
+ peer = sb.webIndex.seedDB.getConnected(peer.hash);
if (peer == null) {
prop.put("table_comment",1);
prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from UNKNOWN");
@@ -246,14 +246,14 @@ public class Network { final int page = Integer.parseInt(post.get("page", "1"));
final int maxCount = Integer.parseInt(post.get("maxCount", "300"));
int conCount = 0;
- if (sb.wordIndex.seedDB == null) {
+ if (sb.webIndex.seedDB == null) {
prop.put("table", 0);//no remote senior/principal proxies known"
} else {
int size = 0;
switch (page) {
- case 1 : size = sb.wordIndex.seedDB.sizeConnected(); break;
- case 2 : size = sb.wordIndex.seedDB.sizeDisconnected(); break;
- case 3 : size = sb.wordIndex.seedDB.sizePotential(); break;
+ case 1 : size = sb.webIndex.seedDB.sizeConnected(); break;
+ case 2 : size = sb.webIndex.seedDB.sizeDisconnected(); break;
+ case 3 : size = sb.webIndex.seedDB.sizePotential(); break;
default: break;
}
if (size == 0) {
@@ -262,7 +262,7 @@ public class Network { // add temporary the own seed to the database
if (iAmActive) {
yacyCore.peerActions.updateMySeed();
- sb.wordIndex.seedDB.addConnected(sb.wordIndex.seedDB.mySeed());
+ sb.webIndex.seedDB.addConnected(sb.webIndex.seedDB.mySeed());
}
// find updated Information using YaCyNews
@@ -271,7 +271,7 @@ public class Network { final HashMap<String, Map<String, String>> updatedBlog = new HashMap<String, Map<String, String>>();
final HashMap<String, String> isCrawling = new HashMap<String, String>();
yacyNewsRecord record;
- Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
+ Iterator<yacyNewsRecord> recordIterator = sb.webIndex.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
while (recordIterator.hasNext()) {
record = recordIterator.next();
if (record == null) {
@@ -292,9 +292,9 @@ public class Network { final boolean complete = post.containsKey("ip");
Iterator<yacySeed> e = null;
switch (page) {
- case 1 : e = sb.wordIndex.seedDB.seedsSortedConnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LCOUNT)); break;
- case 2 : e = sb.wordIndex.seedDB.seedsSortedDisconnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
- case 3 : e = sb.wordIndex.seedDB.seedsSortedPotential(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
+ case 1 : e = sb.webIndex.seedDB.seedsSortedConnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LCOUNT)); break;
+ case 2 : e = sb.webIndex.seedDB.seedsSortedDisconnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
+ case 3 : e = sb.webIndex.seedDB.seedsSortedPotential(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
default: break;
}
String startURL;
@@ -336,7 +336,7 @@ public class Network { prop.put(STR_TABLE_LIST + conCount + "_updatedBlog", 0);
prop.put(STR_TABLE_LIST + conCount + "_isCrawling", 0);
if (conCount >= maxCount) { break; }
- if (seed.hash.equals(sb.wordIndex.seedDB.mySeed().hash)) {
+ if (seed.hash.equals(sb.webIndex.seedDB.mySeed().hash)) {
prop.put(STR_TABLE_LIST + conCount + "_dark", 2);
} else {
prop.put(STR_TABLE_LIST + conCount + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
@@ -374,7 +374,7 @@ public class Network { prop.putHTML(STR_TABLE_LIST + conCount + "_shortname", shortname);
prop.putHTML(STR_TABLE_LIST + conCount + "_fullname", seed.get(yacySeed.NAME, "deadlink"));
userAgent = null;
- if (seed.hash.equals(sb.wordIndex.seedDB.mySeed().hash)) {
+ if (seed.hash.equals(sb.webIndex.seedDB.mySeed().hash)) {
final JakartaCommonsHttpClient httpClient = new JakartaCommonsHttpClient(10000, null, null);
userAgent = httpClient.getUserAgent();
location = HttpClient.generateLocation();
@@ -463,7 +463,7 @@ public class Network { conCount++;
} // seed != null
} // while
- if (iAmActive) { sb.wordIndex.seedDB.removeMySeed(); }
+ if (iAmActive) { sb.webIndex.seedDB.removeMySeed(); }
prop.putNum("table_list", conCount);
prop.put("table", 1);
prop.putNum("table_num", conCount);
diff --git a/htroot/NetworkPicture.java b/htroot/NetworkPicture.java index 712f2931a..58f8bd13e 100644 --- a/htroot/NetworkPicture.java +++ b/htroot/NetworkPicture.java @@ -82,7 +82,7 @@ public class NetworkPicture { if (passiveLimit > 1000000) passiveLimit = 1000000;
if (potentialLimit > 1000000) potentialLimit = 1000000;
if (maxCount > 1000) maxCount = 1000;
- return plasmaGrafics.getNetworkPicture(sb.wordIndex.seedDB, 10000, width, height, passiveLimit, potentialLimit, maxCount, corona, env.getConfig("network.unit.name", "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
+ return plasmaGrafics.getNetworkPicture(sb.webIndex.seedDB, 10000, width, height, passiveLimit, potentialLimit, maxCount, corona, env.getConfig("network.unit.name", "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
}
}
diff --git a/htroot/News.java b/htroot/News.java index e8ff979cf..d36239f63 100644 --- a/htroot/News.java +++ b/htroot/News.java @@ -81,7 +81,7 @@ public class News { if ((check.startsWith("del_")) && (post.get(check, "off").equals("on"))) {
id = check.substring(4);
try {
- sb.wordIndex.newsPool.moveOff(tableID, id);
+ sb.webIndex.newsPool.moveOff(tableID, id);
} catch (IOException ee) {ee.printStackTrace();}
}
}
@@ -94,9 +94,9 @@ public class News { }
try {
if ((tableID == yacyNewsPool.PROCESSED_DB) || (tableID == yacyNewsPool.PUBLISHED_DB)) {
- sb.wordIndex.newsPool.clear(tableID);
+ sb.webIndex.newsPool.clear(tableID);
} else {
- sb.wordIndex.newsPool.moveOffAll(tableID);
+ sb.webIndex.newsPool.moveOffAll(tableID);
}
} catch (IOException e) {
e.printStackTrace();
@@ -109,19 +109,19 @@ public class News { // show overview
prop.put("table", "0");
prop.put("page", "0");
- prop.putNum("table_insize", sb.wordIndex.newsPool.size(yacyNewsPool.INCOMING_DB));
- prop.putNum("table_prsize", sb.wordIndex.newsPool.size(yacyNewsPool.PROCESSED_DB));
- prop.putNum("table_ousize", sb.wordIndex.newsPool.size(yacyNewsPool.OUTGOING_DB));
- prop.putNum("table_pusize", sb.wordIndex.newsPool.size(yacyNewsPool.PUBLISHED_DB));
+ prop.putNum("table_insize", sb.webIndex.newsPool.size(yacyNewsPool.INCOMING_DB));
+ prop.putNum("table_prsize", sb.webIndex.newsPool.size(yacyNewsPool.PROCESSED_DB));
+ prop.putNum("table_ousize", sb.webIndex.newsPool.size(yacyNewsPool.OUTGOING_DB));
+ prop.putNum("table_pusize", sb.webIndex.newsPool.size(yacyNewsPool.PUBLISHED_DB));
} else {
// generate table
prop.put("table", "1");
prop.put("page", tableID + 1);
prop.put("table_page", tableID + 1);
- if (sb.wordIndex.seedDB != null) {
- int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(tableID));
- Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(tableID, false);
+ if (sb.webIndex.seedDB != null) {
+ int maxCount = Math.min(1000, sb.webIndex.newsPool.size(tableID));
+ Iterator<yacyNewsRecord> recordIterator = sb.webIndex.newsPool.recordIterator(tableID, false);
yacyNewsRecord record;
yacySeed seed;
int i = 0;
@@ -129,8 +129,8 @@ public class News { record = recordIterator.next();
if (record == null) continue;
- seed = sb.wordIndex.seedDB.getConnected(record.originator());
- if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(record.originator());
+ seed = sb.webIndex.seedDB.getConnected(record.originator());
+ if (seed == null) seed = sb.webIndex.seedDB.getDisconnected(record.originator());
String category = record.category();
prop.put("table_list_" + i + "_id", record.id());
prop.putHTML("table_list_" + i + "_ori", (seed == null) ? record.originator() : seed.getName());
@@ -201,7 +201,7 @@ public class News { }
// adding the peer address
- prop.put("address", sb.wordIndex.seedDB.mySeed().getPublicAddress());
+ prop.put("address", sb.webIndex.seedDB.mySeed().getPublicAddress());
// return rewrite properties
return prop;
diff --git a/htroot/PerformanceGraph.java b/htroot/PerformanceGraph.java index 1eeef24b9..09b33582a 100644 --- a/htroot/PerformanceGraph.java +++ b/htroot/PerformanceGraph.java @@ -41,7 +41,7 @@ public class PerformanceGraph { int width = post.getInt("width", 660); int height = post.getInt("height", 240); - return plasmaProfiling.performanceGraph(width, height, sb.wordIndex.countURL() + " URLS / " + sb.wordIndex.collectionsSize() + " WORDS IN COLLECTIONS / " + sb.wordIndex.cacheSize() + " WORDS IN CACHE"); + return plasmaProfiling.performanceGraph(width, height, sb.webIndex.countURL() + " URLS / " + sb.webIndex.collectionsSize() + " WORDS IN COLLECTIONS / " + sb.webIndex.cacheSize() + " WORDS IN CACHE"); } }
\ No newline at end of file diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index 79200044e..c129502f8 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -177,7 +177,7 @@ public class PerformanceQueues_p { // disallow setting of memprereq for indexer to prevent db from throwing OOMs
prop.put("table_" + c + "_disabled", /*(threadName.endsWith("_indexing")) ? 1 :*/ "0");
prop.put("table_" + c + "_recommendation", threadName.endsWith("_indexing") ? "1" : "0");
- prop.putNum("table_" + c + "_recommendation_value", threadName.endsWith("_indexing") ? (switchboard.wordIndex.minMem() / 1024) : 0);
+ prop.putNum("table_" + c + "_recommendation_value", threadName.endsWith("_indexing") ? (switchboard.webIndex.minMem() / 1024) : 0);
c++;
}
prop.put("table", c);
@@ -185,7 +185,7 @@ public class PerformanceQueues_p { if ((post != null) && (post.containsKey("cacheSizeSubmit"))) {
int wordCacheMaxCount = post.getInt("wordCacheMaxCount", 20000);
switchboard.setConfig(plasmaSwitchboard.WORDCACHE_MAX_COUNT, Integer.toString(wordCacheMaxCount));
- switchboard.wordIndex.setMaxWordCount(wordCacheMaxCount);
+ switchboard.webIndex.setMaxWordCount(wordCacheMaxCount);
int wordCacheInitCount = post.getInt(plasmaSwitchboard.WORDCACHE_INIT_COUNT, 30000);
switchboard.setConfig(plasmaSwitchboard.WORDCACHE_INIT_COUNT, Integer.toString(wordCacheInitCount));
@@ -231,17 +231,17 @@ public class PerformanceQueues_p { }
// table cache settings
- prop.putNum("urlCacheSize", switchboard.wordIndex.getURLwriteCacheSize());
- prop.putNum("wordCacheWSize", switchboard.wordIndex.dhtOutCacheSize());
- prop.putNum("wordCacheKSize", switchboard.wordIndex.dhtInCacheSize());
- prop.putNum("wordCacheWSizeKBytes", switchboard.wordIndex.dhtCacheSizeBytes(false)/1024);
- prop.putNum("wordCacheKSizeKBytes", switchboard.wordIndex.dhtCacheSizeBytes(true)/1024);
- prop.putNum("maxURLinWCache", switchboard.wordIndex.maxURLinDHTOutCache());
- prop.putNum("maxURLinKCache", switchboard.wordIndex.maxURLinDHTInCache());
- prop.putNum("maxAgeOfWCache", switchboard.wordIndex.maxAgeOfDHTOutCache() / 1000 / 60); // minutes
- prop.putNum("maxAgeOfKCache", switchboard.wordIndex.maxAgeOfDHTInCache() / 1000 / 60); // minutes
- prop.putNum("minAgeOfWCache", switchboard.wordIndex.minAgeOfDHTOutCache() / 1000 / 60); // minutes
- prop.putNum("minAgeOfKCache", switchboard.wordIndex.minAgeOfDHTInCache() / 1000 / 60); // minutes
+ prop.putNum("urlCacheSize", switchboard.webIndex.getURLwriteCacheSize());
+ prop.putNum("wordCacheWSize", switchboard.webIndex.dhtOutCacheSize());
+ prop.putNum("wordCacheKSize", switchboard.webIndex.dhtInCacheSize());
+ prop.putNum("wordCacheWSizeKBytes", switchboard.webIndex.dhtCacheSizeBytes(false)/1024);
+ prop.putNum("wordCacheKSizeKBytes", switchboard.webIndex.dhtCacheSizeBytes(true)/1024);
+ prop.putNum("maxURLinWCache", switchboard.webIndex.maxURLinDHTOutCache());
+ prop.putNum("maxURLinKCache", switchboard.webIndex.maxURLinDHTInCache());
+ prop.putNum("maxAgeOfWCache", switchboard.webIndex.maxAgeOfDHTOutCache() / 1000 / 60); // minutes
+ prop.putNum("maxAgeOfKCache", switchboard.webIndex.maxAgeOfDHTInCache() / 1000 / 60); // minutes
+ prop.putNum("minAgeOfWCache", switchboard.webIndex.minAgeOfDHTOutCache() / 1000 / 60); // minutes
+ prop.putNum("minAgeOfKCache", switchboard.webIndex.minAgeOfDHTInCache() / 1000 / 60); // minutes
prop.putNum("maxWaitingWordFlush", switchboard.getConfigLong("maxWaitingWordFlush", 180));
prop.put("wordCacheMaxCount", switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_MAX_COUNT, 20000));
prop.put("wordCacheInitCount", switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_INIT_COUNT, 30000));
diff --git a/htroot/ProxyIndexingMonitor_p.java b/htroot/ProxyIndexingMonitor_p.java index 123624367..0fda4ee1d 100644 --- a/htroot/ProxyIndexingMonitor_p.java +++ b/htroot/ProxyIndexingMonitor_p.java @@ -113,15 +113,15 @@ public class ProxyIndexingMonitor_p { sb.setCacheSize(Long.parseLong(newProxyCacheSize));
// implant these settings also into the crawling profile for the proxy
- if (sb.defaultProxyProfile == null) {
+ if (sb.webIndex.defaultProxyProfile == null) {
prop.put("info", "1"); //delete DATA/PLASMADB/crawlProfiles0.db
} else {
try {
- sb.profilesActiveCrawls.changeEntry(sb.defaultProxyProfile, "generalDepth", Integer.toString(newProxyPrefetchDepth));
- sb.profilesActiveCrawls.changeEntry(sb.defaultProxyProfile, "storeHTCache", (proxyStoreHTCache) ? "true": "false");
- sb.profilesActiveCrawls.changeEntry(sb.defaultProxyProfile, "remoteIndexing",proxyIndexingRemote ? "true":"false");
- sb.profilesActiveCrawls.changeEntry(sb.defaultProxyProfile, "indexText",proxyIndexingLocalText ? "true":"false");
- sb.profilesActiveCrawls.changeEntry(sb.defaultProxyProfile, "indexMedia",proxyIndexingLocalMedia ? "true":"false");
+ sb.webIndex.profilesActiveCrawls.changeEntry(sb.webIndex.defaultProxyProfile, "generalDepth", Integer.toString(newProxyPrefetchDepth));
+ sb.webIndex.profilesActiveCrawls.changeEntry(sb.webIndex.defaultProxyProfile, "storeHTCache", (proxyStoreHTCache) ? "true": "false");
+ sb.webIndex.profilesActiveCrawls.changeEntry(sb.webIndex.defaultProxyProfile, "remoteIndexing",proxyIndexingRemote ? "true":"false");
+ sb.webIndex.profilesActiveCrawls.changeEntry(sb.webIndex.defaultProxyProfile, "indexText",proxyIndexingLocalText ? "true":"false");
+ sb.webIndex.profilesActiveCrawls.changeEntry(sb.webIndex.defaultProxyProfile, "indexMedia",proxyIndexingLocalMedia ? "true":"false");
prop.put("info", "2");//new proxyPrefetchdepth
prop.put("info_message", newProxyPrefetchDepth);
diff --git a/htroot/QuickCrawlLink_p.java b/htroot/QuickCrawlLink_p.java index 2c6222160..74a89fc87 100644 --- a/htroot/QuickCrawlLink_p.java +++ b/htroot/QuickCrawlLink_p.java @@ -138,14 +138,14 @@ public class QuickCrawlLink_p { }
String urlhash = crawlingStartURL.hash();
- sb.wordIndex.removeURL(urlhash);
+ sb.webIndex.removeURL(urlhash);
sb.crawlQueues.noticeURL.removeByURLHash(urlhash);
sb.crawlQueues.errorURL.remove(urlhash);
// create crawling profile
CrawlProfile.entry pe = null;
try {
- pe = sb.profilesActiveCrawls.newEntry(
+ pe = sb.webIndex.profilesActiveCrawls.newEntry(
crawlingStartURL.getHost(),
crawlingStartURL,
crawlingFilter,
@@ -178,7 +178,7 @@ public class QuickCrawlLink_p { reasonString = sb.crawlStacker.stackCrawl(
crawlingStartURL,
null,
- sb.wordIndex.seedDB.mySeed().hash,
+ sb.webIndex.seedDB.mySeed().hash,
(title==null)?"CRAWLING-ROOT":title,
new Date(),
0,
diff --git a/htroot/SearchEventPicture.java b/htroot/SearchEventPicture.java index 90ca8363d..b1b1823d2 100644 --- a/htroot/SearchEventPicture.java +++ b/htroot/SearchEventPicture.java @@ -60,7 +60,7 @@ public class SearchEventPicture { plasmaSwitchboard sb = (plasmaSwitchboard) env;
String eventID = (String) header.get("event", plasmaSearchEvent.lastEventID);
if (eventID == null) return null;
- ymageMatrix yp = plasmaGrafics.getSearchEventPicture(sb.wordIndex.seedDB, eventID);
+ ymageMatrix yp = plasmaGrafics.getSearchEventPicture(sb.webIndex.seedDB, eventID);
if (yp == null) return new ymageMatrix(1, 1, ymageMatrix.MODE_SUB, "000000"); // empty image
return yp;
diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index fdf747314..e71550e06 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -252,7 +252,7 @@ public class SettingsAck_p { } else {
serverCore.useStaticIP = true;
}
- sb.wordIndex.seedDB.mySeed().put(yacySeed.IP, staticIP);
+ sb.webIndex.seedDB.mySeed().put(yacySeed.IP, staticIP);
env.setConfig("staticIP", staticIP);
// server access data
diff --git a/htroot/Status.java b/htroot/Status.java index d1bb74c43..64da94883 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -182,35 +182,35 @@ public class Status { // peer information
String thisHash = "";
final String thisName = sb.getConfig("peerName", "<nameless>");
- if (sb.wordIndex.seedDB.mySeed() == null) {
+ if (sb.webIndex.seedDB.mySeed() == null) {
thisHash = "not assigned";
prop.put("peerAddress", "0"); // not assigned
prop.put("peerStatistics", "0"); // unknown
} else {
- final long uptime = 60000 * Long.parseLong(sb.wordIndex.seedDB.mySeed().get(yacySeed.UPTIME, "0"));
+ final long uptime = 60000 * Long.parseLong(sb.webIndex.seedDB.mySeed().get(yacySeed.UPTIME, "0"));
prop.put("peerStatistics", "1");
prop.put("peerStatistics_uptime", serverDate.formatInterval(uptime));
- prop.putNum("peerStatistics_pagesperminute", sb.wordIndex.seedDB.mySeed().getPPM());
- prop.putNum("peerStatistics_queriesperhour", Math.round(6000d * sb.wordIndex.seedDB.mySeed().getQPM()) / 100d);
- prop.putNum("peerStatistics_links", sb.wordIndex.seedDB.mySeed().getLinkCount());
- prop.put("peerStatistics_words", yFormatter.number(sb.wordIndex.seedDB.mySeed().get(yacySeed.ICOUNT, "0")));
+ prop.putNum("peerStatistics_pagesperminute", sb.webIndex.seedDB.mySeed().getPPM());
+ prop.putNum("peerStatistics_queriesperhour", Math.round(6000d * sb.webIndex.seedDB.mySeed().getQPM()) / 100d);
+ prop.putNum("peerStatistics_links", sb.webIndex.seedDB.mySeed().getLinkCount());
+ prop.put("peerStatistics_words", yFormatter.number(sb.webIndex.seedDB.mySeed().get(yacySeed.ICOUNT, "0")));
prop.putNum("peerStatistics_juniorConnects", yacyCore.peerActions.juniorConnects);
prop.putNum("peerStatistics_seniorConnects", yacyCore.peerActions.seniorConnects);
prop.putNum("peerStatistics_principalConnects", yacyCore.peerActions.principalConnects);
prop.putNum("peerStatistics_disconnects", yacyCore.peerActions.disconnects);
- prop.put("peerStatistics_connects", yFormatter.number(sb.wordIndex.seedDB.mySeed().get(yacySeed.CCOUNT, "0")));
- if (sb.wordIndex.seedDB.mySeed().getPublicAddress() == null) {
- thisHash = sb.wordIndex.seedDB.mySeed().hash;
+ prop.put("peerStatistics_connects", yFormatter.number(sb.webIndex.seedDB.mySeed().get(yacySeed.CCOUNT, "0")));
+ if (sb.webIndex.seedDB.mySeed().getPublicAddress() == null) {
+ thisHash = sb.webIndex.seedDB.mySeed().hash;
prop.put("peerAddress", "0"); // not assigned + instructions
prop.put("warningGoOnline", "1");
} else {
- thisHash = sb.wordIndex.seedDB.mySeed().hash;
+ thisHash = sb.webIndex.seedDB.mySeed().hash;
prop.put("peerAddress", "1"); // Address
- prop.put("peerAddress_address", sb.wordIndex.seedDB.mySeed().getPublicAddress());
+ prop.put("peerAddress_address", sb.webIndex.seedDB.mySeed().getPublicAddress());
prop.putHTML("peerAddress_peername", sb.getConfig("peerName", "<nameless>").toLowerCase(), true);
}
}
- final String peerStatus = ((sb.wordIndex.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_VIRGIN : sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN));
+ final String peerStatus = ((sb.webIndex.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_VIRGIN : sb.webIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN));
if (peerStatus.equals(yacySeed.PEERTYPE_VIRGIN)) {
prop.put(PEERSTATUS, "0");
prop.put("urgentStatusVirgin", "1");
@@ -223,7 +223,7 @@ public class Status { } else if (peerStatus.equals(yacySeed.PEERTYPE_PRINCIPAL)) {
prop.put(PEERSTATUS, "3");
prop.put("hintStatusPrincipal", "1");
- prop.put("hintStatusPrincipal_seedURL", sb.wordIndex.seedDB.mySeed().get("seedURL", "?"));
+ prop.put("hintStatusPrincipal_seedURL", sb.webIndex.seedDB.mySeed().get("seedURL", "?"));
}
prop.putHTML("peerName", thisName);
prop.put("hash", thisHash);
@@ -252,14 +252,14 @@ public class Status { prop.put("seedServer_seedFile", sb.getConfig("seedFilePath", ""));
}
prop.put("seedServer_lastUpload",
- serverDate.formatInterval(System.currentTimeMillis() - sb.wordIndex.seedDB.lastSeedUpload_timeStamp));
+ serverDate.formatInterval(System.currentTimeMillis() - sb.webIndex.seedDB.lastSeedUpload_timeStamp));
} else {
prop.put(SEEDSERVER, "0"); // disabled
}
- if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0){
+ if (sb.webIndex.seedDB != null && sb.webIndex.seedDB.sizeConnected() > 0){
prop.put("otherPeers", "1");
- prop.putNum("otherPeers_num", sb.wordIndex.seedDB.sizeConnected());
+ prop.putNum("otherPeers_num", sb.webIndex.seedDB.sizeConnected());
}else{
prop.put("otherPeers", "0"); // not online
}
@@ -295,7 +295,7 @@ public class Status { prop.putNum("connectionsMax", httpd.getMaxSessionCount());
// Queue information
- int indexingJobCount = sb.getThread("80_indexing").getJobCount() + sb.sbQueue.getActiveQueueSize();
+ int indexingJobCount = sb.getThread("80_indexing").getJobCount() + sb.webIndex.queuePreStack.getActiveQueueSize();
int indexingMaxCount = (int) sb.getConfigLong(plasmaSwitchboard.INDEXER_SLOTS, 30);
int indexingPercent = (indexingMaxCount==0)?0:indexingJobCount*100/indexingMaxCount;
prop.putNum("indexingQueueSize", indexingJobCount);
diff --git a/htroot/Supporter.java b/htroot/Supporter.java index 091a96c6f..e7169e2d0 100644 --- a/htroot/Supporter.java +++ b/htroot/Supporter.java @@ -78,7 +78,7 @@ public class Supporter { map.put("urlhash", hash); map.put("vote", "negative"); map.put("refid", post.get("refid", "")); - sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.webIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } if ((post != null) && ((hash = post.get("votePositive", null)) != null)) { if (!sb.verifyAuthentication(header, false)) { @@ -94,7 +94,7 @@ public class Supporter { map.put("vote", "positive"); map.put("refid", post.get("refid", "")); map.put("comment", post.get("comment", "")); - sb.wordIndex.newsPool.publishMyNews(new yacyNewsRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.webIndex.newsPool.publishMyNews(new yacyNewsRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } // create Supporter @@ -131,8 +131,8 @@ public class Supporter { description = row.getColString(2,"UTF-8"); if ((url == null) || (title == null) || (description == null)) continue; refid = row.getColString(3, null); - voted = (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || - (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null); + voted = (sb.webIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || + (sb.webIndex.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null); prop.put("supporter_results_" + i + "_authorized", authenticated ? "1" : "0"); prop.put("supporter_results_" + i + "_authorized_recommend", voted ? "0" : "1"); @@ -168,9 +168,9 @@ public class Supporter { } private static void accumulateVotes(plasmaSwitchboard sb, HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) { - int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype)); + int maxCount = Math.min(1000, sb.webIndex.newsPool.size(dbtype)); yacyNewsRecord record; - Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true); + Iterator<yacyNewsRecord> recordIterator = sb.webIndex.newsPool.recordIterator(dbtype, true); int j = 0; while ((recordIterator.hasNext()) && (j++ < maxCount)) { record = recordIterator.next(); @@ -198,9 +198,9 @@ public class Supporter { plasmaSwitchboard sb, HashMap<String, Entry> Supporter, kelondroMScoreCluster<String> ranking, kelondroRow rowdef, HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) { - int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype)); + int maxCount = Math.min(1000, sb.webIndex.newsPool.size(dbtype)); yacyNewsRecord record; - Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true); + Iterator<yacyNewsRecord> recordIterator = sb.webIndex.newsPool.recordIterator(dbtype, true); int j = 0; String url = "", urlhash; kelondroRow.Entry entry; @@ -213,7 +213,7 @@ public class Supporter { entry = null; if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_UPDATE)) && - ((seed = sb.wordIndex.seedDB.getConnected(record.originator())) != null)) try { + ((seed = sb.webIndex.seedDB.getConnected(record.originator())) != null)) try { url = record.attribute("homepage", ""); if (url.length() < 12) continue; entry = rowdef.newEntry(new byte[][]{ @@ -226,7 +226,7 @@ public class Supporter { } catch (IOException e) {} if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_BROADCAST)) && - ((seed = sb.wordIndex.seedDB.getConnected(record.originator())) != null)) try { + ((seed = sb.webIndex.seedDB.getConnected(record.originator())) != null)) try { url = record.attribute("homepage", ""); if (url.length() < 12) continue; entry = rowdef.newEntry(new byte[][]{ diff --git a/htroot/Surftips.java b/htroot/Surftips.java index 9f3c3a137..a84519daf 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -86,7 +86,7 @@ public class Surftips { map.put("urlhash", hash); map.put("vote", "negative"); map.put("refid", post.get("refid", "")); - sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.webIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } if ((post != null) && ((hash = post.get("votePositive", null)) != null)) { if (!sb.verifyAuthentication(header, false)) { @@ -102,7 +102,7 @@ public class Surftips { map.put("vote", "positive"); map.put("refid", post.get("refid", "")); map.put("comment", post.get("comment", "")); - sb.wordIndex.newsPool.publishMyNews(new yacyNewsRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.webIndex.newsPool.publishMyNews(new yacyNewsRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } // create surftips @@ -140,8 +140,8 @@ public class Surftips { description = row.getColString(2,"UTF-8"); if ((url == null) || (title == null) || (description == null)) continue; refid = row.getColString(3, null); - voted = (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || - (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null); + voted = (sb.webIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || + (sb.webIndex.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null); prop.put("surftips_results_" + i + "_authorized", (authenticated) ? "1" : "0"); prop.put("surftips_results_" + i + "_authorized_recommend", (voted) ? "0" : "1"); @@ -177,9 +177,9 @@ public class Surftips { } private static void accumulateVotes(plasmaSwitchboard sb, HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) { - int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype)); + int maxCount = Math.min(1000, sb.webIndex.newsPool.size(dbtype)); yacyNewsRecord record; - Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true); + Iterator<yacyNewsRecord> recordIterator = sb.webIndex.newsPool.recordIterator(dbtype, true); int j = 0; while ((recordIterator.hasNext()) && (j++ < maxCount)) { record = recordIterator.next(); @@ -207,9 +207,9 @@ public class Surftips { plasmaSwitchboard sb, HashMap<String, Entry> surftips, kelondroMScoreCluster<String> ranking, kelondroRow rowdef, HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) { - int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype)); + int maxCount = Math.min(1000, sb.webIndex.newsPool.size(dbtype)); yacyNewsRecord record; - Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true); + Iterator<yacyNewsRecord> recordIterator = sb.webIndex.newsPool.recordIterator(dbtype, true); int j = 0; String url = "", urlhash; kelondroRow.Entry entry; @@ -271,8 +271,8 @@ public class Surftips { } catch (IOException e) {} if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) try { - yacySeed seed = sb.wordIndex.seedDB.getConnected(record.originator()); - if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(record.originator()); + yacySeed seed = sb.webIndex.seedDB.getConnected(record.originator()); + if (seed == null) seed = sb.webIndex.seedDB.getDisconnected(record.originator()); if (seed != null) { url = "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ @@ -286,8 +286,8 @@ public class Surftips { } catch (IOException e) {} if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) try { - yacySeed seed = sb.wordIndex.seedDB.getConnected(record.originator()); - if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(record.originator()); + yacySeed seed = sb.webIndex.seedDB.getConnected(record.originator()); + if (seed == null) seed = sb.webIndex.seedDB.getDisconnected(record.originator()); if (seed != null) { url = "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ diff --git a/htroot/ViewFile.java b/htroot/ViewFile.java index 9bf0d3fb5..503239449 100644 --- a/htroot/ViewFile.java +++ b/htroot/ViewFile.java @@ -108,7 +108,7 @@ public class ViewFile { if (urlHash.length() > 0) {
// getting the urlEntry that belongs to the url hash
indexURLReference urlEntry = null;
- urlEntry = sb.wordIndex.getURL(urlHash, null, 0);
+ urlEntry = sb.webIndex.getURL(urlHash, null, 0);
if (urlEntry == null) {
prop.put("error", "2");
prop.put("viewMode",VIEW_MODE_NO_TEXT);
diff --git a/htroot/ViewProfile.java b/htroot/ViewProfile.java index 00de21e05..63cfe4a01 100644 --- a/htroot/ViewProfile.java +++ b/htroot/ViewProfile.java @@ -77,7 +77,7 @@ public class ViewProfile { prop.put("display", display);
String hash = (post == null) ? null : (String) post.get("hash");
- if ((hash == null) || (sb.wordIndex.seedDB == null)) {
+ if ((hash == null) || (sb.webIndex.seedDB == null)) {
// wrong access
prop.put("success", "0");
return prop;
@@ -101,20 +101,20 @@ public class ViewProfile { profile.putAll(p);
prop.put("success", "3"); // everything ok
prop.put("localremotepeer", "0");
- prop.putHTML("success_peername", sb.wordIndex.seedDB.mySeed().getName());
- prop.put("success_peerhash", sb.wordIndex.seedDB.mySeed().hash);
- address = sb.wordIndex.seedDB.mySeed().getPublicAddress();
+ prop.putHTML("success_peername", sb.webIndex.seedDB.mySeed().getName());
+ prop.put("success_peerhash", sb.webIndex.seedDB.mySeed().hash);
+ address = sb.webIndex.seedDB.mySeed().getPublicAddress();
} else {
// read the profile from remote peer
- yacySeed seed = sb.wordIndex.seedDB.getConnected(hash);
- if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(hash);
+ yacySeed seed = sb.webIndex.seedDB.getConnected(hash);
+ if (seed == null) seed = sb.webIndex.seedDB.getDisconnected(hash);
if (seed == null) {
prop.put("success", "1"); // peer unknown
} else {
// process news if existent
try {
- yacyNewsRecord record = sb.wordIndex.newsPool.getByOriginator(yacyNewsPool.INCOMING_DB, yacyNewsPool.CATEGORY_PROFILE_UPDATE, seed.hash);
- if (record != null) sb.wordIndex.newsPool.moveOff(yacyNewsPool.INCOMING_DB, record.id());
+ yacyNewsRecord record = sb.webIndex.newsPool.getByOriginator(yacyNewsPool.INCOMING_DB, yacyNewsPool.CATEGORY_PROFILE_UPDATE, seed.hash);
+ if (record != null) sb.webIndex.newsPool.moveOff(yacyNewsPool.INCOMING_DB, record.id());
} catch (IOException e) {}
// try to get the profile from remote peer
diff --git a/htroot/WatchCrawler_p.java b/htroot/WatchCrawler_p.java index 501eb304e..6d709ee39 100644 --- a/htroot/WatchCrawler_p.java +++ b/htroot/WatchCrawler_p.java @@ -100,7 +100,7 @@ public class WatchCrawler_p { if (post.containsKey("crawlingstart")) { // init crawl - if (sb.wordIndex.seedDB == null) { + if (sb.webIndex.seedDB == null) { prop.put("info", "3"); } else { // set new properties @@ -185,20 +185,20 @@ public class WatchCrawler_p { // first delete old entry, if exists yacyURL url = new yacyURL(crawlingStart, null); String urlhash = url.hash(); - sb.wordIndex.removeURL(urlhash); + sb.webIndex.removeURL(urlhash); sb.crawlQueues.noticeURL.removeByURLHash(urlhash); sb.crawlQueues.errorURL.remove(urlhash); // stack url - sb.profilesPassiveCrawls.removeEntry(crawlingStartURL.hash()); // if there is an old entry, delete it - CrawlProfile.entry pe = sb.profilesActiveCrawls.newEntry( + sb.webIndex.profilesPassiveCrawls.removeEntry(crawlingStartURL.hash()); // if there is an old entry, delete it + CrawlProfile.entry pe = sb.webIndex.profilesActiveCrawls.newEntry( crawlingStartURL.getHost(), crawlingStartURL, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw); - String reasonString = sb.crawlStacker.stackCrawl(url, null, sb.wordIndex.seedDB.mySeed().hash, "CRAWLING-ROOT", new Date(), 0, pe); + String reasonString = sb.crawlStacker.stackCrawl(url, null, sb.webIndex.seedDB.mySeed().hash, "CRAWLING-ROOT", new Date(), 0, pe); if (reasonString == null) { // liftoff! @@ -220,7 +220,7 @@ public class WatchCrawler_p { m.remove("generalFilter"); m.remove("specificFilter"); m.put("intention", post.get("intention", "").replace(',', '/')); - sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m)); + sb.webIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m)); } } else { @@ -230,7 +230,7 @@ public class WatchCrawler_p { ZURL.Entry ee = sb.crawlQueues.errorURL.newEntry( new CrawlEntry( - sb.wordIndex.seedDB.mySeed().hash, + sb.webIndex.seedDB.mySeed().hash, crawlingStartURL, "", "", @@ -239,7 +239,7 @@ public class WatchCrawler_p { 0, 0, 0), - sb.wordIndex.seedDB.mySeed().hash, + sb.webIndex.seedDB.mySeed().hash, new Date(), 1, reasonString); @@ -285,7 +285,7 @@ public class WatchCrawler_p { // creating a crawler profile yacyURL crawlURL = new yacyURL("file://" + file.toString(), null); - CrawlProfile.entry profile = sb.profilesActiveCrawls.newEntry(fileName, crawlURL, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw); + CrawlProfile.entry profile = sb.webIndex.profilesActiveCrawls.newEntry(fileName, crawlURL, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw); // pause local crawl here sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); @@ -302,7 +302,7 @@ public class WatchCrawler_p { sb.crawlStacker.enqueueEntry( nexturl, null, - sb.wordIndex.seedDB.mySeed().hash, + sb.webIndex.seedDB.mySeed().hash, (String) e.getValue(), new Date(), 0, @@ -331,7 +331,7 @@ public class WatchCrawler_p { yacyURL sitemapURL = new yacyURL(sitemapURLStr, null); // create a new profile - CrawlProfile.entry pe = sb.profilesActiveCrawls.newEntry( + CrawlProfile.entry pe = sb.webIndex.profilesActiveCrawls.newEntry( sitemapURLStr, sitemapURL, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, diff --git a/htroot/WatchWebStructure_p.java b/htroot/WatchWebStructure_p.java index 7537d57b1..a2dd5fbd3 100644 --- a/htroot/WatchWebStructure_p.java +++ b/htroot/WatchWebStructure_p.java @@ -4,6 +4,7 @@ import java.util.Iterator; import de.anomic.crawler.CrawlProfile.entry; import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.plasma.plasmaWordIndex; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; @@ -31,16 +32,16 @@ public class WatchWebStructure_p { if (host.equals("auto")) { // try to find the host from the crawl profiles - Iterator<entry> it = sb.profilesActiveCrawls.profiles(true); + Iterator<entry> it = sb.webIndex.profilesActiveCrawls.profiles(true); entry e; while (it.hasNext()) { e = it.next(); - if (e.name().equals(plasmaSwitchboard.CRAWL_PROFILE_PROXY) || - e.name().equals(plasmaSwitchboard.CRAWL_PROFILE_REMOTE) || - e.name().equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_LOCAL_TEXT) || - e.name().equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT) || - e.name().equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA) || - e.name().equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA)) + if (e.name().equals(plasmaWordIndex.CRAWL_PROFILE_PROXY) || + e.name().equals(plasmaWordIndex.CRAWL_PROFILE_REMOTE) || + e.name().equals(plasmaWordIndex.CRAWL_PROFILE_SNIPPET_LOCAL_TEXT) || + e.name().equals(plasmaWordIndex.CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT) || + e.name().equals(plasmaWordIndex.CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA) || + e.name().equals(plasmaWordIndex.CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA)) continue; host = e.name(); break; // take the first one diff --git a/htroot/Wiki.java b/htroot/Wiki.java index 00b4ac197..fff711aa6 100644 --- a/htroot/Wiki.java +++ b/htroot/Wiki.java @@ -92,8 +92,8 @@ public class Wiki { if (author.equals("anonymous")) {
author = wikiBoard.guessAuthor(ip);
if (author == null) {
- if (sb.wordIndex.seedDB.mySeed() == null) author = "anonymous";
- else author = sb.wordIndex.seedDB.mySeed().get("Name", "anonymous");
+ if (sb.webIndex.seedDB.mySeed() == null) author = "anonymous";
+ else author = sb.webIndex.seedDB.mySeed().get("Name", "anonymous");
}
}
@@ -135,7 +135,7 @@ public class Wiki { map.put("page", pagename);
map.put("author", author.replace(',', ' '));
if (post.get("content", "").trim().length() > 0 && !page.page().equals(content))
- sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_WIKI_UPDATE, map));
+ sb.webIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_WIKI_UPDATE, map));
page = newEntry;
prop.put("LOCATION", "/Wiki.html?page=" + pagename);
}
diff --git a/htroot/YaCySearchPluginFF.java b/htroot/YaCySearchPluginFF.java index 0f50c64c0..3969d8ffa 100644 --- a/htroot/YaCySearchPluginFF.java +++ b/htroot/YaCySearchPluginFF.java @@ -72,7 +72,7 @@ public class YaCySearchPluginFF { prop.put("host", host);
prop.put("port", port);
- prop.putHTML("name", sb.wordIndex.seedDB.mySeed().getName());
+ prop.putHTML("name", sb.webIndex.seedDB.mySeed().getName());
return prop;
}
diff --git a/htroot/index.java b/htroot/index.java index be7a6debc..af5c45ced 100644 --- a/htroot/index.java +++ b/htroot/index.java @@ -79,7 +79,7 @@ public class index { if (cds.equals("app")) contentdom = plasmaSearchQuery.CONTENTDOM_APP;
//long mylinks = 0;
- prop.putNum("links", sb.wordIndex.seedDB.mySeed().getLinkCount());
+ prop.putNum("links", sb.webIndex.seedDB.mySeed().getLinkCount());
// we create empty entries for template strings
String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", "");
diff --git a/htroot/opensearchdescription.java b/htroot/opensearchdescription.java index ca6dfabac..9de2ea01a 100644 --- a/htroot/opensearchdescription.java +++ b/htroot/opensearchdescription.java @@ -45,7 +45,7 @@ public class opensearchdescription { final serverObjects prop = new serverObjects(); prop.putHTML("thisaddress", thisaddress, true); prop.putHTML("SearchPageGreeting", promoteSearchPageGreeting, true); - prop.putHTML("clientname", sb.wordIndex.seedDB.mySeed().getName(), true); + prop.putHTML("clientname", sb.webIndex.seedDB.mySeed().getName(), true); // return rewrite properties return prop; diff --git a/htroot/rct_p.java b/htroot/rct_p.java index 99c0c770a..1c809c645 100644 --- a/htroot/rct_p.java +++ b/htroot/rct_p.java @@ -52,8 +52,8 @@ public class rct_p { if (post != null) { if (post.containsKey("retrieve")) { String peerhash = post.get("peer", null); - yacySeed seed = (peerhash == null) ? null : sb.wordIndex.seedDB.getConnected(peerhash); - RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(sb.wordIndex.seedDB, seed, 10); + yacySeed seed = (peerhash == null) ? null : sb.webIndex.seedDB.getConnected(peerhash); + RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(sb.webIndex.seedDB, seed, 10); if (feed != null) { for (RSSMessage item: feed) { //System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ", pubDate=" + item.getPubDate()); @@ -76,7 +76,7 @@ public class rct_p { if (urlRejectReason == null) { // stack url sb.getLog().logFinest("crawlOrder: stack: url='" + url + "'"); - String reasonString = sb.crawlStacker.stackCrawl(url, referrer, peerhash, "REMOTE-CRAWLING", loaddate, 0, sb.defaultRemoteProfile); + String reasonString = sb.crawlStacker.stackCrawl(url, referrer, peerhash, "REMOTE-CRAWLING", loaddate, 0, sb.webIndex.defaultRemoteProfile); if (reasonString == null) { // done @@ -105,7 +105,7 @@ public class rct_p { // list known hosts yacySeed seed; int hc = 0; - if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0) { + if (sb.webIndex.seedDB != null && sb.webIndex.seedDB.sizeConnected() > 0) { Iterator<yacySeed> e = yacyCore.peerActions.dhtAction.getProvidesRemoteCrawlURLs(); while (e.hasNext()) { seed = e.next(); diff --git a/htroot/sharedBlacklist_p.java b/htroot/sharedBlacklist_p.java index 26704e693..7a4dfc5a1 100644 --- a/htroot/sharedBlacklist_p.java +++ b/htroot/sharedBlacklist_p.java @@ -104,8 +104,8 @@ public class sharedBlacklist_p { // generate the download URL
String downloadURL = null;
- if( sb.wordIndex.seedDB != null ){ //no nullpointer error..
- yacySeed seed = sb.wordIndex.seedDB.getConnected(Hash);
+ if( sb.webIndex.seedDB != null ){ //no nullpointer error..
+ yacySeed seed = sb.webIndex.seedDB.getConnected(Hash);
if (seed != null) {
String IP = seed.get(yacySeed.IP, "127.0.0.1");
String Port = seed.get(yacySeed.PORT, "8080");
diff --git a/htroot/www/welcome.java b/htroot/www/welcome.java index 5b7941a83..3a7e02212 100644 --- a/htroot/www/welcome.java +++ b/htroot/www/welcome.java @@ -73,7 +73,7 @@ public class welcome { prop.putHTML("peername", env.getConfig("peerName", "<nameless>"));
prop.putHTML("peerdomain", env.getConfig("peerName", "<nameless>").toLowerCase());
- prop.putHTML("peeraddress", sb.wordIndex.seedDB.mySeed().getPublicAddress());
+ prop.putHTML("peeraddress", sb.webIndex.seedDB.mySeed().getPublicAddress());
prop.put("hostname", serverDomains.myPublicIP());
try{
prop.put("hostip", InetAddress.getByName(serverDomains.myPublicIP()).getHostAddress());
@@ -83,7 +83,7 @@ public class welcome { prop.put("port", serverCore.getPortNr(env.getConfig("port","8080")));
prop.put("clientip", (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, ""));
- final String peertype = (sb.wordIndex.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_JUNIOR : sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
+ final String peertype = (sb.webIndex.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_JUNIOR : sb.webIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
final boolean senior = (peertype.equals(yacySeed.PEERTYPE_SENIOR)) || (peertype.equals(yacySeed.PEERTYPE_PRINCIPAL));
if (senior) { prop.put("couldcan", "can"); } else { prop.put("couldcan", "could"); }
if (senior) { prop.put("seniorinfo", "This peer runs in senior mode which means that your peer can be accessed using the addresses shown above."); } else { prop.putHTML("seniorinfo", "<b>Nobody can access your peer from the outside of your intranet. You must open your firewall and/or set a 'virtual server' in the settings of your router to enable access to the addresses as shown below.</b>"); }
diff --git a/htroot/xml/queues_p.java b/htroot/xml/queues_p.java index a880b4a8f..acdb6c056 100644 --- a/htroot/xml/queues_p.java +++ b/htroot/xml/queues_p.java @@ -55,9 +55,9 @@ import java.util.Locale; import de.anomic.crawler.CrawlEntry;
import de.anomic.crawler.NoticedURL;
+import de.anomic.crawler.IndexingStack;
import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.plasma.plasmaSwitchboardQueue;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacySeed;
@@ -86,25 +86,25 @@ public class queues_p { yacySeed initiator;
//indexing queue - prop.putNum("indexingSize", sb.getThread(plasmaSwitchboard.INDEXER).getJobCount() + sb.sbQueue.getActiveQueueSize());
+ prop.putNum("indexingSize", sb.getThread(plasmaSwitchboard.INDEXER).getJobCount() + sb.webIndex.queuePreStack.getActiveQueueSize());
prop.putNum("indexingMax", (int) sb.getConfigLong(plasmaSwitchboard.INDEXER_SLOTS, 30));
- prop.putNum("urlpublictextSize", sb.wordIndex.countURL());
- prop.putNum("rwipublictextSize", sb.wordIndex.size());
- if ((sb.sbQueue.size() == 0) && (sb.sbQueue.getActiveQueueSize() == 0)) { + prop.putNum("urlpublictextSize", sb.webIndex.countURL());
+ prop.putNum("rwipublictextSize", sb.webIndex.size());
+ if ((sb.webIndex.queuePreStack.size() == 0) && (sb.webIndex.queuePreStack.getActiveQueueSize() == 0)) { prop.put("list", "0"); //is empty
} else {
- plasmaSwitchboardQueue.QueueEntry pcentry;
+ IndexingStack.QueueEntry pcentry;
long totalSize = 0;
int i=0; //counter
// getting all entries that are currently in process
- ArrayList<plasmaSwitchboardQueue.QueueEntry> entryList = new ArrayList<plasmaSwitchboardQueue.QueueEntry>();
- entryList.addAll(sb.sbQueue.getActiveQueueEntries());
+ ArrayList<IndexingStack.QueueEntry> entryList = new ArrayList<IndexingStack.QueueEntry>();
+ entryList.addAll(sb.webIndex.queuePreStack.getActiveQueueEntries());
int inProcessCount = entryList.size();
// getting all enqueued entries
- if ((sb.sbQueue.size() > 0)) {
- Iterator<plasmaSwitchboardQueue.QueueEntry> i1 = sb.sbQueue.entryIterator(false);
+ if ((sb.webIndex.queuePreStack.size() > 0)) {
+ Iterator<IndexingStack.QueueEntry> i1 = sb.webIndex.queuePreStack.entryIterator(false);
while (i1.hasNext()) entryList.add(i1.next());
}
@@ -118,7 +118,7 @@ public class queues_p { if ((pcentry != null) && (pcentry.url() != null)) {
long entrySize = pcentry.size();
totalSize += entrySize;
- initiator = sb.wordIndex.seedDB.getConnected(pcentry.initiator());
+ initiator = sb.webIndex.seedDB.getConnected(pcentry.initiator());
prop.put("list-indexing_"+i+"_profile", (pcentry.profile() != null) ? pcentry.profile().name() : "deleted");
prop.put("list-indexing_"+i+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("list-indexing_"+i+"_depth", pcentry.depth());
@@ -145,7 +145,7 @@ public class queues_p { for (int i = 0; i < w.length; i++) {
if (w[i] == null) continue;
prop.put("list-loader_"+count+"_profile", w[i].profileHandle());
- initiator = sb.wordIndex.seedDB.getConnected(w[i].initiator());
+ initiator = sb.webIndex.seedDB.getConnected(w[i].initiator());
prop.put("list-loader_"+count+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("list-loader_"+count+"_depth", w[i].depth());
prop.putHTML("list-loader_"+count+"_url", w[i].url().toString(), true);
@@ -189,7 +189,7 @@ public class queues_p { for (int i = 0; i < crawlerList.length; i++) {
urle = crawlerList[i];
if ((urle != null) && (urle.url() != null)) {
- initiator = sb.wordIndex.seedDB.getConnected(urle.initiator());
+ initiator = sb.webIndex.seedDB.getConnected(urle.initiator());
prop.put(tableName + "_" + showNum + "_profile", urle.profileHandle());
prop.put(tableName + "_" + showNum + "_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put(tableName + "_" + showNum + "_depth", urle.depth());
diff --git a/htroot/xml/status_p.java b/htroot/xml/status_p.java index 11258d3eb..0ed2b5c0e 100644 --- a/htroot/xml/status_p.java +++ b/htroot/xml/status_p.java @@ -60,13 +60,13 @@ public class status_p { prop.setLocalized(false);
prop.put("rejected", "0");
yacyCore.peerActions.updateMySeed();
- final int cacheOutSize = sb.wordIndex.dhtOutCacheSize();
+ final int cacheOutSize = sb.webIndex.dhtOutCacheSize();
final long cacheMaxSize = sb.getConfigLong(plasmaSwitchboard.WORDCACHE_MAX_COUNT, 10000);
- prop.putNum("ppm", sb.wordIndex.seedDB.mySeed().getPPM());
- prop.putNum("qpm", sb.wordIndex.seedDB.mySeed().getQPM());
- prop.putNum("wordCacheSize", sb.wordIndex.dhtOutCacheSize() + sb.wordIndex.dhtInCacheSize());
+ prop.putNum("ppm", sb.webIndex.seedDB.mySeed().getPPM());
+ prop.putNum("qpm", sb.webIndex.seedDB.mySeed().getQPM());
+ prop.putNum("wordCacheSize", sb.webIndex.dhtOutCacheSize() + sb.webIndex.dhtInCacheSize());
prop.putNum("wordCacheWSize", cacheOutSize);
- prop.putNum("wordCacheKSize", sb.wordIndex.dhtInCacheSize());
+ prop.putNum("wordCacheKSize", sb.webIndex.dhtInCacheSize());
prop.putNum("wordCacheMaxSize", cacheMaxSize);
prop.put("wordCacheWCount", cacheOutSize);
prop.put("wordCacheMaxCount", cacheMaxSize);
diff --git a/htroot/yacy/crawlReceipt.java b/htroot/yacy/crawlReceipt.java index 0665a13d2..bfbbd7467 100644 --- a/htroot/yacy/crawlReceipt.java +++ b/htroot/yacy/crawlReceipt.java @@ -110,10 +110,10 @@ public final class crawlReceipt { */
- final yacySeed otherPeer = sb.wordIndex.seedDB.get(iam);
+ final yacySeed otherPeer = sb.webIndex.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
- if ((sb.wordIndex.seedDB.mySeed() == null) || (!(sb.wordIndex.seedDB.mySeed().hash.equals(youare)))) {
+ if ((sb.webIndex.seedDB.mySeed() == null) || (!(sb.webIndex.seedDB.mySeed().hash.equals(youare)))) {
// no yacy connection / unknown peers
prop.put("delay", "3600");
return prop;
@@ -155,7 +155,7 @@ public final class crawlReceipt { if (result.equals("fill")) try {
// put new entry into database
- sb.wordIndex.putURL(entry);
+ sb.webIndex.putURL(entry);
sb.crawlResults.stack(entry, youare, iam, 1);
sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
log.logInfo("crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + entry.hash() + ":" + comp.url().toNormalform(false, true));
diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 29ee8b622..2392600fb 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -194,15 +194,15 @@ public final class hello { serverCore.checkInterruption();
final StringBuffer seeds = new StringBuffer(768);
// attach some more seeds, as requested
- if ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.sizeConnected() > 0)) {
- if (count > sb.wordIndex.seedDB.sizeConnected()) { count = sb.wordIndex.seedDB.sizeConnected(); }
+ if ((sb.webIndex.seedDB != null) && (sb.webIndex.seedDB.sizeConnected() > 0)) {
+ if (count > sb.webIndex.seedDB.sizeConnected()) { count = sb.webIndex.seedDB.sizeConnected(); }
if (count > 100) { count = 100; }
// latest seeds
- final Map<String, yacySeed> ySeeds = sb.wordIndex.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation
+ final Map<String, yacySeed> ySeeds = sb.webIndex.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation
// attach also my own seed
- seeds.append("seed0=").append(sb.wordIndex.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
+ seeds.append("seed0=").append(sb.webIndex.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
count = 1;
// attach other seeds
@@ -222,7 +222,7 @@ public final class hello { }
} else {
// attach also my own seed
- seeds.append("seed0=").append(sb.wordIndex.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
+ seeds.append("seed0=").append(sb.webIndex.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
}
prop.put("seedlist", seeds.toString());
diff --git a/htroot/yacy/list.java b/htroot/yacy/list.java index 601d0a652..6f80e67a3 100644 --- a/htroot/yacy/list.java +++ b/htroot/yacy/list.java @@ -54,7 +54,7 @@ public final class list { String otherPeerName = null; if (post.containsKey("iam")) { - yacySeed bla = sb.wordIndex.seedDB.get(post.get("iam", "")); + yacySeed bla = sb.webIndex.seedDB.get(post.get("iam", "")); if (bla != null) otherPeerName = bla.getName(); } if (otherPeerName == null) otherPeerName = (String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP); diff --git a/htroot/yacy/message.java b/htroot/yacy/message.java index 35dc6b340..bd6b18157 100644 --- a/htroot/yacy/message.java +++ b/htroot/yacy/message.java @@ -92,7 +92,7 @@ public final class message { String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// check if we are the right target and requester has correct information about this peer
- if ((sb.wordIndex.seedDB.mySeed() == null) || (!(sb.wordIndex.seedDB.mySeed().hash.equals(youare)))) {
+ if ((sb.webIndex.seedDB.mySeed() == null) || (!(sb.webIndex.seedDB.mySeed().hash.equals(youare)))) {
// this request has a wrong target
prop.put("response", "-1"); // request rejected
return prop;
@@ -152,7 +152,7 @@ public final class message { sb.messageDB.write(msgEntry = sb.messageDB.newEntry(
"remote",
otherSeed.get(yacySeed.NAME, "anonymous"), otherSeed.hash,
- sb.wordIndex.seedDB.mySeed().getName(), sb.wordIndex.seedDB.mySeed().hash,
+ sb.webIndex.seedDB.mySeed().getName(), sb.webIndex.seedDB.mySeed().hash,
subject, mb));
messageForwardingViaEmail(sb, msgEntry);
@@ -198,7 +198,7 @@ public final class message { .append(sendMailTo)
.append("\nFrom: ")
.append("yacy@")
- .append(sb.wordIndex.seedDB.mySeed().getName())
+ .append(sb.webIndex.seedDB.mySeed().getName())
.append("\nSubject: [YaCy] ")
.append(msgEntry.subject().replace('\n', ' '))
.append("\nDate: ")
diff --git a/htroot/yacy/query.java b/htroot/yacy/query.java index 82a63ec6f..ecb699705 100644 --- a/htroot/yacy/query.java +++ b/htroot/yacy/query.java @@ -87,7 +87,7 @@ public final class query { prop.put("mytime", serverDate.formatShortSecond());
// check if we are the right target and requester has correct information about this peer
- if (sb.wordIndex.seedDB.mySeed() == null || !sb.wordIndex.seedDB.mySeed().hash.equals(youare)) {
+ if (sb.webIndex.seedDB.mySeed() == null || !sb.webIndex.seedDB.mySeed().hash.equals(youare)) {
// this request has a wrong target
prop.put("response", "-1"); // request rejected
return prop;
@@ -97,19 +97,19 @@ public final class query { if (obj.equals("rwiurlcount")) {
// the total number of different urls in the rwi is returned
// <env> shall contain a word hash, the number of assigned lurls to this hash is returned
- prop.put("response", sb.wordIndex.indexSize(env));
+ prop.put("response", sb.webIndex.indexSize(env));
return prop;
}
if (obj.equals("rwicount")) {
// return the total number of available word indexes
- prop.put("response", sb.wordIndex.size());
+ prop.put("response", sb.webIndex.size());
return prop;
}
if (obj.equals("lurlcount")) {
// return the number of all available l-url's
- prop.put("response", sb.wordIndex.countURL());
+ prop.put("response", sb.webIndex.countURL());
return prop;
}
diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 64b50ac7d..b91304f32 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -128,7 +128,7 @@ public final class search { // store accessing peer
yacySeed remoteSeed = yacySeed.genRemoteSeed(oseed, key, true);
- if (sb.wordIndex.seedDB == null) {
+ if (sb.webIndex.seedDB == null) {
yacyCore.log.logSevere("yacy.search: seed cache not initialized");
} else {
yacyCore.peerActions.peerArrival(remoteSeed, true);
@@ -156,7 +156,7 @@ public final class search { yacyCore.log.logInfo("INIT HASH SEARCH (abstracts only): " + plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes) + " - " + theQuery.displayResults() + " links");
long timer = System.currentTimeMillis();
- Map<String, indexContainer>[] containers = sb.wordIndex.localSearchContainers(theQuery, plasmaSearchQuery.hashes2Set(urls));
+ Map<String, indexContainer>[] containers = sb.webIndex.localSearchContainers(theQuery, plasmaSearchQuery.hashes2Set(urls));
serverProfiling.update("SEARCH", new plasmaProfiling.searchEvent(theQuery.id(true), plasmaSearchEvent.COLLECTION, containers[0].size(), System.currentTimeMillis() - timer));
if (containers != null) {
Iterator<Map.Entry<String, indexContainer>> ci = containers[0].entrySet().iterator();
@@ -183,7 +183,7 @@ public final class search { RSSFeed.channels(RSSFeed.REMOTESEARCH).addMessage(new RSSMessage("Remote Search Request from " + remoteSeed.getName(), plasmaSearchQuery.anonymizedQueryHashes(theQuery.queryHashes), ""));
// make event
- theSearch = plasmaSearchEvent.getEvent(theQuery, rankingProfile, sb.wordIndex, sb.crawlResults, null, true);
+ theSearch = plasmaSearchEvent.getEvent(theQuery, rankingProfile, sb.webIndex, sb.crawlResults, null, true);
// set statistic details of search result and find best result index set
if (theSearch.getRankingResult().getLocalResourceSize() == 0) {
@@ -285,7 +285,7 @@ public final class search { prop.put("fwrec", ""); // peers that would have helped to construct this result (recommendations)
// prepare search statistics
- theQuery.remotepeer = sb.wordIndex.seedDB.lookupByIP(natLib.getInetAddress(client), true, false, false);
+ theQuery.remotepeer = sb.webIndex.seedDB.lookupByIP(natLib.getInetAddress(client), true, false, false);
theQuery.resultcount = (theSearch == null) ? 0 : theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
theQuery.searchtime = System.currentTimeMillis() - timestamp;
theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime();
@@ -306,8 +306,8 @@ public final class search { prop.put("searchtime", System.currentTimeMillis() - timestamp);
final int links = Integer.parseInt(prop.get("linkcount","0"));
- sb.wordIndex.seedDB.mySeed().incSI(links);
- sb.wordIndex.seedDB.mySeed().incSU(links);
+ sb.webIndex.seedDB.mySeed().incSI(links);
+ sb.webIndex.seedDB.mySeed().incSU(links);
return prop;
}
diff --git a/htroot/yacy/transfer.java b/htroot/yacy/transfer.java index bcfd077f5..4023f8d12 100644 --- a/htroot/yacy/transfer.java +++ b/htroot/yacy/transfer.java @@ -85,7 +85,7 @@ public final class transfer { return prop;
}
- yacySeed otherseed = sb.wordIndex.seedDB.get(otherpeer);
+ yacySeed otherseed = sb.webIndex.seedDB.get(otherpeer);
if ((otherseed == null) || (filename.indexOf("..") >= 0)) {
// reject unknown peers: this does not appear fair, but anonymous senders are dangerous
// reject paths that contain '..' because they are dangerous
@@ -105,7 +105,7 @@ public final class transfer { String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(otherpeer + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis()));
prop.put("response", "ok");
prop.put("process_access", access);
- prop.put("process_address", sb.wordIndex.seedDB.mySeed().getPublicAddress());
+ prop.put("process_address", sb.webIndex.seedDB.mySeed().getPublicAddress());
prop.put("process_protocol", "http");
prop.put("process_path", ""); // currently empty; the store process will find a path
prop.put("process_maxsize", "-1"); // if response is too big we return the size of the file
diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index ace2ed616..c554c0472 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -88,7 +88,7 @@ public final class transferRWI { boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true");
boolean checkLimit = sb.getConfigBool("indexDistribution.transferRWIReceiptLimitEnabled", true);
final long cachelimit = sb.getConfigLong("indexDistribution.dhtReceiptLimit", 10000);
- final yacySeed otherPeer = sb.wordIndex.seedDB.get(iam);
+ final yacySeed otherPeer = sb.webIndex.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
// response values
@@ -96,8 +96,8 @@ public final class transferRWI { StringBuffer unknownURLs = new StringBuffer();
int pause = 10000;
- if ((youare == null) || (!youare.equals(sb.wordIndex.seedDB.mySeed().hash))) {
- sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.wordIndex.seedDB.mySeed().hash);
+ if ((youare == null) || (!youare.equals(sb.webIndex.seedDB.mySeed().hash))) {
+ sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.webIndex.seedDB.mySeed().hash);
result = "wrong_target";
pause = 0;
} else if ((!granted) || (sb.isRobinsonMode())) {
@@ -105,9 +105,9 @@ public final class transferRWI { sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Not granted.");
result = "not_granted";
pause = 0;
- } else if (checkLimit && sb.wordIndex.dhtInCacheSize() > cachelimit) {
+ } else if (checkLimit && sb.webIndex.dhtInCacheSize() > cachelimit) {
// we are too busy to receive indexes
- sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (buffersize=" + sb.wordIndex.dhtInCacheSize() + ").");
+ sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". We are too busy (buffersize=" + sb.webIndex.dhtInCacheSize() + ").");
granted = false; // don't accept more words if there are too many words to flush
result = "busy";
pause = 60000;
@@ -168,12 +168,12 @@ public final class transferRWI { }
// learn entry
- sb.wordIndex.addEntry(wordHash, iEntry, System.currentTimeMillis(), true);
+ sb.webIndex.addEntry(wordHash, iEntry, System.currentTimeMillis(), true);
serverCore.checkInterruption();
// check if we need to ask for the corresponding URL
if (!(knownURL.contains(urlHash)||unknownURL.contains(urlHash))) try {
- if (sb.wordIndex.existsURL(urlHash)) {
+ if (sb.webIndex.existsURL(urlHash)) {
knownURL.add(urlHash);
} else {
unknownURL.add(urlHash);
@@ -186,7 +186,7 @@ public final class transferRWI { }
received++;
}
- sb.wordIndex.seedDB.mySeed().incRI(received);
+ sb.webIndex.seedDB.mySeed().incRI(received);
// finally compose the unknownURL hash list
final Iterator<String> it = unknownURL.iterator();
@@ -198,14 +198,14 @@ public final class transferRWI { if ((wordhashes.length == 0) || (received == 0)) {
sb.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");
} else {
- final double avdist = (yacyDHTAction.dhtDistance(sb.wordIndex.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(sb.wordIndex.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0;
+ final double avdist = (yacyDHTAction.dhtDistance(sb.webIndex.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(sb.webIndex.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0;
sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs");
RSSFeed.channels(RSSFeed.INDEXRECEIVE).addMessage(new RSSMessage("Received " + received + " RWIs [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + otherPeerName + ", requesting " + unknownURL.size() + " URLs, blocked " + blocked, "", ""));
}
result = "ok";
if (checkLimit) {
- pause = (sb.wordIndex.dhtInCacheSize() < 500) ? 0 : sb.wordIndex.dhtInCacheSize(); // estimation of necessary pause time
+ pause = (sb.webIndex.dhtInCacheSize() < 500) ? 0 : sb.webIndex.dhtInCacheSize(); // estimation of necessary pause time
}
}
diff --git a/htroot/yacy/transferURL.java b/htroot/yacy/transferURL.java index 535f50a23..ffe588877 100644 --- a/htroot/yacy/transferURL.java +++ b/htroot/yacy/transferURL.java @@ -88,11 +88,11 @@ public final class transferURL { String result = "";
String doublevalues = "0";
- final yacySeed otherPeer = sb.wordIndex.seedDB.get(iam);
+ final yacySeed otherPeer = sb.webIndex.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
- if ((youare == null) || (!youare.equals(sb.wordIndex.seedDB.mySeed().hash))) {
- sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.wordIndex.seedDB.mySeed().hash);
+ if ((youare == null) || (!youare.equals(sb.webIndex.seedDB.mySeed().hash))) {
+ sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.webIndex.seedDB.mySeed().hash);
result = "wrong_target";
} else if ((!granted) || (sb.isRobinsonMode())) {
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Not granted.");
@@ -100,7 +100,7 @@ public final class transferURL { } else {
int received = 0;
int blocked = 0;
- final int sizeBefore = sb.wordIndex.countURL();
+ final int sizeBefore = sb.webIndex.countURL();
// read the urls from the other properties and store
String urls;
indexURLReference lEntry;
@@ -157,7 +157,7 @@ public final class transferURL { // write entry to database
try {
- sb.wordIndex.putURL(lEntry);
+ sb.webIndex.putURL(lEntry);
sb.crawlResults.stack(lEntry, iam, iam, 3);
yacyCore.log.logFine("transferURL: received URL '" + comp.url().toNormalform(false, true) + "' from peer " + otherPeerName);
received++;
@@ -166,10 +166,10 @@ public final class transferURL { }
}
- sb.wordIndex.seedDB.mySeed().incRU(received);
+ sb.webIndex.seedDB.mySeed().incRU(received);
// return rewrite properties
- final int more = sb.wordIndex.countURL() - sizeBefore;
+ final int more = sb.webIndex.countURL() - sizeBefore;
doublevalues = Integer.toString(received - more);
sb.getLog().logInfo("Received " + received + " URLs from peer " + otherPeerName + " in " + (System.currentTimeMillis() - start) + " ms, blocked " + blocked + " URLs");
RSSFeed.channels(RSSFeed.INDEXRECEIVE).addMessage(new RSSMessage("Received " + received + " URLs from peer " + otherPeerName + ", blocked " + blocked, "", ""));
diff --git a/htroot/yacy/ui/result.java b/htroot/yacy/ui/result.java index 5af8e0a68..16e27ac05 100644 --- a/htroot/yacy/ui/result.java +++ b/htroot/yacy/ui/result.java @@ -162,7 +162,7 @@ public class result { } // prepare search properties - final boolean yacyonline = ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.mySeed() != null) && (sb.wordIndex.seedDB.mySeed().getPublicAddress() != null)); + final boolean yacyonline = ((sb.webIndex.seedDB != null) && (sb.webIndex.seedDB.mySeed() != null) && (sb.webIndex.seedDB.mySeed().getPublicAddress() != null)); final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false)); // do the search @@ -206,7 +206,7 @@ public class result { theQuery.setOffset(0); // in case that this is a new search, always start without a offset offset = 0; } - plasmaSearchEvent theSearch = plasmaSearchEvent.getEvent(theQuery, ranking, sb.wordIndex, sb.crawlResults, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false); + plasmaSearchEvent theSearch = plasmaSearchEvent.getEvent(theQuery, ranking, sb.webIndex, sb.crawlResults, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false); // generate result object serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER ORDERING OF SEARCH RESULTS: " + ((System.currentTimeMillis() - timestamp) / 1000) + " seconds"); diff --git a/htroot/yacy/ui/ymarks.java b/htroot/yacy/ui/ymarks.java index b58abd4fb..4741adc63 100644 --- a/htroot/yacy/ui/ymarks.java +++ b/htroot/yacy/ui/ymarks.java @@ -110,7 +110,7 @@ public class ymarks { */ // set peer address - final String address = sb.wordIndex.seedDB.mySeed().getPublicAddress(); + final String address = sb.webIndex.seedDB.mySeed().getPublicAddress(); prop.put("address", address); //defaultvalues @@ -194,7 +194,7 @@ public class ymarks { bookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(urlHash); if (bookmark == null) { // try to get the bookmark from the LURL database - indexURLReference urlentry = sb.wordIndex.getURL(urlHash, null, 0); + indexURLReference urlentry = sb.webIndex.getURL(urlHash, null, 0); plasmaParserDocument document = null; if (urlentry != null) { indexURLReference.Components comp = urlentry.comp(); @@ -442,7 +442,7 @@ public class ymarks { map.put("title", title.replace(',', ' ')); map.put("description", description.replace(',', ' ')); map.put("tags", tagsString.replace(',', ' ')); - sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); + sb.webIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); } } diff --git a/htroot/yacy/urls.java b/htroot/yacy/urls.java index af5bb3f37..4e83fc4fb 100644 --- a/htroot/yacy/urls.java +++ b/htroot/yacy/urls.java @@ -45,7 +45,7 @@ public class urls { // insert default values serverObjects prop = new serverObjects(); - prop.put("iam", sb.wordIndex.seedDB.mySeed().hash); + prop.put("iam", sb.webIndex.seedDB.mySeed().hash); prop.put("response", "rejected - insufficient call parameters"); prop.put("channel_title", ""); prop.put("channel_description", ""); @@ -77,7 +77,7 @@ public class urls { sb.crawlQueues.delegatedURL.push( sb.crawlQueues.delegatedURL.newEntry( entry, - sb.wordIndex.seedDB.mySeed().hash, + sb.webIndex.seedDB.mySeed().hash, new Date(), 0, "client=____________") @@ -108,7 +108,7 @@ public class urls { indexURLReference.Components comp; yacyURL referrer; for (int i = 0; i < count; i++) { - entry = sb.wordIndex.getURL(urlhashes.substring(12 * i, 12 * (i + 1)), null, 0); + entry = sb.webIndex.getURL(urlhashes.substring(12 * i, 12 * (i + 1)), null, 0); if (entry == null) continue; // find referrer, if there is one referrer = sb.getURL(entry.referrerHash()); diff --git a/htroot/yacy/user/ysearch.java b/htroot/yacy/user/ysearch.java index 012b26110..b835bfecb 100644 --- a/htroot/yacy/user/ysearch.java +++ b/htroot/yacy/user/ysearch.java @@ -162,7 +162,7 @@ public class ysearch { }
// prepare search properties
- final boolean yacyonline = ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.mySeed() != null) && (sb.wordIndex.seedDB.mySeed().getPublicAddress() != null));
+ final boolean yacyonline = ((sb.webIndex.seedDB != null) && (sb.webIndex.seedDB.mySeed() != null) && (sb.webIndex.seedDB.mySeed().getPublicAddress() != null));
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
// do the search
@@ -206,7 +206,7 @@ public class ysearch { theQuery.setOffset(0); // in case that this is a new search, always start without a offset
offset = 0;
}
- plasmaSearchEvent theSearch = plasmaSearchEvent.getEvent(theQuery, ranking, sb.wordIndex, sb.crawlResults, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false);
+ plasmaSearchEvent theSearch = plasmaSearchEvent.getEvent(theQuery, ranking, sb.webIndex, sb.crawlResults, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false);
// generate result object
serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER ORDERING OF SEARCH RESULTS: " + ((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index b1e255d14..078ff071a 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -202,14 +202,14 @@ public class yacysearch { // delete the index entry locally
final String delHash = post.get("deleteref", ""); // urlhash
- sb.wordIndex.removeWordReferences(query[0], delHash);
+ sb.webIndex.removeWordReferences(query[0], delHash);
// make new news message with negative voting
HashMap<String, String> map = new HashMap<String, String>();
map.put("urlhash", delHash);
map.put("vote", "negative");
map.put("refid", "");
- sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
+ sb.webIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
}
// if a plus-button was hit, create new voting message
@@ -219,7 +219,7 @@ public class yacysearch { return prop;
}
final String recommendHash = post.get("recommendref", ""); // urlhash
- indexURLReference urlentry = sb.wordIndex.getURL(recommendHash, null, 0);
+ indexURLReference urlentry = sb.webIndex.getURL(recommendHash, null, 0);
if (urlentry != null) {
indexURLReference.Components comp = urlentry.comp();
plasmaParserDocument document;
@@ -232,14 +232,14 @@ public class yacysearch { map.put("description", ((document == null) ? comp.dc_title() : document.dc_title()).replace(',', ' '));
map.put("author", ((document == null) ? "" : document.dc_creator()));
map.put("tags", ((document == null) ? "" : document.dc_subject(' ')));
- sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_ADD, map));
+ sb.webIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.webIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_ADD, map));
document.close();
}
}
}
// prepare search properties
- final boolean yacyonline = ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.mySeed() != null) && (sb.wordIndex.seedDB.mySeed().getPublicAddress() != null));
+ final boolean yacyonline = ((sb.webIndex.seedDB != null) && (sb.webIndex.seedDB.mySeed() != null) && (sb.webIndex.seedDB.mySeed().getPublicAddress() != null));
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
// do the search
@@ -282,7 +282,7 @@ public class yacysearch { theQuery.setOffset(0); // in case that this is a new search, always start without a offset
offset = 0;
}
- plasmaSearchEvent theSearch = plasmaSearchEvent.getEvent(theQuery, ranking, sb.wordIndex, sb.crawlResults, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false);
+ plasmaSearchEvent theSearch = plasmaSearchEvent.getEvent(theQuery, ranking, sb.webIndex, sb.crawlResults, (sb.isRobinsonMode()) ? sb.clusterhashes : null, false);
// generate result object
serverLog.logFine("LOCAL_SEARCH", "SEARCH TIME AFTER ORDERING OF SEARCH RESULTS: " + ((System.currentTimeMillis() - timestamp) / 1000) + " seconds");
diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index 77846b2d7..17acb16e7 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -191,7 +191,7 @@ public class yacysearchitem { prop.put("content", theQuery.contentdom + 1); // switch on specific content prop.put("content_authorized", authenticated ? "1" : "0"); - prop.put("content_authorized_recommend", (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? "1" : "0"); + prop.put("content_authorized_recommend", (sb.webIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? "1" : "0"); prop.put("content_authorized_recommend_deletelink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&deleteref=" + result.hash() + "&urlmaskfilter=.*"); prop.put("content_authorized_recommend_recommendlink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&recommendref=" + result.hash() + "&urlmaskfilter=.*"); prop.put("content_authorized_urlhash", result.hash()); |
