diff options
110 files changed, 1099 insertions, 2113 deletions
diff --git a/.classpath b/.classpath index c93b44b06..ee599b1e0 100644 --- a/.classpath +++ b/.classpath @@ -39,5 +39,6 @@ <classpathentry kind="lib" path="libx/bcprov-jdk14-132.jar"/> <classpathentry kind="lib" path="libx/commons-codec-1.3.jar"/> <classpathentry kind="lib" path="lib/commons-httpclient-3.1.jar"/> + <classpathentry kind="lib" path="/Users/admin/Documents/workspace/yacy/lib/commons-httpclient-3.1.jar"/> <classpathentry kind="output" path="gen"/> </classpath> diff --git a/defaults/yacy.init b/defaults/yacy.init index 26b1860a7..1516677dd 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -624,22 +624,6 @@ proxy.sendViaHeader=true # Specifies if the proxy should send the X-Forwarded-For header
proxy.sendXForwardedForHeader=true
-# Configuration options needed to configure server port forwarding
-portForwarding.Enabled=false
-portForwarding.Type=none
-
-# port forwarding via sch
-portForwarding.sch=de.anomic.server.portForwarding.sch.serverPortForwardingSch
-portForwarding.sch.UseProxy=false
-portForwarding.sch.Port=
-portForwarding.sch.Host=
-portForwarding.sch.HostPort=22
-portForwarding.sch.HostUser=
-portForwarding.sch.HostPwd=
-
-# port forwarding via upnp
-portForwarding.upnp=de.anomic.server.portForwarding.upnp.serverPortForwardingUpnp
-
# msgForwarding: Specifies if yacy should forward received messages via
# email to the configured email address
msgForwardingEnabled=false
diff --git a/htroot/AccessTracker_p.java b/htroot/AccessTracker_p.java index 7543596c1..f9d712040 100644 --- a/htroot/AccessTracker_p.java +++ b/htroot/AccessTracker_p.java @@ -40,7 +40,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverDate; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySeed; public class AccessTracker_p { @@ -53,8 +52,8 @@ public class AccessTracker_p { return accessClone; } - public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> sb) { - plasmaSwitchboard switchboard = (plasmaSwitchboard) sb; + public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) { + plasmaSwitchboard sb = (plasmaSwitchboard) env; // return variable that accumulates replacements serverObjects prop = new serverObjects(); @@ -66,14 +65,14 @@ public class AccessTracker_p { int maxCount = 1000; boolean dark = true; if (page == 0) { - Iterator<String> i = switchboard.accessHosts(); + Iterator<String> i = sb.accessHosts(); String host; TreeMap<Long, String> access; int entCount = 0; try { while ((entCount < maxCount) && (i.hasNext())) { host = (String) i.next(); - access = switchboard.accessTrack(host); + access = sb.accessTrack(host); prop.putHTML("page_list_" + entCount + "_host", host); prop.putNum("page_list_" + entCount + "_countSecond", access.tailMap(new Long(System.currentTimeMillis() - 1000)).size()); prop.putNum("page_list_" + entCount + "_countMinute", access.tailMap(new Long(System.currentTimeMillis() - 1000 * 60)).size()); @@ -91,7 +90,7 @@ public class AccessTracker_p { TreeMap<Long, String> access; Map.Entry<Long, String> entry; if (host.length() > 0) { - access = switchboard.accessTrack(host); + access = sb.accessTrack(host); if (access != null) { try { Iterator<Map.Entry<Long, String>> ii = treemapclone(access).entrySet().iterator(); @@ -106,10 +105,10 @@ public class AccessTracker_p { } } else { try { - Iterator<String> i = switchboard.accessHosts(); + Iterator<String> i = sb.accessHosts(); while ((entCount < maxCount) && (i.hasNext())) { host = (String) i.next(); - access = switchboard.accessTrack(host); + access = sb.accessTrack(host); Iterator<Map.Entry<Long, String>> ii = treemapclone(access).entrySet().iterator(); while (ii.hasNext()) { entry = ii.next(); @@ -125,7 +124,7 @@ public class AccessTracker_p { prop.put("page_num", entCount); } if ((page == 2) || (page == 4)) { - ArrayList<plasmaSearchQuery> array = (page == 2) ? switchboard.localSearches : switchboard.remoteSearches; + ArrayList<plasmaSearchQuery> array = (page == 2) ? sb.localSearches : sb.remoteSearches; plasmaSearchQuery searchProfile; int m = Math.min(maxCount, array.size()); long qcountSum = 0; @@ -173,10 +172,10 @@ public class AccessTracker_p { prop.putNum("page_urltime_avg", (double) utimeSum / m); prop.putNum("page_snippettime_avg", (double) stimeSum / m); prop.putNum("page_resulttime_avg", (double) rtimeSum / m); - prop.putNum("page_total", (page == 2) ? switchboard.localSearches.size() : switchboard.remoteSearches.size()); + prop.putNum("page_total", (page == 2) ? sb.localSearches.size() : sb.remoteSearches.size()); } if ((page == 3) || (page == 5)) { - Iterator<Entry<String, TreeSet<Long>>> i = (page == 3) ? switchboard.localSearchTracker.entrySet().iterator() : switchboard.remoteSearchTracker.entrySet().iterator(); + Iterator<Entry<String, TreeSet<Long>>> i = (page == 3) ? sb.localSearchTracker.entrySet().iterator() : sb.remoteSearchTracker.entrySet().iterator(); String host; TreeSet<Long> handles; int entCount = 0; @@ -204,7 +203,7 @@ public class AccessTracker_p { prop.put("page_list_" + entCount + "_dark", ((dark) ? 1 : 0) ); dark =! dark; prop.putHTML("page_list_" + entCount + "_host", host); if (page == 5) { - yacySeed remotepeer = yacyCore.seedDB.lookupByIP(natLib.getInetAddress(host), true, true, true); + yacySeed remotepeer = sb.wordIndex.seedDB.lookupByIP(natLib.getInetAddress(host), true, true, true); prop.putHTML("page_list_" + entCount + "_peername", (remotepeer == null) ? "UNKNOWN" : remotepeer.getName()); } prop.putNum("page_list_" + entCount + "_count", handles.size()); @@ -215,7 +214,7 @@ public class AccessTracker_p { } catch (ConcurrentModificationException e) {} // we dont want to synchronize this prop.put("page_list", entCount); prop.putNum("page_num", entCount); - prop.putNum("page_total", (page == 3) ? switchboard.localSearches.size() : switchboard.remoteSearches.size()); + prop.putNum("page_total", (page == 3) ? sb.localSearches.size() : sb.remoteSearches.size()); prop.putNum("page_qph_sum", qphSum); } // return rewrite properties diff --git a/htroot/Banner.java b/htroot/Banner.java index 8b1cc88b9..400dced39 100644 --- a/htroot/Banner.java +++ b/htroot/Banner.java @@ -46,10 +46,10 @@ import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaGrafics; +import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.ymage.ymageMatrix; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySeed; import java.awt.image.BufferedImage; @@ -61,7 +61,7 @@ import javax.imageio.ImageIO; public class Banner { public static ymageMatrix respond(httpHeader header, serverObjects post, serverSwitch<?> env) throws IOException { - + plasmaSwitchboard sb = (plasmaSwitchboard) env; final String IMAGE = "htroot/env/grafics/yacy.gif"; int width = 468; int height = 60; @@ -84,14 +84,14 @@ public class Banner { double myqph = 0; String type = ""; String network = env.getConfig("network.unit.name", "unspecified").toUpperCase(); - int peers = yacyCore.seedDB.sizeConnected() + 1; // the '+ 1': the own peer is not included in sizeConnected() - long nlinks = yacyCore.seedDB.countActiveURL(); - long nwords = yacyCore.seedDB.countActiveRWI(); - double nqpm = yacyCore.seedDB.countActiveQPM(); - long nppm = yacyCore.seedDB.countActivePPM(); + int peers = sb.wordIndex.seedDB.sizeConnected() + 1; // the '+ 1': the own peer is not included in sizeConnected() + long nlinks = sb.wordIndex.seedDB.countActiveURL(); + long nwords = sb.wordIndex.seedDB.countActiveRWI(); + double nqpm = sb.wordIndex.seedDB.countActiveQPM(); + long nppm = sb.wordIndex.seedDB.countActivePPM(); double nqph = 0; - yacySeed seed = yacyCore.seedDB.mySeed(); + yacySeed seed = sb.wordIndex.seedDB.mySeed(); if (seed != null){ name = seed.get(yacySeed.NAME, "-").toUpperCase(); links = Long.parseLong(seed.get(yacySeed.LCOUNT, "0")); @@ -99,19 +99,19 @@ public class Banner { myppm = seed.getPPM(); myqph = 60d * seed.getQPM(); - if (yacyCore.seedDB.mySeed().isVirgin()) { + if (sb.wordIndex.seedDB.mySeed().isVirgin()) { type = "VIRGIN"; nqph = Math.round(6000d * nqpm) / 100d; - } else if(yacyCore.seedDB.mySeed().isJunior()) { + } else if(sb.wordIndex.seedDB.mySeed().isJunior()) { type = "JUNIOR"; nqph = Math.round(6000d * nqpm) / 100d; - } else if(yacyCore.seedDB.mySeed().isSenior()) { + } else if(sb.wordIndex.seedDB.mySeed().isSenior()) { type = "SENIOR"; nlinks = nlinks + links; nwords = nwords + words; nqph = Math.round(6000d * nqpm + 100d * myqph) / 100d; nppm = nppm + myppm; - } else if(yacyCore.seedDB.mySeed().isPrincipal()) { + } else if(sb.wordIndex.seedDB.mySeed().isPrincipal()) { type = "PRINCIPAL"; nlinks = nlinks + links; nwords = nwords + words; diff --git a/htroot/Blacklist_p.java b/htroot/Blacklist_p.java index 0f16994f4..0f548a7bc 100644 --- a/htroot/Blacklist_p.java +++ b/htroot/Blacklist_p.java @@ -65,7 +65,6 @@ import de.anomic.index.indexReferenceBlacklist; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@@ -75,7 +74,8 @@ public class Blacklist_p { private final static String BLACKLIST_SHARED = "BlackLists.Shared";
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
-
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
+
// initialize the list manager
listManager.switchboard = (plasmaSwitchboard) env;
listManager.listsPath = new File(listManager.switchboard.getRootPath(),listManager.switchboard.getConfig("listManager.listsPath", "DATA/LISTS"));
@@ -321,11 +321,11 @@ prop.putHTML("asd", "0"); // List known hosts for BlackList retrieval
- if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) { // no nullpointer error
+ if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0) { // no nullpointer error
int peerCount = 0;
try {
TreeMap<String, String> hostList = new TreeMap<String, String>();
- final Iterator<yacySeed> e = yacyCore.seedDB.seedsConnected(true, false, null, (float) 0.0);
+ final Iterator<yacySeed> e = sb.wordIndex.seedDB.seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
yacySeed seed = (yacySeed) e.next();
if (seed != null) hostList.put(seed.get(yacySeed.NAME, "nameless"),seed.hash);
diff --git a/htroot/Blog.java b/htroot/Blog.java index f4aa7e978..f276e6a99 100644 --- a/htroot/Blog.java +++ b/htroot/Blog.java @@ -60,7 +60,6 @@ import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyNewsRecord; @@ -76,11 +75,11 @@ public class Blog { } public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) { - final plasmaSwitchboard switchboard = (plasmaSwitchboard) env; + final plasmaSwitchboard sb = (plasmaSwitchboard) env; final serverObjects prop = new serverObjects(); blogBoard.BlogEntry page = null; - boolean hasRights = switchboard.verifyAuthentication(header, true); + boolean hasRights = sb.verifyAuthentication(header, true); //final int display = (hasRights || post == null) ? 1 : post.getInt("display", 0); //prop.put("display", display); @@ -88,7 +87,7 @@ public class Blog { final boolean xml = ((String)header.get(httpHeader.CONNECTION_PROP_PATH)).endsWith(".xml"); - final String address = yacyCore.seedDB.mySeed().getPublicAddress(); + final String address = sb.wordIndex.seedDB.mySeed().getPublicAddress(); if(hasRights) { prop.put("mode_admin", "1"); @@ -97,16 +96,16 @@ public class Blog { } if (post == null) { - prop.putHTML("peername", yacyCore.seedDB.mySeed().getName()); + prop.putHTML("peername", sb.wordIndex.seedDB.mySeed().getName()); prop.put("address", address); - return putBlogDefault(prop, switchboard, address, 0, 10, hasRights, xml); + return putBlogDefault(prop, sb, address, 0, 10, hasRights, xml); } final int start = post.getInt("start",0); //indicates from where entries should be shown final int num = post.getInt("num",10); //indicates how many entries should be shown if(!hasRights){ - final userDB.Entry userentry = switchboard.userDB.proxyAuth((String)header.get("Authorization", "xxxxxx")); + final userDB.Entry userentry = sb.userDB.proxyAuth((String)header.get("Authorization", "xxxxxx")); if(userentry != null && userentry.hasRight(userDB.Entry.BLOG_RIGHT)){ hasRights=true; } else if(post.containsKey("login")) { @@ -121,13 +120,13 @@ public class Blog { String StrAuthor = post.get("author", ""); if (StrAuthor.equals("anonymous")) { - StrAuthor = switchboard.blogDB.guessAuthor(ip); + StrAuthor = sb.blogDB.guessAuthor(ip); if (StrAuthor == null || StrAuthor.length() == 0) { - if (de.anomic.yacy.yacyCore.seedDB.mySeed() == null) { + if (sb.wordIndex.seedDB.mySeed() == null) { StrAuthor = "anonymous"; } else { - StrAuthor = de.anomic.yacy.yacyCore.seedDB.mySeed().get("Name", "anonymous"); + StrAuthor = sb.wordIndex.seedDB.mySeed().get("Name", "anonymous"); } } } @@ -140,12 +139,12 @@ public class Blog { } if(hasRights && post.containsKey("delete") && post.get("delete").equals("sure")) { - page = switchboard.blogDB.readBlogEntry(pagename); + page = sb.blogDB.readBlogEntry(pagename); final Iterator<String> i = page.getComments().iterator(); while(i.hasNext()) { - switchboard.blogCommentDB.delete(i.next()); + sb.blogCommentDB.delete(i.next()); } - switchboard.blogDB.deleteBlogEntry(pagename); + sb.blogDB.deleteBlogEntry(pagename); pagename = DEFAULT_PAGE; } @@ -169,7 +168,7 @@ public class Blog { if(pagename.equals(DEFAULT_PAGE)) { pagename = String.valueOf(System.currentTimeMillis()); } else { - page = switchboard.blogDB.readBlogEntry(pagename); + page = sb.blogDB.readBlogEntry(pagename); comments = page.getComments(); date = page.getDate(); } @@ -182,17 +181,17 @@ public class Blog { subject = StrSubject.getBytes(); } - switchboard.blogDB.writeBlogEntry(switchboard.blogDB.newEntry(pagename, subject, author, ip, date, content, comments, commentMode)); + sb.blogDB.writeBlogEntry(sb.blogDB.newEntry(pagename, subject, author, ip, date, content, comments, commentMode)); // create a news message final HashMap<String, String> map = new HashMap<String, String>(); map.put("page", pagename); map.put("subject", StrSubject.replace(',', ' ')); map.put("author", StrAuthor.replace(',', ' ')); - yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_BLOG_ADD, map)); + sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BLOG_ADD, map)); } - page = switchboard.blogDB.readBlogEntry(pagename); //maybe "if(page == null)" + page = sb.blogDB.readBlogEntry(pagename); //maybe "if(page == null)" if (post.containsKey("edit")) { //edit an entry @@ -253,7 +252,7 @@ public class Blog { } else if (post.containsKey("xmlfile")) { prop.put("mode", "5"); - if(switchboard.blogDB.importXML(post.get("xmlfile$file"))) { + if(sb.blogDB.importXML(post.get("xmlfile$file"))) { prop.put("mode_state", "1"); } else { @@ -266,10 +265,10 @@ public class Blog { if(pagename.equals(DEFAULT_PAGE)) { // XXX: where are "peername" and "address" used in the template? // XXX: "clientname" is already set to the peername, no need for a new setting - prop.putHTML("peername", yacyCore.seedDB.mySeed().getName(), xml); + prop.putHTML("peername", sb.wordIndex.seedDB.mySeed().getName(), xml); prop.put("address", address); //index all entries - putBlogDefault(prop, switchboard, address, start, num, hasRights, xml); + putBlogDefault(prop, sb, address, start, num, hasRights, xml); } else { //only show 1 entry diff --git a/htroot/BlogComments.java b/htroot/BlogComments.java index e8a8b3bb7..ed75f3c0b 100644 --- a/htroot/BlogComments.java +++ b/htroot/BlogComments.java @@ -77,10 +77,10 @@ public class BlogComments { } public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) { - plasmaSwitchboard switchboard = (plasmaSwitchboard) env; + plasmaSwitchboard sb = (plasmaSwitchboard) env; serverObjects prop = new serverObjects(); blogBoard.BlogEntry page = null; - boolean hasRights = switchboard.verifyAuthentication(header, true); + boolean hasRights = sb.verifyAuthentication(header, true); if (hasRights) prop.put("mode_admin", "1"); else prop.put("mode_admin", "0"); @@ -91,7 +91,7 @@ public class BlogComments { } if(!hasRights){ - userDB.Entry userentry = switchboard.userDB.proxyAuth((String)header.get("Authorization", "xxxxxx")); + userDB.Entry userentry = sb.userDB.proxyAuth((String)header.get("Authorization", "xxxxxx")); if(userentry != null && userentry.hasRight(userDB.Entry.BLOG_RIGHT)){ hasRights=true; } @@ -107,14 +107,14 @@ public class BlogComments { String StrAuthor = post.get("author", "anonymous"); if (StrAuthor.equals("anonymous")) { - StrAuthor = switchboard.blogDB.guessAuthor(ip); + StrAuthor = sb.blogDB.guessAuthor(ip); if (StrAuthor == null || StrAuthor.length() == 0) { - if (de.anomic.yacy.yacyCore.seedDB.mySeed() == null) { + if (sb.wordIndex.seedDB.mySeed() == null) { StrAuthor = "anonymous"; } else { - StrAuthor = de.anomic.yacy.yacyCore.seedDB.mySeed().get("Name", "anonymous"); + StrAuthor = sb.wordIndex.seedDB.mySeed().get("Name", "anonymous"); } } } @@ -126,7 +126,7 @@ public class BlogComments { author = StrAuthor.getBytes(); } - page = switchboard.blogDB.readBlogEntry(pagename); //maybe "if(page == null)" + page = sb.blogDB.readBlogEntry(pagename); //maybe "if(page == null)" // comments not allowed if (page.getCommentMode() == 0) { @@ -158,34 +158,34 @@ public class BlogComments { subject = StrSubject.getBytes(); } String commentID = String.valueOf(System.currentTimeMillis()); - BlogEntry blogEntry = switchboard.blogDB.readBlogEntry(pagename); + BlogEntry blogEntry = sb.blogDB.readBlogEntry(pagename); blogEntry.addComment(commentID); - switchboard.blogDB.writeBlogEntry(blogEntry); - switchboard.blogCommentDB.write(switchboard.blogCommentDB.newEntry(commentID, subject, author, ip, date, content)); + sb.blogDB.writeBlogEntry(blogEntry); + sb.blogCommentDB.write(sb.blogCommentDB.newEntry(commentID, subject, author, ip, date, content)); prop.put("LOCATION","BlogComments.html?page=" + pagename); messageBoard.entry msgEntry = null; try { - switchboard.messageDB.write(msgEntry = switchboard.messageDB.newEntry( + sb.messageDB.write(msgEntry = sb.messageDB.newEntry( "blogComment", StrAuthor, - yacyCore.seedDB.mySeed().hash, - yacyCore.seedDB.mySeed().getName(), yacyCore.seedDB.mySeed().hash, + sb.wordIndex.seedDB.mySeed().hash, + sb.wordIndex.seedDB.mySeed().getName(), sb.wordIndex.seedDB.mySeed().hash, "new blog comment: " + new String(blogEntry.getSubject(),"UTF-8"), content)); } catch (UnsupportedEncodingException e1) { - switchboard.messageDB.write(msgEntry = switchboard.messageDB.newEntry( + sb.messageDB.write(msgEntry = sb.messageDB.newEntry( "blogComment", StrAuthor, - yacyCore.seedDB.mySeed().hash, - yacyCore.seedDB.mySeed().getName(), yacyCore.seedDB.mySeed().hash, + sb.wordIndex.seedDB.mySeed().hash, + sb.wordIndex.seedDB.mySeed().getName(), sb.wordIndex.seedDB.mySeed().hash, "new blog comment: " + new String(blogEntry.getSubject()), content)); } - messageForwardingViaEmail(env, msgEntry); + messageForwardingViaEmail(sb, msgEntry); // finally write notification - File notifierSource = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath","htroot") + "/env/grafics/message.gif"); - File notifierDest = new File(switchboard.getConfigPath("htDocsPath", "DATA/HTDOCS"), "notifier.gif"); + File notifierSource = new File(sb.getRootPath(), sb.getConfig("htRootPath","htroot") + "/env/grafics/message.gif"); + File notifierDest = new File(sb.getConfigPath("htDocsPath", "DATA/HTDOCS"), "notifier.gif"); try { serverFileUtils.copy(notifierSource, notifierDest); } catch (IOException e) { @@ -197,14 +197,14 @@ public class BlogComments { if(hasRights && post.containsKey("delete") && post.containsKey("page") && post.containsKey("comment")) { if(page.removeComment((String) post.get("comment"))) { - switchboard.blogCommentDB.delete((String) post.get("comment")); + sb.blogCommentDB.delete((String) post.get("comment")); } } if(hasRights && post.containsKey("allow") && post.containsKey("page") && post.containsKey("comment")) { - blogBoardComments.CommentEntry entry = switchboard.blogCommentDB.read((String) post.get("comment")); + blogBoardComments.CommentEntry entry = sb.blogCommentDB.read((String) post.get("comment")); entry.allow(); - switchboard.blogCommentDB.write(entry); + sb.blogCommentDB.write(entry); } if(post.containsKey("preview") && page.getCommentMode() != 0) { @@ -279,7 +279,7 @@ public class BlogComments { continue; } - entry = switchboard.blogCommentDB.read(pageid); + entry = sb.blogCommentDB.read(pageid); if (commentMode == 2 && !hasRights && !entry.isAllowed()) continue; @@ -338,15 +338,15 @@ public class BlogComments { return prop; } - private static void messageForwardingViaEmail(serverSwitch<?> env, messageBoard.entry msgEntry) { + private static void messageForwardingViaEmail(plasmaSwitchboard sb, messageBoard.entry msgEntry) { try { - if (!Boolean.valueOf(env.getConfig("msgForwardingEnabled","false")).booleanValue()) return; + if (!Boolean.valueOf(sb.getConfig("msgForwardingEnabled","false")).booleanValue()) return; // getting the recipient address - String sendMailTo = env.getConfig("msgForwardingTo","root@localhost").trim(); + String sendMailTo = sb.getConfig("msgForwardingTo","root@localhost").trim(); // getting the sendmail configuration - String sendMailStr = env.getConfig("msgForwardingCmd","/usr/bin/sendmail")+" "+sendMailTo; + String sendMailStr = sb.getConfig("msgForwardingCmd","/usr/bin/sendmail")+" "+sendMailTo; String[] sendMail = sendMailStr.trim().split(" "); // building the message text @@ -355,7 +355,7 @@ public class BlogComments { .append(sendMailTo) .append("\nFrom: ") .append("yacy@") - .append(yacyCore.seedDB.mySeed().getName()) + .append(sb.wordIndex.seedDB.mySeed().getName()) .append("\nSubject: [YaCy] ") .append(msgEntry.subject().replace('\n', ' ')) .append("\nDate: ") diff --git a/htroot/Bookmarks.java b/htroot/Bookmarks.java index 91e77be04..4fd5cd838 100644 --- a/htroot/Bookmarks.java +++ b/htroot/Bookmarks.java @@ -69,7 +69,6 @@ import de.anomic.server.serverDate; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.logging.serverLog; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacyURL; @@ -78,7 +77,7 @@ import de.anomic.yacy.yacyURL; public class Bookmarks { private static final serverObjects prop = new serverObjects(); - private static plasmaSwitchboard switchboard = null; + private static plasmaSwitchboard sb = null; private static userDB.Entry user = null; private static boolean isAdmin = false; @@ -96,9 +95,9 @@ public class Bookmarks { String username=""; prop.clear(); - switchboard = (plasmaSwitchboard) env; - user = switchboard.userDB.getUser(header); - isAdmin=(switchboard.verifyAuthentication(header, true) || user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT)); + sb = (plasmaSwitchboard) env; + user = sb.userDB.getUser(header); + isAdmin=(sb.verifyAuthentication(header, true) || user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT)); // set user name if(user != null) username=user.getUserName(); @@ -112,7 +111,7 @@ public class Bookmarks { */ // set peer address - final String address = yacyCore.seedDB.mySeed().getPublicAddress(); + final String address = sb.wordIndex.seedDB.mySeed().getPublicAddress(); prop.put("address", address); //defaultvalues @@ -157,7 +156,7 @@ public class Bookmarks { tagsString="unsorted"; //default tag } Set<String> tags=listManager.string2set(bookmarksDB.cleanTagsString(tagsString)); - bookmarksDB.Bookmark bookmark = switchboard.bookmarksDB.createBookmark(url, username); + bookmarksDB.Bookmark bookmark = sb.bookmarksDB.createBookmark(url, username); if(bookmark != null){ bookmark.setProperty(bookmarksDB.Bookmark.BOOKMARK_TITLE, title); bookmark.setProperty(bookmarksDB.Bookmark.BOOKMARK_DESCRIPTION, description); @@ -176,7 +175,7 @@ public class Bookmarks { bookmark.setFeed(false); } bookmark.setTags(tags, true); - switchboard.bookmarksDB.saveBookmark(bookmark); + sb.bookmarksDB.saveBookmark(bookmark); //}else{ //ERROR } @@ -193,10 +192,10 @@ public class Bookmarks { prop.put("mode_public", "0"); prop.put("mode_feed", "0"); } else { - bookmarksDB.Bookmark bookmark = switchboard.bookmarksDB.getBookmark(urlHash); + bookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(urlHash); if (bookmark == null) { // try to get the bookmark from the LURL database - indexURLReference urlentry = switchboard.wordIndex.getURL(urlHash, null, 0); + indexURLReference urlentry = sb.wordIndex.getURL(urlHash, null, 0); plasmaParserDocument document = null; if (urlentry != null) { indexURLReference.Components comp = urlentry.comp(); @@ -244,7 +243,7 @@ public class Bookmarks { serverLog.logInfo("BOOKMARKS", "I try to import bookmarks from HTML-file"); try { File file=new File((String)post.get("htmlfile")); - switchboard.bookmarksDB.importFromBookmarks(new yacyURL(file) , post.get("htmlfile$file"), tags, isPublic); + sb.bookmarksDB.importFromBookmarks(new yacyURL(file) , post.get("htmlfile$file"), tags, isPublic); } catch (MalformedURLException e) {} serverLog.logInfo("BOOKMARKS", "success!!"); }else if(post.containsKey("xmlfile")){ @@ -252,10 +251,10 @@ public class Bookmarks { if(((String) post.get("public")).equals("public")){ isPublic=true; } - switchboard.bookmarksDB.importFromXML(post.get("xmlfile$file"), isPublic); + sb.bookmarksDB.importFromXML(post.get("xmlfile$file"), isPublic); }else if(post.containsKey("delete")){ String urlHash=(String) post.get("delete"); - switchboard.bookmarksDB.removeBookmark(urlHash); + sb.bookmarksDB.removeBookmark(urlHash); } if(post.containsKey("tag")){ tagName=(String) post.get("tag"); @@ -285,15 +284,15 @@ public class Bookmarks { Iterator<String> tagsIt; int tagCount; - prop.put("num-bookmarks", switchboard.bookmarksDB.bookmarksSize()); + prop.put("num-bookmarks", sb.bookmarksDB.bookmarksSize()); count=0; if(!tagName.equals("")){ prop.put("selected", ""); - it=switchboard.bookmarksDB.getBookmarksIterator(tagName, isAdmin); + it=sb.bookmarksDB.getBookmarksIterator(tagName, isAdmin); }else{ prop.put("selected", " selected=\"selected\""); - it=switchboard.bookmarksDB.getBookmarksIterator(isAdmin); + it=sb.bookmarksDB.getBookmarksIterator(isAdmin); } //skip the first entries (display next page) @@ -305,7 +304,7 @@ public class Bookmarks { count=0; while(count<max_count && it.hasNext()){ - bookmark=switchboard.bookmarksDB.getBookmark((String)it.next()); + bookmark=sb.bookmarksDB.getBookmark((String)it.next()); if(bookmark!=null){ if(bookmark.getFeed() && isAdmin) prop.put("bookmarks_"+count+"_link", "/FeedReader_p.html?url="+bookmark.getUrl()); @@ -359,7 +358,7 @@ public class Bookmarks { //----------------------- count = 0; - count = recurseFolders(switchboard.bookmarksDB.getFolderList(isAdmin),"/",0,true,""); + count = recurseFolders(sb.bookmarksDB.getFolderList(isAdmin),"/",0,true,""); prop.put("folderlist", count); @@ -372,9 +371,9 @@ public class Bookmarks { Iterator<Tag> it = null; if (tagName.equals("")) { - it = switchboard.bookmarksDB.getTagIterator(isAdmin, comp, max); + it = sb.bookmarksDB.getTagIterator(isAdmin, comp, max); } else { - it = switchboard.bookmarksDB.getTagIterator(tagName, isAdmin, comp, max); + it = sb.bookmarksDB.getTagIterator(tagName, isAdmin, comp, max); } while(it.hasNext()){ tag=(Tag) it.next(); @@ -418,9 +417,9 @@ public class Bookmarks { if(fn.startsWith(root)){ prop.put("folderlist_"+count+"_folder", "<li>"+fn.replaceFirst(root+"/*","")+"<ul class=\"folder\">"); count++; - Iterator<String> bit=switchboard.bookmarksDB.getBookmarksIterator(fn, isAdmin); + Iterator<String> bit=sb.bookmarksDB.getBookmarksIterator(fn, isAdmin); while(bit.hasNext()){ - bookmark=switchboard.bookmarksDB.getBookmark((String)bit.next()); + bookmark=sb.bookmarksDB.getBookmark((String)bit.next()); prop.put("folderlist_"+count+"_folder", "<li><a href=\""+bookmark.getUrl()+"\" title=\""+bookmark.getDescription()+"\">"+ bookmark.getTitle()+"</a></li>"); count++; } @@ -444,7 +443,7 @@ public class Bookmarks { map.put("title", title.replace(',', ' ')); map.put("description", description.replace(',', ' ')); map.put("tags", tagsString.replace(',', ' ')); - yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); + sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); } } diff --git a/htroot/ConfigBasic.java b/htroot/ConfigBasic.java index 9f8a5de3c..732c3d901 100644 --- a/htroot/ConfigBasic.java +++ b/htroot/ConfigBasic.java @@ -62,7 +62,6 @@ import de.anomic.server.serverDomains; import de.anomic.server.serverInstantBusyThread; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySeed; public class ConfigBasic { @@ -76,7 +75,6 @@ public class ConfigBasic { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) { // return variable that accumulates replacements - ConfigBasic config = new ConfigBasic(); plasmaSwitchboard sb = (plasmaSwitchboard) env; serverObjects prop = new serverObjects(); String langPath = env.getConfigPath("locale.work", "DATA/LOCALE/locales").getAbsolutePath(); @@ -89,45 +87,14 @@ public class ConfigBasic { return prop; } - // reconfigure port forwarding - if ((post != null)) config.reinitPortForwarding(post, env); - // starting a peer ping //boolean doPeerPing = false; - if ((yacyCore.seedDB.mySeed().isVirgin()) || (yacyCore.seedDB.mySeed().isJunior())) { + if ((sb.wordIndex.seedDB.mySeed().isVirgin()) || (sb.wordIndex.seedDB.mySeed().isJunior())) { serverInstantBusyThread.oneTimeJob(sb.yc, "peerPing", null, 0); //doPeerPing = true; } - // scan for Upnp routers - /* - long begin = System.currentTimeMillis(); - boolean upnpRouterFound = false; - if (yacyCore.seedDB.mySeed.isVirgin() || yacyCore.seedDB.mySeed.isJunior()) { - upnpRouterFound = config.findUPnPRouter(3000); - } - long end = System.currentTimeMillis(); - - // if the upnp router scan has taken less than 3 sec, we need to wait - // a little bit for success of peer ping - if ((doPeerPing) && ((end - begin) < 3000 )) { - try {Thread.sleep(3000-(end - begin));} catch (InterruptedException e) {/* *///} - /*} - - // if a UPnP router is available - String currentForwarder = env.getConfig("portForwarding.Type", "none"); - boolean forwardingEnabled = env.getConfigBool("portForwarding.Enabled",false); - boolean otherForwarderEnabled = serverCore.portForwardingEnabled && serverCore.portForwarding != null && !currentForwarder.equalsIgnoreCase("upnp"); - if (otherForwarderEnabled) { - prop.put("upnp",0); - } else { - prop.put("upnp", upnpRouterFound ? 1 : 0); - } - - // if UPnp is already enabled - prop.put("upnp_enabled", currentForwarder.equalsIgnoreCase("upnp") && forwardingEnabled ? 1 : 0); - */ // language settings if ((post != null) && (!(post.get("language", "default").equals(lang)))) { translator.changeLang(env, langPath, post.get("language", "default") + ".lng"); @@ -158,7 +125,7 @@ public class ConfigBasic { } // check if peer name already exists - yacySeed oldSeed = yacyCore.seedDB.lookupByName(peerName); + yacySeed oldSeed = sb.wordIndex.seedDB.lookupByName(peerName); if ((oldSeed == null) && (!(env.getConfig("peerName", "").equals(peerName)))) { // the name is new boolean nameOK = Pattern.compile("[A-Za-z0-9\\-_]{3,80}").matcher(peerName).matches(); @@ -201,7 +168,7 @@ public class ConfigBasic { // check if values are proper boolean properPW = (env.getConfig("adminAccount", "").length() == 0) && (env.getConfig(httpd.ADMIN_ACCOUNT_B64MD5, "").length() > 0); boolean properName = (env.getConfig("peerName","").length() >= 3) && (!(yacySeed.isDefaultPeerName(env.getConfig("peerName","")))); - boolean properPort = (yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal()); + boolean properPort = (sb.wordIndex.seedDB.mySeed().isSenior()) || (sb.wordIndex.seedDB.mySeed().isPrincipal()); if ((properPW) && (env.getConfig("defaultFiles", "").startsWith("ConfigBasic.html,"))) { env.setConfig("defaultFiles", env.getConfig("defaultFiles", "").substring(17)); @@ -240,80 +207,4 @@ public class ConfigBasic { } return prop; } - /* - private boolean findUPnPRouter(int timeout) { - - // determine if the upnp port forwarding class is available and load it dynamically - Object[] UpnpForwarder = this.getUpnpForwarderClasses(); - serverPortForwarding upnp = (serverPortForwarding) UpnpForwarder[0]; - Method scanForRouter = (Method) UpnpForwarder[1]; - if ((upnp == null) || (scanForRouter == null)) return false; - - // trying to find a upnp router - try { - Object result = scanForRouter.invoke(upnp, new Object[]{new Integer(timeout)}); - if ((result != null)&&(result instanceof Boolean)) { - return ((Boolean)result).booleanValue(); - } - } catch (Exception e) { // ignore this error - } catch (Error e) {} // ignore this error - return false; - } - - private Object[] getUpnpForwarderClasses() { - serverPortForwarding upnp = null; - Method scanForRouter = null; - - try { - - // trying to load the upnp forwarder class - Class forwarderClass = Class.forName("de.anomic.server.portForwarding.upnp.serverPortForwardingUpnp"); - // create a new instance - upnp = (serverPortForwarding) forwarderClass.newInstance(); - // trying to get the proper method for router scanning - scanForRouter = upnp.getClass().getMethod("routerAvailable", new Class[] {int.class}); - - } catch (Exception e) { // ignore this error - } catch (Error e) {} // ignore this error - - return new Object[]{upnp,scanForRouter}; - } - */ - private void reinitPortForwarding(serverObjects post, serverSwitch<?> env) { - if ((post != null)) { - try { - boolean reinitPortForwarding = false; - - if (post.containsKey("enableUpnp")) { - // upnp should be enabled - env.setConfig("portForwarding.Enabled","true"); - env.setConfig("portForwarding.Type", "upnp"); - reinitPortForwarding = true; - } else { - String currentForwarder = env.getConfig("portForwarding.Type", "none"); - boolean otherForwarderEnabled = serverCore.portForwardingEnabled && serverCore.portForwarding != null && !currentForwarder.equalsIgnoreCase("upnp"); - - // if no other forwarder is running we deactivate forwarding - // and try to stop an eventually running upnp forwarder - if (!otherForwarderEnabled) { - env.setConfig("portForwarding.Enabled","false"); - env.setConfig("portForwarding.Type", "none"); - reinitPortForwarding = true; - } - } - - if (reinitPortForwarding) { - if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) { - // trying to shutdown the current port forwarding channel - serverCore.portForwarding.disconnect(); - } - - // trying to reinitialize the port forwarding - serverCore httpd = (serverCore) env.getThread("10_httpd"); - httpd.initPortForwarding(); - } - - } catch (Exception e) { /* */ } - } - } } diff --git a/htroot/ConfigNetwork_p.java b/htroot/ConfigNetwork_p.java index 911a3d253..ed9abb674 100644 --- a/htroot/ConfigNetwork_p.java +++ b/htroot/ConfigNetwork_p.java @@ -31,7 +31,6 @@ import de.anomic.server.serverBusyThread; import de.anomic.server.serverCodings; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import de.anomic.yacy.yacyCore; public class ConfigNetwork_p { @@ -103,10 +102,10 @@ public class ConfigNetwork_p { if (indexReceive) { sb.setConfig("allowReceiveIndex", "true"); - yacyCore.seedDB.mySeed().setFlagAcceptRemoteIndex(true); + sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(true); } else { sb.setConfig("allowReceiveIndex", "false"); - yacyCore.seedDB.mySeed().setFlagAcceptRemoteIndex(false); + sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(false); } if (post.get("indexReceiveBlockBlacklist", "").equals("on")) { @@ -116,7 +115,7 @@ public class ConfigNetwork_p { } if (post.containsKey("peertags")) { - yacyCore.seedDB.mySeed().setPeerTags(serverCodings.string2set(normalizedList((String) post.get("peertags")), ",")); + sb.wordIndex.seedDB.mySeed().setPeerTags(serverCodings.string2set(normalizedList((String) post.get("peertags")), ",")); } sb.setConfig("cluster.mode", post.get("cluster.mode", "publicpeer")); @@ -136,7 +135,7 @@ public class ConfigNetwork_p { sb.setConfig("cluster.peers.yacydomain", checkYaCyDomainList(post.get("cluster.peers.yacydomain", ""))); // update the cluster hash set - sb.clusterhashes = yacyCore.seedDB.clusterHashes(sb.getConfig("cluster.peers.yacydomain", "")); + sb.clusterhashes = sb.wordIndex.seedDB.clusterHashes(sb.getConfig("cluster.peers.yacydomain", "")); } @@ -162,11 +161,11 @@ public class ConfigNetwork_p { prop.put("indexReceiveChecked", (indexReceive) ? "1" : "0"); prop.put("indexReceiveBlockBlacklistChecked.on", (sb.getConfig("indexReceiveBlockBlacklist", "true").equals("true")) ? "1" : "0"); prop.put("indexReceiveBlockBlacklistChecked.off", (sb.getConfig("indexReceiveBlockBlacklist", "true").equals("true")) ? "0" : "1"); - prop.putHTML("peertags", serverCodings.set2string(yacyCore.seedDB.mySeed().getPeerTags(), ",", false)); + prop.putHTML("peertags", serverCodings.set2string(sb.wordIndex.seedDB.mySeed().getPeerTags(), ",", false)); // set seed information directly - yacyCore.seedDB.mySeed().setFlagAcceptRemoteCrawl(sb.getConfigBool("crawlResponse", false)); - yacyCore.seedDB.mySeed().setFlagAcceptRemoteIndex(indexReceive); + sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteCrawl(sb.getConfigBool("crawlResponse", false)); + sb.wordIndex.seedDB.mySeed().setFlagAcceptRemoteIndex(indexReceive); // set p2p/robinson mode flags and values prop.put("p2p.checked", (indexDistribute || indexReceive) ? "1" : "0"); diff --git a/htroot/ConfigProfile_p.java b/htroot/ConfigProfile_p.java index 57632ffa7..74a49b4d3 100644 --- a/htroot/ConfigProfile_p.java +++ b/htroot/ConfigProfile_p.java @@ -55,16 +55,16 @@ import java.io.IOException; import java.util.Properties;
import de.anomic.http.httpHeader;
+import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
public class ConfigProfile_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
- //listManager.switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
final serverObjects prop = new serverObjects();
final Properties profile = new Properties();
FileInputStream fileIn = null;
@@ -113,7 +113,7 @@ public class ConfigProfile_p { // generate a news message
Properties news = profile;
news.remove("comment");
- yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_PROFILE_UPDATE, news));
+ sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_PROFILE_UPDATE, news));
//yacyCore.newsPool.publishMyNews(new yacyNewsRecord(yacyNewsRecord.CATEGORY_PROFILE_UPDATE, profile));
} catch(IOException e) {
} finally {
diff --git a/htroot/ConfigRobotsTxt_p.java b/htroot/ConfigRobotsTxt_p.java index 0c527982b..077e05142 100644 --- a/htroot/ConfigRobotsTxt_p.java +++ b/htroot/ConfigRobotsTxt_p.java @@ -54,15 +54,15 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.servletProperties; -import de.anomic.yacy.yacyCore; public class ConfigRobotsTxt_p { public static servletProperties respond(httpHeader header, serverObjects post, serverSwitch<?> env) { + plasmaSwitchboard sb = (plasmaSwitchboard) env; final servletProperties prop = new servletProperties(); httpdRobotsTxtConfig rbc = ((plasmaSwitchboard)env).robotstxtConfig; - prop.put("clientname", yacyCore.seedDB.mySeed().getPublicAddress()); + prop.put("clientname", sb.wordIndex.seedDB.mySeed().getPublicAddress()); if (post != null) { if (post.containsKey("save")) { diff --git a/htroot/Connections_p.java b/htroot/Connections_p.java index 4eb4175a1..bbbbd6a6e 100644 --- a/htroot/Connections_p.java +++ b/htroot/Connections_p.java @@ -65,22 +65,21 @@ import de.anomic.server.serverSwitch; import de.anomic.server.serverThread;
import de.anomic.server.serverCore.Session;
import de.anomic.urlRedirector.urlRedirectord;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public final class Connections_p {
- public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> sb) {
+ public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) sb;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
// get the virtualHost string
- String virtualHost = switchboard.getConfig("fileHost","localhost");
+ String virtualHost = sb.getConfig("fileHost","localhost");
// get the serverCore thread
- serverThread httpd = switchboard.getThread("10_httpd");
+ serverThread httpd = sb.getThread("10_httpd");
/* waiting for all threads to finish */
int threadCount = serverCore.sessionThreadGroup.activeCount();
@@ -177,14 +176,14 @@ public final class Connections_p { commandLine = urlRedir.getURL();
}
- if ((dest != null) && (dest.equals(virtualHost))) dest = yacyCore.seedDB.mySeed().getName() + ".yacy";
+ if ((dest != null) && (dest.equals(virtualHost))) dest = sb.wordIndex.seedDB.mySeed().getName() + ".yacy";
// determining if the source is a yacy host
yacySeed seed = null;
if (doNameLookup) {
- seed = yacyCore.seedDB.lookupByIP(userAddress,true,false,false);
+ seed = sb.wordIndex.seedDB.lookupByIP(userAddress,true,false,false);
if (seed != null) {
- if ((seed.hash.equals(yacyCore.seedDB.mySeed().hash)) &&
+ if ((seed.hash.equals(sb.wordIndex.seedDB.mySeed().hash)) &&
(!seed.get(yacySeed.PORT,"").equals(Integer.toString(userPort)))) {
seed = null;
}
diff --git a/htroot/CrawlResults.java b/htroot/CrawlResults.java index e96182646..c19cffb69 100644 --- a/htroot/CrawlResults.java +++ b/htroot/CrawlResults.java @@ -36,7 +36,6 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.nxTools;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@@ -170,8 +169,8 @@ public class CrawlResults { urle = sb.wordIndex.getURL(urlHash, null, 0);
indexURLReference.Components comp = urle.comp();
// serverLog.logFinest("PLASMA", "plasmaCrawlLURL/genTableProps urle=" + urle.toString());
- initiatorSeed = yacyCore.seedDB.getConnected(initiatorHash);
- executorSeed = yacyCore.seedDB.getConnected(executorHash);
+ initiatorSeed = sb.wordIndex.seedDB.getConnected(initiatorHash);
+ executorSeed = sb.wordIndex.seedDB.getConnected(executorHash);
urlstr = comp.url().toNormalform(false, true);
urltxt = nxTools.shortenURLString(urlstr, 72); // shorten the string text like a URL
diff --git a/htroot/CrawlStartSimple_p.java b/htroot/CrawlStartSimple_p.java index ac4b70569..f953146ee 100644 --- a/htroot/CrawlStartSimple_p.java +++ b/htroot/CrawlStartSimple_p.java @@ -34,12 +34,12 @@ import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; -import de.anomic.yacy.yacyURL; public class CrawlStartSimple_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) { // return variable that accumulates replacements + plasmaSwitchboard sb = (plasmaSwitchboard) env; serverObjects prop = new serverObjects(); // define visible variables @@ -100,7 +100,7 @@ public class CrawlStartSimple_p { boolean dark = true; // create other peer crawl table using YaCyNews - Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true); + Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true); int showedCrawl = 0; yacyNewsRecord record; yacySeed peer; @@ -109,7 +109,7 @@ public class CrawlStartSimple_p { record = recordIterator.next(); if (record == null) continue; if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) { - peer = yacyCore.seedDB.get(record.originator()); + peer = sb.wordIndex.seedDB.get(record.originator()); if (peer == null) peername = record.originator(); else peername = peer.getName(); prop.put("otherCrawlStartInProgress_" + showedCrawl + "_dark", dark ? "1" : "0"); prop.put("otherCrawlStartInProgress_" + showedCrawl + "_cre", record.created().toString()); @@ -125,13 +125,13 @@ public class CrawlStartSimple_p { prop.put("otherCrawlStartInProgress", showedCrawl); // finished remote crawls - recordIterator = yacyCore.newsPool.recordIterator(yacyNewsPool.PROCESSED_DB, true); + recordIterator = sb.wordIndex.newsPool.recordIterator(yacyNewsPool.PROCESSED_DB, true); showedCrawl = 0; while (recordIterator.hasNext()) { record = (yacyNewsRecord) recordIterator.next(); if (record == null) continue; if (record.category().equals(yacyNewsPool.CATEGORY_CRAWL_START)) { - peer = yacyCore.seedDB.get(record.originator()); + peer = sb.wordIndex.seedDB.get(record.originator()); if (peer == null) peername = record.originator(); else peername = peer.getName(); prop.put("otherCrawlStartFinished_" + showedCrawl + "_dark", dark ? "1" : "0"); prop.put("otherCrawlStartFinished_" + showedCrawl + "_cre", record.created().toString()); @@ -148,11 +148,11 @@ public class CrawlStartSimple_p { // remote crawl peers - if ((yacyCore.seedDB == null) || (yacyCore.seedDB.mySeed().isVirgin()) || (yacyCore.seedDB.mySeed().isJunior())) { + if ((sb.wordIndex.seedDB == null) || (sb.wordIndex.seedDB.mySeed().isVirgin()) || (sb.wordIndex.seedDB.mySeed().isJunior())) { prop.put("remoteCrawlPeers", "0"); } else { - Iterator<yacySeed> crawlavail = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(yacyURL.dummyHash, true); - Iterator<yacySeed> crawlpendi = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(yacyURL.dummyHash, false); + Iterator<yacySeed> crawlavail = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(null, true); + Iterator<yacySeed> crawlpendi = yacyCore.dhtAgent.getAcceptRemoteCrawlSeeds(null, false); if ((!(crawlavail.hasNext())) && (!(crawlpendi.hasNext()))) { prop.put("remoteCrawlPeers", "0"); //no peers availible } else { diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index 67da8d440..4733cb2b8 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -50,7 +50,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyClient; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyURL; @@ -174,14 +173,14 @@ public class IndexControlRWIs_p { if (host.length() != 0) { if (host.length() == 12) { // the host string is a peer hash - seed = yacyCore.seedDB.getConnected(host); + seed = sb.wordIndex.seedDB.getConnected(host); } else { // the host string can be a host name - seed = yacyCore.seedDB.lookupByName(host); + seed = sb.wordIndex.seedDB.lookupByName(host); } } else { host = post.get("hostHash", ""); // if input field is empty, get from select box - seed = yacyCore.seedDB.getConnected(host); + seed = sb.wordIndex.seedDB.getConnected(host); } // prepare index @@ -210,6 +209,7 @@ public class IndexControlRWIs_p { String gzipBody = sb.getConfig("indexControl.gzipBody","false"); int timeout = (int) sb.getConfigLong("indexControl.timeout",60000); HashMap<String, Object> resultObj = yacyClient.transferIndex( + sb.wordIndex.seedDB, seed, new indexContainer[]{index}, knownURLs, diff --git a/htroot/IndexCreateIndexingQueue_p.java b/htroot/IndexCreateIndexingQueue_p.java index 5cf109af9..8f7f3c2f3 100644 --- a/htroot/IndexCreateIndexingQueue_p.java +++ b/htroot/IndexCreateIndexingQueue_p.java @@ -54,7 +54,6 @@ import de.anomic.plasma.plasmaSwitchboardQueue; import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@@ -62,7 +61,7 @@ public class IndexCreateIndexingQueue_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
prop.put("rejected", "0");
int showRejectedCount = 100;
@@ -76,27 +75,27 @@ public class IndexCreateIndexingQueue_p { }
if (post.containsKey("clearRejected")) {
- switchboard.crawlQueues.errorURL.clearStack();
+ sb.crawlQueues.errorURL.clearStack();
}
if (post.containsKey("moreRejected")) {
showRejectedCount = post.getInt("showRejected", 10);
}
if (post.containsKey("clearIndexingQueue")) {
try {
- synchronized (switchboard.sbQueue) {
+ synchronized (sb.sbQueue) {
plasmaSwitchboardQueue.QueueEntry entry = null;
- while ((entry = switchboard.sbQueue.pop()) != null) {
+ while ((entry = sb.sbQueue.pop()) != null) {
if ((entry != null) && (entry.profile() != null) && (!(entry.profile().storeHTCache()))) {
plasmaHTCache.deleteURLfromCache(entry.url());
}
}
- switchboard.sbQueue.clear(); // reset file to clean up content completely
+ sb.sbQueue.clear(); // reset file to clean up content completely
}
} catch (Exception e) {}
} else if (post.containsKey("deleteEntry")) {
String urlHash = (String) post.get("deleteEntry");
try {
- switchboard.sbQueue.remove(urlHash);
+ sb.sbQueue.remove(urlHash);
} catch (Exception e) {}
prop.put("LOCATION","");
return prop;
@@ -106,7 +105,7 @@ public class IndexCreateIndexingQueue_p { yacySeed initiator;
boolean dark;
- if ((switchboard.sbQueue.size() == 0) && (switchboard.sbQueue.getActiveQueueSize() == 0)) {
+ if ((sb.sbQueue.size() == 0) && (sb.sbQueue.getActiveQueueSize() == 0)) {
prop.put("indexing-queue", "0"); //is empty
} else {
prop.put("indexing-queue", "1"); // there are entries in the queue or in process
@@ -118,12 +117,12 @@ public class IndexCreateIndexingQueue_p { // getting all entries that are currently in process
ArrayList<plasmaSwitchboardQueue.QueueEntry> entryList = new ArrayList<plasmaSwitchboardQueue.QueueEntry>();
- entryList.addAll(switchboard.sbQueue.getActiveQueueEntries());
+ entryList.addAll(sb.sbQueue.getActiveQueueEntries());
int inProcessCount = entryList.size();
// getting all enqueued entries
- if ((switchboard.sbQueue.size() > 0)) {
- Iterator<plasmaSwitchboardQueue.QueueEntry> i = switchboard.sbQueue.entryIterator(false);
+ if ((sb.sbQueue.size() > 0)) {
+ Iterator<plasmaSwitchboardQueue.QueueEntry> i = sb.sbQueue.entryIterator(false);
while (i.hasNext()) entryList.add(i.next());
}
@@ -136,7 +135,7 @@ public class IndexCreateIndexingQueue_p { if ((pcentry != null)&&(pcentry.url() != null)) {
long entrySize = pcentry.size();
totalSize += entrySize;
- initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
+ initiator = sb.wordIndex.seedDB.getConnected(pcentry.initiator());
prop.put("indexing-queue_list_"+entryCount+"_dark", inProcess ? "2" : (dark ? "1" : "0"));
prop.putHTML("indexing-queue_list_"+entryCount+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("indexing-queue_list_"+entryCount+"_depth", pcentry.depth());
@@ -158,11 +157,11 @@ public class IndexCreateIndexingQueue_p { }
// failure cases
- if (switchboard.crawlQueues.errorURL.stackSize() != 0) {
- if (showRejectedCount > switchboard.crawlQueues.errorURL.stackSize()) showRejectedCount = switchboard.crawlQueues.errorURL.stackSize();
+ if (sb.crawlQueues.errorURL.stackSize() != 0) {
+ if (showRejectedCount > sb.crawlQueues.errorURL.stackSize()) showRejectedCount = sb.crawlQueues.errorURL.stackSize();
prop.put("rejected", "1");
- prop.putNum("rejected_num", switchboard.crawlQueues.errorURL.stackSize());
- if (showRejectedCount != switchboard.crawlQueues.errorURL.stackSize()) {
+ prop.putNum("rejected_num", sb.crawlQueues.errorURL.stackSize());
+ if (showRejectedCount != sb.crawlQueues.errorURL.stackSize()) {
prop.put("rejected_only-latest", "1");
prop.putNum("rejected_only-latest_num", showRejectedCount);
prop.put("rejected_only-latest_newnum", ((int) (showRejectedCount * 1.5)));
@@ -175,16 +174,16 @@ public class IndexCreateIndexingQueue_p { plasmaCrawlZURL.Entry entry;
yacySeed initiatorSeed, executorSeed;
int j=0;
- for (int i = switchboard.crawlQueues.errorURL.stackSize() - 1; i >= (switchboard.crawlQueues.errorURL.stackSize() - showRejectedCount); i--) {
- entry = switchboard.crawlQueues.errorURL.top(i);
+ for (int i = sb.crawlQueues.errorURL.stackSize() - 1; i >= (sb.crawlQueues.errorURL.stackSize() - showRejectedCount); i--) {
+ entry = sb.crawlQueues.errorURL.top(i);
if (entry == null) continue;
url = entry.url();
if (url == null) continue;
initiatorHash = entry.initiator();
executorHash = entry.executor();
- initiatorSeed = yacyCore.seedDB.getConnected(initiatorHash);
- executorSeed = yacyCore.seedDB.getConnected(executorHash);
+ initiatorSeed = sb.wordIndex.seedDB.getConnected(initiatorHash);
+ executorSeed = sb.wordIndex.seedDB.getConnected(executorHash);
prop.putHTML("rejected_list_"+j+"_initiator", ((initiatorSeed == null) ? "proxy" : initiatorSeed.getName()));
prop.putHTML("rejected_list_"+j+"_executor", ((executorSeed == null) ? "proxy" : executorSeed.getName()));
prop.putHTML("rejected_list_"+j+"_url", url.toNormalform(false, true));
diff --git a/htroot/IndexCreateLoaderQueue_p.java b/htroot/IndexCreateLoaderQueue_p.java index e7dda9a0e..203aa06f0 100644 --- a/htroot/IndexCreateLoaderQueue_p.java +++ b/htroot/IndexCreateLoaderQueue_p.java @@ -48,29 +48,28 @@ import de.anomic.plasma.plasmaCrawlEntry; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class IndexCreateLoaderQueue_p {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
- if (switchboard.crawlQueues.size() == 0) {
+ if (sb.crawlQueues.size() == 0) {
prop.put("loader-set", "0");
} else {
prop.put("loader-set", "1");
boolean dark = true;
- plasmaCrawlEntry[] w = switchboard.crawlQueues.activeWorkerEntries();
+ plasmaCrawlEntry[] w = sb.crawlQueues.activeWorkerEntries();
yacySeed initiator;
int count = 0;
for (int i = 0; i < w.length; i++) {
if (w[i] == null) continue;
- initiator = yacyCore.seedDB.getConnected(w[i].initiator());
+ initiator = sb.wordIndex.seedDB.getConnected(w[i].initiator());
prop.put("loader-set_list_"+count+"_dark", dark ? "1" : "0");
prop.putHTML("loader-set_list_"+count+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("loader-set_list_"+count+"_depth", w[i].depth());
diff --git a/htroot/IndexCreateWWWGlobalQueue_p.java b/htroot/IndexCreateWWWGlobalQueue_p.java index 3291bd9ea..f6fde9923 100644 --- a/htroot/IndexCreateWWWGlobalQueue_p.java +++ b/htroot/IndexCreateWWWGlobalQueue_p.java @@ -54,7 +54,6 @@ import de.anomic.plasma.plasmaCrawlProfile; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class IndexCreateWWWGlobalQueue_p {
@@ -67,7 +66,7 @@ public class IndexCreateWWWGlobalQueue_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
int showLimit = 100;
@@ -79,9 +78,9 @@ public class IndexCreateWWWGlobalQueue_p { }
if (post.containsKey("clearcrawlqueue")) {
- int c = switchboard.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
- switchboard.crawlQueues.noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_LIMIT);
- try { switchboard.cleanProfiles(); } catch (InterruptedException e) { /* Ignore this */}
+ int c = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
+ sb.crawlQueues.noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_LIMIT);
+ try { sb.cleanProfiles(); } catch (InterruptedException e) { /* Ignore this */}
/*
int c = 0;
while (switchboard.urlPool.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) > 0) {
@@ -93,18 +92,18 @@ public class IndexCreateWWWGlobalQueue_p { prop.putNum("info_numEntries", c);
} else if (post.containsKey("deleteEntry")) {
String urlHash = (String) post.get("deleteEntry");
- switchboard.crawlQueues.noticeURL.removeByURLHash(urlHash);
+ sb.crawlQueues.noticeURL.removeByURLHash(urlHash);
prop.put("LOCATION","");
return prop;
}
}
- int stackSize = switchboard.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
+ int stackSize = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
if (stackSize == 0) {
prop.put("crawler-queue", "0");
} else {
prop.put("crawler-queue", "1");
- plasmaCrawlEntry[] crawlerList = switchboard.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, showLimit);
+ plasmaCrawlEntry[] crawlerList = sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, showLimit);
plasmaCrawlEntry urle;
boolean dark = true;
@@ -115,9 +114,9 @@ public class IndexCreateWWWGlobalQueue_p { for (i = 0; (i < crawlerList.length) && (showNum < showLimit); i++) {
urle = crawlerList[i];
if ((urle != null)&&(urle.url()!=null)) {
- initiator = yacyCore.seedDB.getConnected(urle.initiator());
+ initiator = sb.wordIndex.seedDB.getConnected(urle.initiator());
profileHandle = urle.profileHandle();
- profileEntry = (profileHandle == null) ? null : switchboard.profilesActiveCrawls.getEntry(profileHandle);
+ profileEntry = (profileHandle == null) ? null : sb.profilesActiveCrawls.getEntry(profileHandle);
prop.put("crawler-queue_list_"+showNum+"_dark", dark ? "1" : "0");
prop.putHTML("crawler-queue_list_"+showNum+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()) );
prop.put("crawler-queue_list_"+showNum+"_profile", ((profileEntry == null) ? "unknown" : profileEntry.name()));
diff --git a/htroot/IndexCreateWWWLocalQueue_p.java b/htroot/IndexCreateWWWLocalQueue_p.java index fad668b6d..51cb63979 100644 --- a/htroot/IndexCreateWWWLocalQueue_p.java +++ b/htroot/IndexCreateWWWLocalQueue_p.java @@ -58,7 +58,6 @@ import de.anomic.plasma.plasmaCrawlProfile; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class IndexCreateWWWLocalQueue_p {
@@ -182,7 +181,7 @@ public class IndexCreateWWWLocalQueue_p { for (i = 0; (i < crawlerList.length) && (showNum < showLimit); i++) {
urle = crawlerList[i];
if ((urle != null)&&(urle.url()!=null)) {
- initiator = yacyCore.seedDB.getConnected(urle.initiator());
+ initiator = sb.wordIndex.seedDB.getConnected(urle.initiator());
profileHandle = urle.profileHandle();
profileEntry = (profileHandle == null) ? null : sb.profilesActiveCrawls.getEntry(profileHandle);
prop.put("crawler-queue_list_"+showNum+"_dark", dark ? "1" : "0");
diff --git a/htroot/IndexCreateWWWRemoteQueue_p.java b/htroot/IndexCreateWWWRemoteQueue_p.java index 84198c029..3a2a34184 100644 --- a/htroot/IndexCreateWWWRemoteQueue_p.java +++ b/htroot/IndexCreateWWWRemoteQueue_p.java @@ -55,7 +55,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.servletProperties; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySeed; public class IndexCreateWWWRemoteQueue_p { @@ -115,7 +114,7 @@ public class IndexCreateWWWRemoteQueue_p { for (i = 0; (i < crawlerList.length) && (showNum < showLimit); i++) { urle = crawlerList[i]; if (urle != null && urle.url() != null) { - initiator = yacyCore.seedDB.getConnected(urle.initiator()); + initiator = sb.wordIndex.seedDB.getConnected(urle.initiator()); profileHandle = urle.profileHandle(); profileEntry = (profileHandle == null) ? null : sb.profilesActiveCrawls.getEntry(profileHandle); prop.put("crawler-queue_list_" + showNum + "_dark", dark ? "1" : "0"); diff --git a/htroot/IndexTransfer_p.java b/htroot/IndexTransfer_p.java index f73756038..de017b353 100644 --- a/htroot/IndexTransfer_p.java +++ b/htroot/IndexTransfer_p.java @@ -65,12 +65,12 @@ public final class IndexTransfer_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if (post != null) {
if (post.containsKey("startIndexTransfer")) {
- yacySeed seed = yacyCore.seedDB.getConnected(post.get("hostHash", ""));
+ yacySeed seed = sb.wordIndex.seedDB.getConnected(post.get("hostHash", ""));
if (seed == null) {
prop.put("running_status","Disconnected peer");
} else {
@@ -78,30 +78,30 @@ public final class IndexTransfer_p { if(prop.containsKey("overwriteIP") && ! ((String)prop.get("overwriteIP")).equals("")){
seed.setIP((String) prop.get("overwriteIP"));
}
- switchboard.startTransferWholeIndex(seed,deleteIndex);
+ sb.startTransferWholeIndex(seed,deleteIndex);
prop.put("LOCATION","");
return prop;
}
} else if (post.containsKey("stopIndexTransfer")) {
- switchboard.stopTransferWholeIndex(true);
+ sb.stopTransferWholeIndex(true);
prop.put("LOCATION","");
return prop;
} else if (post.containsKey("newIndexTransfer")) {
- switchboard.abortTransferWholeIndex(true);
+ sb.abortTransferWholeIndex(true);
prop.put("LOCATION","");
return prop;
}
}
// insert constants
- prop.putNum("wcount", switchboard.wordIndex.size());
- prop.putNum("ucount", switchboard.wordIndex.countURL());
- prop.put("running",(switchboard.transferIdxThread==null) ? "0" : "1");
- if (switchboard.transferIdxThread != null) {
- String[] status = switchboard.transferIdxThread.getStatus();
- String[] range = switchboard.transferIdxThread.getRange();
- int[] chunk = switchboard.transferIdxThread.getIndexCount();
+ prop.putNum("wcount", sb.wordIndex.size());
+ prop.putNum("ucount", sb.wordIndex.countURL());
+ prop.put("running",(sb.transferIdxThread==null) ? "0" : "1");
+ if (sb.transferIdxThread != null) {
+ String[] status = sb.transferIdxThread.getStatus();
+ String[] range = sb.transferIdxThread.getRange();
+ int[] chunk = sb.transferIdxThread.getIndexCount();
prop.put("running_selection.status",status[0]);
prop.put("running_selection.twrange", range[0]);
@@ -112,15 +112,15 @@ public final class IndexTransfer_p { prop.put("running_transfer.twchunk", chunk[1]);
- prop.putNum("running_twEntityCount", switchboard.transferIdxThread.getTransferedContainerCount());
- prop.putNum("running_twEntryCount", switchboard.transferIdxThread.getTransferedEntryCount());
- prop.put("running_twPayloadSize", serverMemory.bytesToString(switchboard.transferIdxThread.getTransferedBytes()));
- prop.putNum("running_twEntityPercent", switchboard.transferIdxThread.getTransferedContainerPercent());
- prop.putNum("running_twEntrySpeed", switchboard.transferIdxThread.getTransferedEntrySpeed());
+ prop.putNum("running_twEntityCount", sb.transferIdxThread.getTransferedContainerCount());
+ prop.putNum("running_twEntryCount", sb.transferIdxThread.getTransferedEntryCount());
+ prop.put("running_twPayloadSize", serverMemory.bytesToString(sb.transferIdxThread.getTransferedBytes()));
+ prop.putNum("running_twEntityPercent", sb.transferIdxThread.getTransferedContainerPercent());
+ prop.putNum("running_twEntrySpeed", sb.transferIdxThread.getTransferedEntrySpeed());
- prop.put("running_deleteIndex", switchboard.transferIdxThread.deleteIndex() ? "1" : "0");
- prop.put("running_peerName",switchboard.transferIdxThread.getSeed().getName());
- prop.put("running_stopped",(switchboard.transferIdxThread.isFinished()) || (!switchboard.transferIdxThread.isAlive()) ? "1" : "0");
+ prop.put("running_deleteIndex", sb.transferIdxThread.deleteIndex() ? "1" : "0");
+ prop.put("running_peerName",sb.transferIdxThread.getSeed().getName());
+ prop.put("running_stopped",(sb.transferIdxThread.isFinished()) || (!sb.transferIdxThread.isAlive()) ? "1" : "0");
} else {
if (!prop.containsKey("running_status")) prop.put("running_status","Not running");
}
@@ -130,7 +130,7 @@ public final class IndexTransfer_p { //List known hosts
yacySeed seed;
int hc = 0;
- if ((yacyCore.seedDB != null) && (yacyCore.seedDB.sizeConnected() > 0)) {
+ if ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.sizeConnected() > 0)) {
Iterator<yacySeed> e = yacyCore.dhtAgent.getAcceptRemoteIndexSeeds("------------");
TreeMap<String, String> hostList = new TreeMap<String, String>();
while (e.hasNext()) {
diff --git a/htroot/MessageSend_p.java b/htroot/MessageSend_p.java index d9c276df1..f9299b313 100644 --- a/htroot/MessageSend_p.java +++ b/htroot/MessageSend_p.java @@ -50,6 +50,7 @@ import java.util.Date; import java.util.HashMap;
import de.anomic.http.httpHeader;
+import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyClient;
@@ -65,6 +66,7 @@ public class MessageSend_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if ((post == null) || (post.get("hash","").length() == 0)) {
@@ -85,14 +87,14 @@ public class MessageSend_p { // open an editor page for the message
// first ask if the other peer is online, and also what kind of document it accepts
- HashMap<String, String> result = yacyClient.permissionMessage(hash);
+ HashMap<String, String> result = yacyClient.permissionMessage(sb.wordIndex.seedDB, hash);
//System.out.println("DEBUG: permission request result = " + result.toString());
String peerName;
yacySeed targetPeer = null;
- if (hash.equals(yacyCore.seedDB.mySeed().hash)) {
- peerName = yacyCore.seedDB.mySeed().get(yacySeed.NAME,"nameless");
+ if (hash.equals(sb.wordIndex.seedDB.mySeed().hash)) {
+ peerName = sb.wordIndex.seedDB.mySeed().get(yacySeed.NAME,"nameless");
} else {
- targetPeer = yacyCore.seedDB.getConnected(hash);
+ targetPeer = sb.wordIndex.seedDB.getConnected(hash);
if (targetPeer == null)
peerName = "nameless";
else
@@ -149,7 +151,7 @@ public class MessageSend_p { } catch (UnsupportedEncodingException e) {
mb = message.getBytes();
}
- HashMap<String, String> result = yacyClient.postMessage(hash, subject, mb);
+ HashMap<String, String> result = yacyClient.postMessage(sb.wordIndex.seedDB, hash, subject, mb);
//message has been sent
prop.put("mode_status_response", (String) result.get("response"));
diff --git a/htroot/Messages_p.java b/htroot/Messages_p.java index fd00ad64a..236c2c177 100644 --- a/htroot/Messages_p.java +++ b/htroot/Messages_p.java @@ -58,7 +58,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class Messages_p {
@@ -71,22 +70,22 @@ public class Messages_p { }
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
// set peer address / name
- final String peerAddress = yacyCore.seedDB.mySeed().getPublicAddress();
- final String peerName = yacyCore.seedDB.mySeed().getName();
+ final String peerAddress = sb.wordIndex.seedDB.mySeed().getPublicAddress();
+ final String peerName = sb.wordIndex.seedDB.mySeed().getName();
prop.put("peerAddress", peerAddress);
prop.putHTML("peerName", peerName, true);
// List known hosts for message sending (from Blacklist_p.java)
- if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) {
+ if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0) {
prop.put("peersKnown", "1");
int peerCount = 0;
try {
TreeMap<String, String> hostList = new TreeMap<String, String>();
- final Iterator<yacySeed> e = yacyCore.seedDB.seedsConnected(true, false, null, (float) 0.0);
+ final Iterator<yacySeed> e = sb.wordIndex.seedDB.seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext()) {
yacySeed seed = e.next();
if (seed != null) hostList.put(seed.get(yacySeed.NAME, "nameless"),seed.hash);
@@ -113,8 +112,8 @@ public class Messages_p { messageBoard.entry message;
// first reset notification
- File notifierSource = new File(switchboard.getRootPath(), switchboard.getConfig("htRootPath", "htroot") + "/env/grafics/empty.gif");
- File notifierDest = new File(switchboard.getConfigPath("htDocsPath", "DATA/HTDOCS"), "notifier.gif");
+ File notifierSource = new File(sb.getRootPath(), sb.getConfig("htRootPath", "htroot") + "/env/grafics/empty.gif");
+ File notifierDest = new File(sb.getConfigPath("htDocsPath", "DATA/HTDOCS"), "notifier.gif");
try {
serverFileUtils.copy(notifierSource, notifierDest);
} catch (IOException e) {
@@ -122,21 +121,21 @@ public class Messages_p { if (action.equals("delete")) {
String key = post.get("object", "");
- switchboard.messageDB.remove(key);
+ sb.messageDB.remove(key);
action = "list";
}
if (action.equals("list")) {
prop.put("mode", "0"); //list
try {
- Iterator<String> i = switchboard.messageDB.keys(null, true);
+ Iterator<String> i = sb.messageDB.keys(null, true);
String key;
boolean dark = true;
int count=0;
while (i.hasNext()) {
key = i.next();
- message = switchboard.messageDB.read(key);
+ message = sb.messageDB.read(key);
prop.put("mode_messages_"+count+"_dark", ((dark) ? "1" : "0") );
prop.put("mode_messages_"+count+"_date", dateString(message.date()));
prop.putHTML("mode_messages_"+count+"_from", message.author(), true);
@@ -174,7 +173,7 @@ public class Messages_p { if (action.equals("view")) {
prop.put("mode", "1"); //view
String key = post.get("object", "");
- message = switchboard.messageDB.read(key);
+ message = sb.messageDB.read(key);
if (message == null) throw new NullPointerException("Message with ID " + key + " does not exist");
prop.putHTML("mode_from", message.author(), true);
diff --git a/htroot/Network.java b/htroot/Network.java index 7ef2fde04..2d14f5dc5 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -84,30 +84,30 @@ public class Network { prop.putHTML("page_networkName", sb.getConfig("network.unit.name", "unspecified"));
final boolean overview = (post == null) || (post.get("page", "0").equals("0"));
- final String mySeedType = yacyCore.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
+ final String mySeedType = sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
final boolean iAmActive = (mySeedType.equals(yacySeed.PEERTYPE_SENIOR) || mySeedType.equals(yacySeed.PEERTYPE_PRINCIPAL));
if (overview) {
- long accActLinks = yacyCore.seedDB.countActiveURL();
- long accActWords = yacyCore.seedDB.countActiveRWI();
- final long accPassLinks = yacyCore.seedDB.countPassiveURL();
- final long accPassWords = yacyCore.seedDB.countPassiveRWI();
- long accPotLinks = yacyCore.seedDB.countPotentialURL();
- long accPotWords = yacyCore.seedDB.countPotentialRWI();
+ long accActLinks = sb.wordIndex.seedDB.countActiveURL();
+ long accActWords = sb.wordIndex.seedDB.countActiveRWI();
+ final long accPassLinks = sb.wordIndex.seedDB.countPassiveURL();
+ final long accPassWords = sb.wordIndex.seedDB.countPassiveRWI();
+ long accPotLinks = sb.wordIndex.seedDB.countPotentialURL();
+ long accPotWords = sb.wordIndex.seedDB.countPotentialRWI();
- int conCount = yacyCore.seedDB.sizeConnected();
- final int disconCount = yacyCore.seedDB.sizeDisconnected();
- int potCount = yacyCore.seedDB.sizePotential();
+ int conCount = sb.wordIndex.seedDB.sizeConnected();
+ final int disconCount = sb.wordIndex.seedDB.sizeDisconnected();
+ int potCount = sb.wordIndex.seedDB.sizePotential();
// final boolean complete = ((post == null) ? false : post.get("links", "false").equals("true"));
- final long otherppm = yacyCore.seedDB.countActivePPM();
- final double otherqpm = yacyCore.seedDB.countActiveQPM();
+ final long otherppm = sb.wordIndex.seedDB.countActivePPM();
+ final double otherqpm = sb.wordIndex.seedDB.countActiveQPM();
long myppm = 0;
double myqph = 0d;
// create own peer info
- yacySeed seed = yacyCore.seedDB.mySeed();
- if (yacyCore.seedDB.mySeed() != null){ //our Peer
+ yacySeed seed = sb.wordIndex.seedDB.mySeed();
+ if (sb.wordIndex.seedDB.mySeed() != null){ //our Peer
// update seed info
yacyCore.peerActions.updateMySeed();
@@ -123,17 +123,17 @@ public class Network { // my-info
prop.putHTML("table_my-name", seed.get(yacySeed.NAME, "-") );
prop.put("table_my-hash", seed.hash );
- if (yacyCore.seedDB.mySeed().isVirgin()) {
+ if (sb.wordIndex.seedDB.mySeed().isVirgin()) {
prop.put("table_my-info", 0);
- } else if(yacyCore.seedDB.mySeed().isJunior()) {
+ } else if(sb.wordIndex.seedDB.mySeed().isJunior()) {
prop.put("table_my-info", 1);
accPotLinks += LCount;
accPotWords += ICount;
- } else if(yacyCore.seedDB.mySeed().isSenior()) {
+ } else if(sb.wordIndex.seedDB.mySeed().isSenior()) {
prop.put("table_my-info", 2);
accActLinks += LCount;
accActWords += ICount;
- } else if(yacyCore.seedDB.mySeed().isPrincipal()) {
+ } else if(sb.wordIndex.seedDB.mySeed().isPrincipal()) {
prop.put("table_my-info", 3);
accActLinks += LCount;
accActWords += ICount;
@@ -214,13 +214,13 @@ public class Network { yacySeed peer = new yacySeed(post.get("peerHash"),map);
yacyCore.peerActions.updateMySeed();
- final int added = yacyClient.publishMySeed(peer.getPublicAddress(), peer.hash);
+ final int added = yacyClient.publishMySeed(sb.wordIndex.seedDB.mySeed(), peer.getPublicAddress(), peer.hash);
if (added <= 0) {
prop.put("table_comment",1);
prop.putHTML("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getPublicAddress());
} else {
- peer = yacyCore.seedDB.getConnected(peer.hash);
+ peer = sb.wordIndex.seedDB.getConnected(peer.hash);
if (peer == null) {
prop.put("table_comment",1);
prop.put("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from UNKNOWN");
@@ -246,14 +246,14 @@ public class Network { final int page = Integer.parseInt(post.get("page", "1"));
final int maxCount = Integer.parseInt(post.get("maxCount", "300"));
int conCount = 0;
- if (yacyCore.seedDB == null) {
+ if (sb.wordIndex.seedDB == null) {
prop.put("table", 0);//no remote senior/principal proxies known"
} else {
int size = 0;
switch (page) {
- case 1 : size = yacyCore.seedDB.sizeConnected(); break;
- case 2 : size = yacyCore.seedDB.sizeDisconnected(); break;
- case 3 : size = yacyCore.seedDB.sizePotential(); break;
+ case 1 : size = sb.wordIndex.seedDB.sizeConnected(); break;
+ case 2 : size = sb.wordIndex.seedDB.sizeDisconnected(); break;
+ case 3 : size = sb.wordIndex.seedDB.sizePotential(); break;
default: break;
}
if (size == 0) {
@@ -262,7 +262,7 @@ public class Network { // add temporary the own seed to the database
if (iAmActive) {
yacyCore.peerActions.updateMySeed();
- yacyCore.seedDB.addConnected(yacyCore.seedDB.mySeed());
+ sb.wordIndex.seedDB.addConnected(sb.wordIndex.seedDB.mySeed());
}
// find updated Information using YaCyNews
@@ -271,7 +271,7 @@ public class Network { final HashMap<String, Map<String, String>> updatedBlog = new HashMap<String, Map<String, String>>();
final HashMap<String, String> isCrawling = new HashMap<String, String>();
yacyNewsRecord record;
- Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
+ Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(yacyNewsPool.INCOMING_DB, true);
while (recordIterator.hasNext()) {
record = recordIterator.next();
if (record == null) {
@@ -292,9 +292,9 @@ public class Network { final boolean complete = post.containsKey("ip");
Iterator<yacySeed> e = null;
switch (page) {
- case 1 : e = yacyCore.seedDB.seedsSortedConnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LCOUNT)); break;
- case 2 : e = yacyCore.seedDB.seedsSortedDisconnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
- case 3 : e = yacyCore.seedDB.seedsSortedPotential(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
+ case 1 : e = sb.wordIndex.seedDB.seedsSortedConnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LCOUNT)); break;
+ case 2 : e = sb.wordIndex.seedDB.seedsSortedDisconnected(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
+ case 3 : e = sb.wordIndex.seedDB.seedsSortedPotential(post.get("order", "down").equals("up"), post.get("sort", yacySeed.LASTSEEN)); break;
default: break;
}
String startURL;
@@ -336,7 +336,7 @@ public class Network { prop.put(STR_TABLE_LIST + conCount + "_updatedBlog", 0);
prop.put(STR_TABLE_LIST + conCount + "_isCrawling", 0);
if (conCount >= maxCount) { break; }
- if (seed.hash.equals(yacyCore.seedDB.mySeed().hash)) {
+ if (seed.hash.equals(sb.wordIndex.seedDB.mySeed().hash)) {
prop.put(STR_TABLE_LIST + conCount + "_dark", 2);
} else {
prop.put(STR_TABLE_LIST + conCount + "_dark", ((dark) ? 1 : 0) ); dark=!dark;
@@ -374,7 +374,7 @@ public class Network { prop.putHTML(STR_TABLE_LIST + conCount + "_shortname", shortname);
prop.putHTML(STR_TABLE_LIST + conCount + "_fullname", seed.get(yacySeed.NAME, "deadlink"));
userAgent = null;
- if (seed.hash.equals(yacyCore.seedDB.mySeed().hash)) {
+ if (seed.hash.equals(sb.wordIndex.seedDB.mySeed().hash)) {
final JakartaCommonsHttpClient httpClient = new JakartaCommonsHttpClient(10000, null, null);
userAgent = httpClient.getUserAgent();
location = HttpClient.generateLocation();
@@ -463,7 +463,7 @@ public class Network { conCount++;
} // seed != null
} // while
- if (iAmActive) { yacyCore.seedDB.removeMySeed(); }
+ if (iAmActive) { sb.wordIndex.seedDB.removeMySeed(); }
prop.putNum("table_list", conCount);
prop.put("table", 1);
prop.putNum("table_num", conCount);
diff --git a/htroot/NetworkPicture.java b/htroot/NetworkPicture.java index e88cdd3b4..712f2931a 100644 --- a/htroot/NetworkPicture.java +++ b/htroot/NetworkPicture.java @@ -45,6 +45,7 @@ import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaGrafics;
+import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.ymage.ymageMatrix;
@@ -53,7 +54,8 @@ import de.anomic.ymage.ymageMatrix; public class NetworkPicture {
public static ymageMatrix respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
-
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
+
int width = 768;
int height = 576;
int passiveLimit = 300;
@@ -80,7 +82,7 @@ public class NetworkPicture { if (passiveLimit > 1000000) passiveLimit = 1000000;
if (potentialLimit > 1000000) potentialLimit = 1000000;
if (maxCount > 1000) maxCount = 1000;
- return plasmaGrafics.getNetworkPicture(10000, width, height, passiveLimit, potentialLimit, maxCount, corona, env.getConfig("network.unit.name", "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
+ return plasmaGrafics.getNetworkPicture(sb.wordIndex.seedDB, 10000, width, height, passiveLimit, potentialLimit, maxCount, corona, env.getConfig("network.unit.name", "unspecified"), env.getConfig("network.unit.description", "unspecified"), bgcolor);
}
}
diff --git a/htroot/News.java b/htroot/News.java index 635e30aa4..e8ff979cf 100644 --- a/htroot/News.java +++ b/htroot/News.java @@ -53,7 +53,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacySeed;
@@ -61,7 +60,7 @@ import de.anomic.yacy.yacySeed; public class News {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
boolean overview = (post == null) || (post.get("page", "0").equals("0"));
int tableID = (overview) ? -1 : Integer.parseInt(post.get("page", "0")) - 1;
@@ -70,7 +69,7 @@ public class News { if (post != null) {
if ((post.containsKey("deletespecific")) && (tableID >= 0)) {
- if (switchboard.adminAuthenticated(header) < 2) {
+ if (sb.adminAuthenticated(header) < 2) {
prop.put("AUTHENTICATE", "admin log-in");
return prop; // this button needs authentication, force log-in
}
@@ -82,22 +81,22 @@ public class News { if ((check.startsWith("del_")) && (post.get(check, "off").equals("on"))) {
id = check.substring(4);
try {
- yacyCore.newsPool.moveOff(tableID, id);
+ sb.wordIndex.newsPool.moveOff(tableID, id);
} catch (IOException ee) {ee.printStackTrace();}
}
}
}
if ((post.containsKey("deleteall")) && (tableID >= 0)) {
- if (switchboard.adminAuthenticated(header) < 2) {
+ if (sb.adminAuthenticated(header) < 2) {
prop.put("AUTHENTICATE", "admin log-in");
return prop; // this button needs authentication, force log-in
}
try {
if ((tableID == yacyNewsPool.PROCESSED_DB) || (tableID == yacyNewsPool.PUBLISHED_DB)) {
- yacyCore.newsPool.clear(tableID);
+ sb.wordIndex.newsPool.clear(tableID);
} else {
- yacyCore.newsPool.moveOffAll(tableID);
+ sb.wordIndex.newsPool.moveOffAll(tableID);
}
} catch (IOException e) {
e.printStackTrace();
@@ -110,19 +109,19 @@ public class News { // show overview
prop.put("table", "0");
prop.put("page", "0");
- prop.putNum("table_insize", yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB));
- prop.putNum("table_prsize", yacyCore.newsPool.size(yacyNewsPool.PROCESSED_DB));
- prop.putNum("table_ousize", yacyCore.newsPool.size(yacyNewsPool.OUTGOING_DB));
- prop.putNum("table_pusize", yacyCore.newsPool.size(yacyNewsPool.PUBLISHED_DB));
+ prop.putNum("table_insize", sb.wordIndex.newsPool.size(yacyNewsPool.INCOMING_DB));
+ prop.putNum("table_prsize", sb.wordIndex.newsPool.size(yacyNewsPool.PROCESSED_DB));
+ prop.putNum("table_ousize", sb.wordIndex.newsPool.size(yacyNewsPool.OUTGOING_DB));
+ prop.putNum("table_pusize", sb.wordIndex.newsPool.size(yacyNewsPool.PUBLISHED_DB));
} else {
// generate table
prop.put("table", "1");
prop.put("page", tableID + 1);
prop.put("table_page", tableID + 1);
- if (yacyCore.seedDB != null) {
- int maxCount = Math.min(1000, yacyCore.newsPool.size(tableID));
- Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(tableID, false);
+ if (sb.wordIndex.seedDB != null) {
+ int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(tableID));
+ Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(tableID, false);
yacyNewsRecord record;
yacySeed seed;
int i = 0;
@@ -130,8 +129,8 @@ public class News { record = recordIterator.next();
if (record == null) continue;
- seed = yacyCore.seedDB.getConnected(record.originator());
- if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator());
+ seed = sb.wordIndex.seedDB.getConnected(record.originator());
+ if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(record.originator());
String category = record.category();
prop.put("table_list_" + i + "_id", record.id());
prop.putHTML("table_list_" + i + "_ori", (seed == null) ? record.originator() : seed.getName());
@@ -202,7 +201,7 @@ public class News { }
// adding the peer address
- prop.put("address",yacyCore.seedDB.mySeed().getPublicAddress());
+ prop.put("address", sb.wordIndex.seedDB.mySeed().getPublicAddress());
// return rewrite properties
return prop;
diff --git a/htroot/QuickCrawlLink_p.java b/htroot/QuickCrawlLink_p.java index 02c05a002..150872086 100644 --- a/htroot/QuickCrawlLink_p.java +++ b/htroot/QuickCrawlLink_p.java @@ -58,7 +58,6 @@ import de.anomic.plasma.plasmaCrawlProfile; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyURL;
public class QuickCrawlLink_p {
@@ -74,7 +73,7 @@ public class QuickCrawlLink_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
serverObjects prop = new serverObjects();
- plasmaSwitchboard switchboard = (plasmaSwitchboard)env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
if (post == null) {
// send back usage example
@@ -139,14 +138,14 @@ public class QuickCrawlLink_p { }
String urlhash = crawlingStartURL.hash();
- switchboard.wordIndex.removeURL(urlhash);
- switchboard.crawlQueues.noticeURL.removeByURLHash(urlhash);
- switchboard.crawlQueues.errorURL.remove(urlhash);
+ sb.wordIndex.removeURL(urlhash);
+ sb.crawlQueues.noticeURL.removeByURLHash(urlhash);
+ sb.crawlQueues.errorURL.remove(urlhash);
// create crawling profile
plasmaCrawlProfile.entry pe = null;
try {
- pe = switchboard.profilesActiveCrawls.newEntry(
+ pe = sb.profilesActiveCrawls.newEntry(
crawlingStartURL.getHost(),
crawlingStartURL,
crawlingFilter,
@@ -176,10 +175,10 @@ public class QuickCrawlLink_p { // stack URL
String reasonString = null;
- reasonString = switchboard.crawlStacker.stackCrawl(
+ reasonString = sb.crawlStacker.stackCrawl(
crawlingStartURL,
null,
- yacyCore.seedDB.mySeed().hash,
+ sb.wordIndex.seedDB.mySeed().hash,
(title==null)?"CRAWLING-ROOT":title,
new Date(),
0,
diff --git a/htroot/SearchEventPicture.java b/htroot/SearchEventPicture.java index d09c08298..90ca8363d 100644 --- a/htroot/SearchEventPicture.java +++ b/htroot/SearchEventPicture.java @@ -47,6 +47,7 @@ import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaGrafics;
import de.anomic.plasma.plasmaSearchEvent;
+import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.ymage.ymageMatrix;
@@ -56,10 +57,10 @@ import de.anomic.ymage.ymageMatrix; public class SearchEventPicture {
public static ymageMatrix respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
-
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
String eventID = (String) header.get("event", plasmaSearchEvent.lastEventID);
if (eventID == null) return null;
- ymageMatrix yp = plasmaGrafics.getSearchEventPicture(eventID);
+ ymageMatrix yp = plasmaGrafics.getSearchEventPicture(sb.wordIndex.seedDB, eventID);
if (yp == null) return new ymageMatrix(1, 1, ymageMatrix.MODE_SUB, "000000"); // empty image
return yp;
diff --git a/htroot/SettingsAck_p.html b/htroot/SettingsAck_p.html index 880adae07..54ca1e30a 100644 --- a/htroot/SettingsAck_p.html +++ b/htroot/SettingsAck_p.html @@ -145,38 +145,7 @@ </tr>
</table>
::<!-- 22 -->
- <p><strong>Your port forwarding settings have been changed.</strong></p>
- <table>
- <tr>
- <td>Port Forwarding Support is:</td>
- <td class="settingsValue">#[portForwarding.Enabled]#</td>
- </tr>
- <tr>
- <td>Port Forwarding Method:</td>
- <td class="settingsValue">#[portForwarding.Type]#</td>
- </tr>
- <tr><td colspan="2"><hr /></td></tr>
- <tr><td colspan="2"><strong>Secure channel specific settings:</strong></td></tr>
- <tr>
- <td>Port Forwarding Port:</td>
- <td class="settingsValue">#[portForwarding.sch.Port]#</td>
- </tr>
- <tr>
- <td>Port Forwarding Host:</td>
- <td class="settingsValue">#[portForwarding.sch.HostUser]#@#[portForwarding.sch.Host]#:#[portForwarding.sch.HostPort]#</td>
- </tr>
- <tr>
- <td>Port Forwarding uses proxy:</td>
- <td class="settingsValue">#[portForwarding.sch.UseProxy]#</td>
- </tr>
- </table>
::<!-- 23 : Port forwarding - error -->
- <p><strong>Port Forwarding Settings changed, but something is wrong.</strong></p>
- <p class="error">#[errormsg]#</p>
- <p>
- Port Forwarding was deactivated automatically.<br />
- Please return to the settings page and modify the data.<br />
- </p>
::<!-- 24 -->
<p>You are now <strong>event-based online</strong>. After a short while you should see the effect on the <a href="Status.html">status</a> page.</p>
::<!-- 25 -->
diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index c0c23180b..fdf747314 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -64,7 +64,6 @@ import de.anomic.http.httpdProxyHandler; import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.server.serverBusyThread;
import de.anomic.server.serverCodings;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
@@ -82,6 +81,7 @@ public class SettingsAck_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
// get referer for backlink
String referer = (String) header.get(httpHeader.REFERER);
@@ -234,51 +234,6 @@ public class SettingsAck_p { return prop;
}
- // port forwarding configuration
- if (post.containsKey("portForwarding")) {
- env.setConfig("portForwarding.Enabled", post.containsKey("portForwarding.Enabled")?"true":"false");
- env.setConfig("portForwarding.Type", (String)post.get("portForwarding.Type"));
-
- env.setConfig("portForwarding.sch.UseProxy",post.containsKey("portForwarding.sch.UseProxy")?"true":"false");
- env.setConfig("portForwarding.sch.Port", (String)post.get("portForwarding.sch.Port"));
- env.setConfig("portForwarding.sch.Host", (String)post.get("portForwarding.sch.Host"));
- env.setConfig("portForwarding.sch.HostPort",(String)post.get("portForwarding.sch.HostPort"));
- env.setConfig("portForwarding.sch.HostUser",(String)post.get("portForwarding.sch.HostUser"));
- env.setConfig("portForwarding.sch.HostPwd", (String)post.get("portForwarding.sch.HostPwd"));
-
- // trying to reconnect the port forwarding channel
- try {
- serverCore httpd = (serverCore) env.getThread("10_httpd");
- if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) {
- // trying to shutdown the current port forwarding channel
- serverCore.portForwarding.disconnect();
- }
- // trying to reinitialize the port forwarding
- httpd.initPortForwarding();
-
- // notifying publishSeed Thread
- serverBusyThread peerPing = env.getThread("30_peerping");
- peerPing.notifyThread();
- } catch (Exception e) {
- prop.put("info", "23");
- prop.putHTML("info_errormsg",(e.getMessage() == null) ? "unknown" : e.getMessage().replaceAll("\n","<br>"));
- return prop;
- }
-
- prop.put("info", "22");
- prop.put("info_portForwarding.Enabled", post.containsKey("portForwarding.Enabled")?"on":"off");
- prop.put("info_portForwarding.Type", (String)post.get("portForwarding.Type"));
-
- prop.put("info_portForwarding.sch.UseProxy",post.containsKey("portForwarding.sch.UseProxy")?"on":"off");
- prop.put("info_portForwarding.sch.Port", (String)post.get("portForwarding.sch.Port"));
- prop.put("info_portForwarding.sch.Host", (String)post.get("portForwarding.sch.Host"));
- prop.put("info_portForwarding.sch.HostPort",(String)post.get("portForwarding.sch.HostPort"));
- prop.put("info_portForwarding.sch.HostUser",(String)post.get("portForwarding.sch.HostUser"));
- prop.put("info_portForwarding.sch.HostPwd", (String)post.get("portForwarding.sch.HostPwd"));
- return prop;
- }
-
-
// server access
if (post.containsKey("serveraccount")) {
@@ -297,7 +252,7 @@ public class SettingsAck_p { } else {
serverCore.useStaticIP = true;
}
- yacyCore.seedDB.mySeed().put(yacySeed.IP, staticIP);
+ sb.wordIndex.seedDB.mySeed().put(yacySeed.IP, staticIP);
env.setConfig("staticIP", staticIP);
// server access data
@@ -413,7 +368,6 @@ public class SettingsAck_p { /* ====================================================================
* Enabling settings
* ==================================================================== */
- plasmaSwitchboard sb = (plasmaSwitchboard)env;
httpdProxyHandler.setRemoteProxyConfig(httpRemoteProxyConfig.init(sb));
// httpdProxyHandler.remoteProxyUse = post.get("remoteProxyUse", "").equals("on");
@@ -437,7 +391,7 @@ public class SettingsAck_p { if (post.containsKey("seedUploadRetry")) {
String error;
- if ((error = ((plasmaSwitchboard)env).yc.saveSeedList(env)) == null) {
+ if ((error = yacyCore.saveSeedList(sb)) == null) {
// trying to upload the seed-list file
prop.put("info", "13");
prop.put("info_success", "1");
@@ -468,7 +422,7 @@ public class SettingsAck_p { // try an upload
String error;
- if ((error = ((plasmaSwitchboard)env).yc.saveSeedList(env)) == null) {
+ if ((error = yacyCore.saveSeedList(sb)) == null) {
// we have successfully uploaded the seed-list file
prop.put("info_seedUploadMethod", newSeedUploadMethod);
prop.putHTML("info_seedURL",newSeedURLStr);
@@ -514,7 +468,7 @@ public class SettingsAck_p { // were changed, we now try to upload the seed list with the new settings
if (env.getConfig("seedUploadMethod","none").equalsIgnoreCase(uploaderName)) {
String error;
- if ((error = ((plasmaSwitchboard)env).yc.saveSeedList(env)) == null) {
+ if ((error = yacyCore.saveSeedList(sb)) == null) {
// we have successfully uploaded the seed file
prop.put("info", "13");
diff --git a/htroot/Settings_PortForwarding.inc b/htroot/Settings_PortForwarding.inc deleted file mode 100644 index 59ed9634d..000000000 --- a/htroot/Settings_PortForwarding.inc +++ /dev/null @@ -1,66 +0,0 @@ -<form action="SettingsAck_p.html" method="post" enctype="multipart/form-data">
-<fieldset><legend id="portForwarding">Port Forwarding (optional)</legend>
-<p>
- You can use a remote server running a ssh demon to forward your server/proxy port.
- This is useful if you want to tunnel throug a NAT/router.
- Alternatively, you can simply set a virtual server port on your NAT/Server to enable connections from outside.
-</p>
-#(portForwarding.Available)#<p><b><font color="red">You need to install libx to use this feature</font></b></p>::#(/portForwarding.Available)#
-<table border="0" cellspacing="5">
- <tr valign="top">
- <td><label for="pfwd">Enable port forwarding</label>:</td>
- <td><input type="checkbox" name="portForwarding.Enabled" id="pfwd" #(portForwarding.Enabled)#::checked="checked" #(/portForwarding.Enabled)##(portForwarding.Available)#disabled="disabled" ::#(/portForwarding.Available)#/></td>
- <td>Enabling / disabling port forwarding</td>
- </tr>
- <tr valign="top">
- <td><label for="sel_pfwd">Forwarding method</label>:</td>
- <td><select name="portForwarding.Type" id="sel_pfwd" size="1">
- #{forwardingMethods}#
- <option value="#[name]#" #(selected)#::selected="selected"#(/selected)#>#[name]#</option>
- #{/forwardingMethods}#
- </select>
- </td>
- <td>Select the port forwarding method to use.</td>
- </tr>
- <tr><td colspan="3"><hr /></td></tr>
- <tr><td colspan="3"><b>Special settings for port forwarding using secure channel</b></td></tr>
- <tr valign="top">
- <td><label for="pfwd_proxy">Forwarding via proxy</label>:</td>
- <td><input type="checkbox" name="portForwarding.sch.UseProxy" id="pfwd_proxy" #(portForwarding.sch.UseProxy)#::checked="checked" #(/portForwarding.sch.UseProxy)#disabled="disabled" /></td>
- <td>
- <!-- Specifies if the connection should be established through the <a href="#proxy">remote proxy</a>.-->
- <i>Function not available at the moment.</i>
- </td>
- </tr>
- <tr valign="top">
- <td><label for="pfwd_port">Forwarding port</label>:</td>
- <td><input name="portForwarding.sch.Port" id="pfwd_port" type="text" size="5" maxlength="5" value="#[portForwarding.sch.Port]#" /></td>
- <td>The tcp/ip port on the remote server that should be forwarded via the secure channel to the local host.</td>
- </tr>
- <tr valign="top">
- <td><label for="pfwd_host">Forwarding host</label>:</td>
- <td><input name="portForwarding.sch.Host" id="pfwd_host" type="text" size="32" maxlength="128" value="#[portForwarding.sch.Host]#" /></td>
- <td>IP address or name of the remote server.</td>
- </tr>
- <tr valign="top">
- <td><label for="pfwd_host_port">Forwarding host port</label>:</td>
- <td><input name="portForwarding.sch.HostPort" id="pfwd_host_port" type="text" size="5" maxlength="5" value="#[portForwarding.sch.HostPort]#" /></td>
- <td>SSH port of the remote server.</td>
- </tr>
- <tr valign="top">
- <td><label for="pfwd_host_user">Forwarding host user</label>:</td>
- <td><input name="portForwarding.sch.HostUser" id="pfwd_host_user" type="text" size="32" maxlength="128" value="#[portForwarding.sch.HostUser]#" /></td>
- <td>Username to login on the remote server.</td>
- </tr>
- <tr valign="top">
- <td><label for="pfwd_host_pwd">Forwarding host password</label>:</td>
- <td><input name="portForwarding.sch.HostPwd" id="pfwd_host_pwd" type="password" size="32" maxlength="128" value="#[portForwarding.sch.HostPwd]#" /></td>
- <td>Password to login on the remote server.</td>
- </tr>
- <tr><td colspan="3"><hr /></td></tr>
- <tr valign="top">
- <td colspan="3"><input type="submit" name="portForwarding" value="Submit"#(portForwarding.Available)# disabled="disabled"::#(/portForwarding.Available)# /> <i>Changes will take effect immediately.</i></td>
- </tr>
-</table>
-</fieldset>
-</form><br />
\ No newline at end of file diff --git a/htroot/Settings_p.html b/htroot/Settings_p.html index 447044d6c..0fe8266a8 100644 --- a/htroot/Settings_p.html +++ b/htroot/Settings_p.html @@ -35,7 +35,6 @@ <li><a href="?page=crawler">Crawler Settings</a></li> <li><a href="?page=http">HTTP Networking</a></li> <li><a href="?page=proxy">Remote Proxy (optional)</a></li> - <li><a href="?page=portForwarding">Port Forwarding (optional)</a></li> <li><a href="?page=seed">Seed Upload Settings</a></li> <li><a href="?page=messageForwarding">Message Forwarding (optional)</a></li> </ul> diff --git a/htroot/Settings_p.java b/htroot/Settings_p.java index 10b9edb04..f805e3d55 100644 --- a/htroot/Settings_p.java +++ b/htroot/Settings_p.java @@ -89,9 +89,6 @@ public final class Settings_p { else if (page.equals("messageForwarding")) {
prop.put("settingsTables", "Settings_MessageForwarding.inc");
}
- else if (page.equals("portForwarding")) {
- prop.put("settingsTables", "Settings_PortForwarding.inc");
- }
else if (page.equals("parser")) {
prop.put("settingsTables", "Settings_Parser.inc");
}
@@ -114,37 +111,7 @@ public final class Settings_p { prop.put("connectionKeepAliveSupport", env.getConfig("connectionKeepAliveSupport", "false").equals("true") ? "1" : "0");
prop.put("proxy.sendViaHeader", env.getConfig("proxy.sendViaHeader", "false").equals("true") ? "1" : "0");
prop.put("proxy.sendXForwardedForHeader", env.getConfig("proxy.sendXForwardedForHeader", "true").equals("true") ? "1" : "0");
-
- // remote port forwarding settings
- String[] forwardingMethods = new String[]{"sch","upnp"};
- String currentForwarder = env.getConfig("portForwarding.Type", "none");
- boolean portForwardingAvailable = false;
- int methodCount = 0;
-
- for (int i=0; i < forwardingMethods.length; i++) {
- try {
- //Class forwarder = Class.forName(env.getConfig("portForwarding." + forwardingMethods[i],"")); // FIXME: why is forwarder declared, but never used?
- prop.put("forwardingMethods_" + methodCount + "_name",forwardingMethods[i]);
- prop.put("forwardingMethods_" + methodCount + "_selected", forwardingMethods[i].equals(currentForwarder)?1:0);
- methodCount++;
- } catch (Exception e) {
- } catch (Error e) {}
- }
- prop.put("forwardingMethods", methodCount);
-
- if (methodCount > 0) portForwardingAvailable = true;
-
- prop.put("portForwarding.Type",currentForwarder);
- prop.put("portForwarding.Available",portForwardingAvailable? "1" : "0");
- prop.put("portForwarding.Enabled",env.getConfig("portForwarding.Enabled","false").equals("true")? "1" : "0");
-
- prop.put("portForwarding.sch.UseProxy",env.getConfig("portForwarding.sch.UseProxy", "false").equals("true") ? "1" : "0");
- prop.put("portForwarding.sch.Port",env.getConfig("portForwarding.sch.Port", ""));
- prop.put("portForwarding.sch.Host",env.getConfig("portForwarding.sch.Host", ""));
- prop.put("portForwarding.sch.HostPort",env.getConfig("portForwarding.sch.HostPort", ""));
- prop.put("portForwarding.sch.HostUser",env.getConfig("portForwarding.sch.HostUser", ""));
- prop.put("portForwarding.sch.HostPwd",env.getConfig("portForwarding.sch.HostPwd", ""));
-
+
// set values
String s;
int pos;
diff --git a/htroot/Status.java b/htroot/Status.java index b99f9e3ef..d1bb74c43 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -170,16 +170,6 @@ public class Status { // ssl support
prop.put("sslSupport",sb.getConfig("keyStore", "").length() == 0 ? "0" : "1");
- // port forwarding: hostname and port
- if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) {
- prop.put("portForwarding", "1");
- prop.put("portForwarding_host", serverCore.portForwarding.getHost());
- prop.put("portForwarding_port", Integer.toString(serverCore.portForwarding.getPort()));
- prop.put("portForwarding_status", serverCore.portForwarding.isConnected() ? "1" : "0");
- } else {
- prop.put("portForwarding", "0");
- }
-
if (sb.getConfig("remoteProxyUse", "false").equals("true")) {
prop.put("remoteProxy", "1");
prop.putHTML("remoteProxy_host", sb.getConfig("remoteProxyHost", "<unknown>"), true);
@@ -192,35 +182,35 @@ public class Status { // peer information
String thisHash = "";
final String thisName = sb.getConfig("peerName", "<nameless>");
- if (yacyCore.seedDB.mySeed() == null) {
+ if (sb.wordIndex.seedDB.mySeed() == null) {
thisHash = "not assigned";
prop.put("peerAddress", "0"); // not assigned
prop.put("peerStatistics", "0"); // unknown
} else {
- final long uptime = 60000 * Long.parseLong(yacyCore.seedDB.mySeed().get(yacySeed.UPTIME, "0"));
+ final long uptime = 60000 * Long.parseLong(sb.wordIndex.seedDB.mySeed().get(yacySeed.UPTIME, "0"));
prop.put("peerStatistics", "1");
prop.put("peerStatistics_uptime", serverDate.formatInterval(uptime));
- prop.putNum("peerStatistics_pagesperminute", yacyCore.seedDB.mySeed().getPPM());
- prop.putNum("peerStatistics_queriesperhour", Math.round(6000d * yacyCore.seedDB.mySeed().getQPM()) / 100d);
- prop.putNum("peerStatistics_links", yacyCore.seedDB.mySeed().getLinkCount());
- prop.put("peerStatistics_words", yFormatter.number(yacyCore.seedDB.mySeed().get(yacySeed.ICOUNT, "0")));
+ prop.putNum("peerStatistics_pagesperminute", sb.wordIndex.seedDB.mySeed().getPPM());
+ prop.putNum("peerStatistics_queriesperhour", Math.round(6000d * sb.wordIndex.seedDB.mySeed().getQPM()) / 100d);
+ prop.putNum("peerStatistics_links", sb.wordIndex.seedDB.mySeed().getLinkCount());
+ prop.put("peerStatistics_words", yFormatter.number(sb.wordIndex.seedDB.mySeed().get(yacySeed.ICOUNT, "0")));
prop.putNum("peerStatistics_juniorConnects", yacyCore.peerActions.juniorConnects);
prop.putNum("peerStatistics_seniorConnects", yacyCore.peerActions.seniorConnects);
prop.putNum("peerStatistics_principalConnects", yacyCore.peerActions.principalConnects);
prop.putNum("peerStatistics_disconnects", yacyCore.peerActions.disconnects);
- prop.put("peerStatistics_connects", yFormatter.number(yacyCore.seedDB.mySeed().get(yacySeed.CCOUNT, "0")));
- if (yacyCore.seedDB.mySeed().getPublicAddress() == null) {
- thisHash = yacyCore.seedDB.mySeed().hash;
+ prop.put("peerStatistics_connects", yFormatter.number(sb.wordIndex.seedDB.mySeed().get(yacySeed.CCOUNT, "0")));
+ if (sb.wordIndex.seedDB.mySeed().getPublicAddress() == null) {
+ thisHash = sb.wordIndex.seedDB.mySeed().hash;
prop.put("peerAddress", "0"); // not assigned + instructions
prop.put("warningGoOnline", "1");
} else {
- thisHash = yacyCore.seedDB.mySeed().hash;
+ thisHash = sb.wordIndex.seedDB.mySeed().hash;
prop.put("peerAddress", "1"); // Address
- prop.put("peerAddress_address", yacyCore.seedDB.mySeed().getPublicAddress());
+ prop.put("peerAddress_address", sb.wordIndex.seedDB.mySeed().getPublicAddress());
prop.putHTML("peerAddress_peername", sb.getConfig("peerName", "<nameless>").toLowerCase(), true);
}
}
- final String peerStatus = ((yacyCore.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_VIRGIN : yacyCore.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN));
+ final String peerStatus = ((sb.wordIndex.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_VIRGIN : sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN));
if (peerStatus.equals(yacySeed.PEERTYPE_VIRGIN)) {
prop.put(PEERSTATUS, "0");
prop.put("urgentStatusVirgin", "1");
@@ -233,7 +223,7 @@ public class Status { } else if (peerStatus.equals(yacySeed.PEERTYPE_PRINCIPAL)) {
prop.put(PEERSTATUS, "3");
prop.put("hintStatusPrincipal", "1");
- prop.put("hintStatusPrincipal_seedURL", yacyCore.seedDB.mySeed().get("seedURL", "?"));
+ prop.put("hintStatusPrincipal_seedURL", sb.wordIndex.seedDB.mySeed().get("seedURL", "?"));
}
prop.putHTML("peerName", thisName);
prop.put("hash", thisHash);
@@ -262,14 +252,14 @@ public class Status { prop.put("seedServer_seedFile", sb.getConfig("seedFilePath", ""));
}
prop.put("seedServer_lastUpload",
- serverDate.formatInterval(System.currentTimeMillis() - sb.yc.lastSeedUpload_timeStamp));
+ serverDate.formatInterval(System.currentTimeMillis() - sb.wordIndex.seedDB.lastSeedUpload_timeStamp));
} else {
prop.put(SEEDSERVER, "0"); // disabled
}
- if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0){
+ if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0){
prop.put("otherPeers", "1");
- prop.putNum("otherPeers_num", yacyCore.seedDB.sizeConnected());
+ prop.putNum("otherPeers_num", sb.wordIndex.seedDB.sizeConnected());
}else{
prop.put("otherPeers", "0"); // not online
}
diff --git a/htroot/Supporter.java b/htroot/Supporter.java index 236d88de8..091a96c6f 100644 --- a/htroot/Supporter.java +++ b/htroot/Supporter.java @@ -43,7 +43,6 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.tools.crypt; import de.anomic.tools.nxTools; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; @@ -79,7 +78,7 @@ public class Supporter { map.put("urlhash", hash); map.put("vote", "negative"); map.put("refid", post.get("refid", "")); - yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } if ((post != null) && ((hash = post.get("votePositive", null)) != null)) { if (!sb.verifyAuthentication(header, false)) { @@ -95,19 +94,19 @@ public class Supporter { map.put("vote", "positive"); map.put("refid", post.get("refid", "")); map.put("comment", post.get("comment", "")); - yacyCore.newsPool.publishMyNews(new yacyNewsRecord(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.wordIndex.newsPool.publishMyNews(new yacyNewsRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } // create Supporter HashMap<String, Integer> negativeHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes) HashMap<String, Integer> positiveHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes) - accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); + accumulateVotes(sb, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); kelondroMScoreCluster<String> ranking = new kelondroMScoreCluster<String>(); // score cluster for url hashes kelondroRow rowdef = new kelondroRow("String url-255, String title-120, String description-120, String refid-" + (serverDate.PATTERN_SHORT_SECOND.length() + 12), kelondroNaturalOrder.naturalOrder, 0); HashMap<String, Entry> Supporter = new HashMap<String, Entry>(); // a mapping from an url hash to a kelondroRow.Entry with display properties - accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); + accumulateSupporter(sb, Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); //accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); //accumulateSupporter(Supporter, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); @@ -132,8 +131,8 @@ public class Supporter { description = row.getColString(2,"UTF-8"); if ((url == null) || (title == null) || (description == null)) continue; refid = row.getColString(3, null); - voted = (yacyCore.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || - (yacyCore.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null); + voted = (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || + (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null); prop.put("supporter_results_" + i + "_authorized", authenticated ? "1" : "0"); prop.put("supporter_results_" + i + "_authorized_recommend", voted ? "0" : "1"); @@ -168,10 +167,10 @@ public class Supporter { return (int) Math.max(0, 10 - ((System.currentTimeMillis() - created.getTime()) / 24 / 60 / 60 / 1000)); } - private static void accumulateVotes(HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) { - int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype)); + private static void accumulateVotes(plasmaSwitchboard sb, HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) { + int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype)); yacyNewsRecord record; - Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(dbtype, true); + Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true); int j = 0; while ((recordIterator.hasNext()) && (j++ < maxCount)) { record = recordIterator.next(); @@ -196,11 +195,12 @@ public class Supporter { } private static void accumulateSupporter( + plasmaSwitchboard sb, HashMap<String, Entry> Supporter, kelondroMScoreCluster<String> ranking, kelondroRow rowdef, HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) { - int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype)); + int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype)); yacyNewsRecord record; - Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(dbtype, true); + Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true); int j = 0; String url = "", urlhash; kelondroRow.Entry entry; @@ -213,7 +213,7 @@ public class Supporter { entry = null; if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_UPDATE)) && - ((seed = yacyCore.seedDB.getConnected(record.originator())) != null)) try { + ((seed = sb.wordIndex.seedDB.getConnected(record.originator())) != null)) try { url = record.attribute("homepage", ""); if (url.length() < 12) continue; entry = rowdef.newEntry(new byte[][]{ @@ -226,7 +226,7 @@ public class Supporter { } catch (IOException e) {} if ((record.category().equals(yacyNewsPool.CATEGORY_PROFILE_BROADCAST)) && - ((seed = yacyCore.seedDB.getConnected(record.originator())) != null)) try { + ((seed = sb.wordIndex.seedDB.getConnected(record.originator())) != null)) try { url = record.attribute("homepage", ""); if (url.length() < 12) continue; entry = rowdef.newEntry(new byte[][]{ diff --git a/htroot/Surftips.java b/htroot/Surftips.java index 9a684f592..9f3c3a137 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -43,7 +43,6 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.tools.crypt; import de.anomic.tools.nxTools; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacySeed; @@ -87,7 +86,7 @@ public class Surftips { map.put("urlhash", hash); map.put("vote", "negative"); map.put("refid", post.get("refid", "")); - yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } if ((post != null) && ((hash = post.get("votePositive", null)) != null)) { if (!sb.verifyAuthentication(header, false)) { @@ -103,19 +102,19 @@ public class Surftips { map.put("vote", "positive"); map.put("refid", post.get("refid", "")); map.put("comment", post.get("comment", "")); - yacyCore.newsPool.publishMyNews(new yacyNewsRecord(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); + sb.wordIndex.newsPool.publishMyNews(new yacyNewsRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map)); } // create surftips HashMap<String, Integer> negativeHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes) HashMap<String, Integer> positiveHashes = new HashMap<String, Integer>(); // a mapping from an url hash to Integer (count of votes) - accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); + accumulateVotes(sb , negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); //accumulateVotes(negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); kelondroMScoreCluster<String> ranking = new kelondroMScoreCluster<String>(); // score cluster for url hashes kelondroRow rowdef = new kelondroRow("String url-255, String title-120, String description-120, String refid-" + (serverDate.PATTERN_SHORT_SECOND.length() + 12), kelondroNaturalOrder.naturalOrder, 0); HashMap<String, Entry> surftips = new HashMap<String, Entry>(); // a mapping from an url hash to a kelondroRow.Entry with display properties - accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); + accumulateSurftips(sb, surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.INCOMING_DB); //accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.OUTGOING_DB); //accumulateSurftips(surftips, ranking, rowdef, negativeHashes, positiveHashes, yacyNewsPool.PUBLISHED_DB); @@ -141,8 +140,8 @@ public class Surftips { description = row.getColString(2,"UTF-8"); if ((url == null) || (title == null) || (description == null)) continue; refid = row.getColString(3, null); - voted = (yacyCore.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || - (yacyCore.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null); + voted = (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null) || + (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.PUBLISHED_DB, yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, "refid", refid) != null); prop.put("surftips_results_" + i + "_authorized", (authenticated) ? "1" : "0"); prop.put("surftips_results_" + i + "_authorized_recommend", (voted) ? "0" : "1"); @@ -177,10 +176,10 @@ public class Surftips { return (int) Math.max(0, 10 - ((System.currentTimeMillis() - created.getTime()) / 24 / 60 / 60 / 1000)); } - private static void accumulateVotes(HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) { - int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype)); + private static void accumulateVotes(plasmaSwitchboard sb, HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) { + int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype)); yacyNewsRecord record; - Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(dbtype, true); + Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true); int j = 0; while ((recordIterator.hasNext()) && (j++ < maxCount)) { record = recordIterator.next(); @@ -205,11 +204,12 @@ public class Surftips { } private static void accumulateSurftips( + plasmaSwitchboard sb, HashMap<String, Entry> surftips, kelondroMScoreCluster<String> ranking, kelondroRow rowdef, HashMap<String, Integer> negativeHashes, HashMap<String, Integer> positiveHashes, int dbtype) { - int maxCount = Math.min(1000, yacyCore.newsPool.size(dbtype)); + int maxCount = Math.min(1000, sb.wordIndex.newsPool.size(dbtype)); yacyNewsRecord record; - Iterator<yacyNewsRecord> recordIterator = yacyCore.newsPool.recordIterator(dbtype, true); + Iterator<yacyNewsRecord> recordIterator = sb.wordIndex.newsPool.recordIterator(dbtype, true); int j = 0; String url = "", urlhash; kelondroRow.Entry entry; @@ -271,8 +271,8 @@ public class Surftips { } catch (IOException e) {} if (record.category().equals(yacyNewsPool.CATEGORY_WIKI_UPDATE)) try { - yacySeed seed = yacyCore.seedDB.getConnected(record.originator()); - if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator()); + yacySeed seed = sb.wordIndex.seedDB.getConnected(record.originator()); + if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(record.originator()); if (seed != null) { url = "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ @@ -286,8 +286,8 @@ public class Surftips { } catch (IOException e) {} if (record.category().equals(yacyNewsPool.CATEGORY_BLOG_ADD)) try { - yacySeed seed = yacyCore.seedDB.getConnected(record.originator()); - if (seed == null) seed = yacyCore.seedDB.getDisconnected(record.originator()); + yacySeed seed = sb.wordIndex.seedDB.getConnected(record.originator()); + if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(record.originator()); if (seed != null) { url = "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ diff --git a/htroot/ViewProfile.java b/htroot/ViewProfile.java index e82ccad99..00de21e05 100644 --- a/htroot/ViewProfile.java +++ b/htroot/ViewProfile.java @@ -70,15 +70,14 @@ public class ViewProfile { @SuppressWarnings("unchecked")
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
- // listManager.switchboard = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
- boolean authenticated = switchboard.adminAuthenticated(header) >= 2;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
+ boolean authenticated = sb.adminAuthenticated(header) >= 2;
int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0);
prop.put("display", display);
String hash = (post == null) ? null : (String) post.get("hash");
- if ((hash == null) || (yacyCore.seedDB == null)) {
+ if ((hash == null) || (sb.wordIndex.seedDB == null)) {
// wrong access
prop.put("success", "0");
return prop;
@@ -102,24 +101,24 @@ public class ViewProfile { profile.putAll(p);
prop.put("success", "3"); // everything ok
prop.put("localremotepeer", "0");
- prop.putHTML("success_peername", yacyCore.seedDB.mySeed().getName());
- prop.put("success_peerhash", yacyCore.seedDB.mySeed().hash);
- address = yacyCore.seedDB.mySeed().getPublicAddress();
+ prop.putHTML("success_peername", sb.wordIndex.seedDB.mySeed().getName());
+ prop.put("success_peerhash", sb.wordIndex.seedDB.mySeed().hash);
+ address = sb.wordIndex.seedDB.mySeed().getPublicAddress();
} else {
// read the profile from remote peer
- yacySeed seed = yacyCore.seedDB.getConnected(hash);
- if (seed == null) seed = yacyCore.seedDB.getDisconnected(hash);
+ yacySeed seed = sb.wordIndex.seedDB.getConnected(hash);
+ if (seed == null) seed = sb.wordIndex.seedDB.getDisconnected(hash);
if (seed == null) {
prop.put("success", "1"); // peer unknown
} else {
// process news if existent
try {
- yacyNewsRecord record = yacyCore.newsPool.getByOriginator(yacyNewsPool.INCOMING_DB, yacyNewsPool.CATEGORY_PROFILE_UPDATE, seed.hash);
- if (record != null) yacyCore.newsPool.moveOff(yacyNewsPool.INCOMING_DB, record.id());
+ yacyNewsRecord record = sb.wordIndex.newsPool.getByOriginator(yacyNewsPool.INCOMING_DB, yacyNewsPool.CATEGORY_PROFILE_UPDATE, seed.hash);
+ if (record != null) sb.wordIndex.newsPool.moveOff(yacyNewsPool.INCOMING_DB, record.id());
} catch (IOException e) {}
// try to get the profile from remote peer
- if (switchboard.clusterhashes != null) seed.setAlternativeAddress((String) switchboard.clusterhashes.get(seed.hash));
+ if (sb.clusterhashes != null) seed.setAlternativeAddress((String) sb.clusterhashes.get(seed.hash));
profile = yacyClient.getProfile(seed);
// if profile did not arrive, say that peer is disconnected
diff --git a/htroot/WatchCrawler_p.java b/htroot/WatchCrawler_p.java index 574a95440..0137b8620 100644 --- a/htroot/WatchCrawler_p.java +++ b/htroot/WatchCrawler_p.java @@ -38,6 +38,7 @@ import java.util.regex.PatternSyntaxException; import de.anomic.htmlFilter.htmlFilterContentScraper; import de.anomic.htmlFilter.htmlFilterWriter; import de.anomic.http.httpHeader; +import de.anomic.plasma.plasmaCrawlEntry; import de.anomic.plasma.plasmaCrawlProfile; import de.anomic.plasma.plasmaCrawlZURL; import de.anomic.plasma.plasmaSwitchboard; @@ -45,7 +46,6 @@ import de.anomic.plasma.dbImport.SitemapImporter; import de.anomic.server.serverFileUtils; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacyURL; @@ -61,7 +61,7 @@ public class WatchCrawler_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) { // return variable that accumulates replacements - plasmaSwitchboard switchboard = (plasmaSwitchboard) env; + plasmaSwitchboard sb = (plasmaSwitchboard) env; serverObjects prop = new serverObjects(); prop.put("forwardToCrawlStart", "0"); @@ -72,9 +72,9 @@ public class WatchCrawler_p { prop.put("info", "0"); if ((post.containsKey("autoforward")) && - (switchboard.crawlQueues.coreCrawlJobSize() == 0) && - (switchboard.crawlQueues.remoteTriggeredCrawlJobSize() == 0) && - (switchboard.queueSize() < 30)) { + (sb.crawlQueues.coreCrawlJobSize() == 0) && + (sb.crawlQueues.remoteTriggeredCrawlJobSize() == 0) && + (sb.queueSize() < 30)) { prop.put("forwardToCrawlStart", "1"); } @@ -82,9 +82,9 @@ public class WatchCrawler_p { // continue queue String queue = post.get("continue", ""); if (queue.equals("localcrawler")) { - switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); + sb.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); } else if (queue.equals("remotecrawler")) { - switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); + sb.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); } } @@ -92,15 +92,15 @@ public class WatchCrawler_p { // pause queue String queue = post.get("pause", ""); if (queue.equals("localcrawler")) { - switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); + sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); } else if (queue.equals("remotecrawler")) { - switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); + sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); } } if (post.containsKey("crawlingstart")) { // init crawl - if (yacyCore.seedDB == null) { + if (sb.wordIndex.seedDB == null) { prop.put("info", "3"); } else { // set new properties @@ -185,20 +185,20 @@ public class WatchCrawler_p { // first delete old entry, if exists yacyURL url = new yacyURL(crawlingStart, null); String urlhash = url.hash(); - switchboard.wordIndex.removeURL(urlhash); - switchboard.crawlQueues.noticeURL.removeByURLHash(urlhash); - switchboard.crawlQueues.errorURL.remove(urlhash); + sb.wordIndex.removeURL(urlhash); + sb.crawlQueues.noticeURL.removeByURLHash(urlhash); + sb.crawlQueues.errorURL.remove(urlhash); // stack url - switchboard.profilesPassiveCrawls.removeEntry(crawlingStartURL.hash()); // if there is an old entry, delete it - plasmaCrawlProfile.entry pe = switchboard.profilesActiveCrawls.newEntry( + sb.profilesPassiveCrawls.removeEntry(crawlingStartURL.hash()); // if there is an old entry, delete it + plasmaCrawlProfile.entry pe = sb.profilesActiveCrawls.newEntry( crawlingStartURL.getHost(), crawlingStartURL, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw); - String reasonString = switchboard.crawlStacker.stackCrawl(url, null, yacyCore.seedDB.mySeed().hash, "CRAWLING-ROOT", new Date(), 0, pe); + String reasonString = sb.crawlStacker.stackCrawl(url, null, sb.wordIndex.seedDB.mySeed().hash, "CRAWLING-ROOT", new Date(), 0, pe); if (reasonString == null) { // liftoff! @@ -220,7 +220,7 @@ public class WatchCrawler_p { m.remove("generalFilter"); m.remove("specificFilter"); m.put("intention", post.get("intention", "").replace(',', '/')); - yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_CRAWL_START, m)); + sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_CRAWL_START, m)); } } else { @@ -228,9 +228,24 @@ public class WatchCrawler_p { prop.putHTML("info_crawlingURL", ((String) post.get("crawlingURL"))); prop.putHTML("info_reasonString", reasonString); - plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(crawlingStartURL, reasonString); + plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry( + new plasmaCrawlEntry( + sb.wordIndex.seedDB.mySeed().hash, + crawlingStartURL, + "", + "", + new Date(), + pe.handle(), + 0, + 0, + 0), + sb.wordIndex.seedDB.mySeed().hash, + new Date(), + 1, + reasonString); + ee.store(); - switchboard.crawlQueues.errorURL.push(ee); + sb.crawlQueues.errorURL.push(ee); } } catch (PatternSyntaxException e) { prop.put("info", "4"); //crawlfilter does not match url @@ -270,10 +285,10 @@ public class WatchCrawler_p { // creating a crawler profile yacyURL crawlURL = new yacyURL("file://" + file.toString(), null); - plasmaCrawlProfile.entry profile = switchboard.profilesActiveCrawls.newEntry(fileName, crawlURL, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw); + plasmaCrawlProfile.entry profile = sb.profilesActiveCrawls.newEntry(fileName, crawlURL, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw); // pause local crawl here - switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); + sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); // loop through the contained links Iterator<Map.Entry<yacyURL, String>> linkiterator = hyperlinks.entrySet().iterator(); @@ -284,10 +299,10 @@ public class WatchCrawler_p { if (nexturl == null) continue; // enqueuing the url for crawling - switchboard.crawlStacker.enqueueEntry( + sb.crawlStacker.enqueueEntry( nexturl, null, - yacyCore.seedDB.mySeed().hash, + sb.wordIndex.seedDB.mySeed().hash, (String) e.getValue(), new Date(), 0, @@ -306,7 +321,7 @@ public class WatchCrawler_p { prop.putHTML("info_error", e.getMessage()); e.printStackTrace(); } - switchboard.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); + sb.continueCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); } } else if (crawlingMode.equals(CRAWLING_MODE_SITEMAP)) { String sitemapURLStr = null; @@ -316,7 +331,7 @@ public class WatchCrawler_p { yacyURL sitemapURL = new yacyURL(sitemapURLStr, null); // create a new profile - plasmaCrawlProfile.entry pe = switchboard.profilesActiveCrawls.newEntry( + plasmaCrawlProfile.entry pe = sb.profilesActiveCrawls.newEntry( sitemapURLStr, sitemapURL, newcrawlingfilter, newcrawlingfilter, newcrawlingdepth, newcrawlingdepth, crawlingIfOlder, crawlingDomFilterDepth, crawlingDomMaxPages, @@ -325,9 +340,9 @@ public class WatchCrawler_p { storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw); // create a new sitemap importer - SitemapImporter importerThread = (SitemapImporter) switchboard.dbImportManager.getNewImporter("sitemap"); + SitemapImporter importerThread = (SitemapImporter) sb.dbImportManager.getNewImporter("sitemap"); if (importerThread != null) { - importerThread.init(switchboard, 0); + importerThread.init(sb, 0); importerThread.initSitemap(new yacyURL(sitemapURLStr, null), pe); importerThread.startIt(); } @@ -343,7 +358,7 @@ public class WatchCrawler_p { } if (post.containsKey("crawlingPerformance")) { - setPerformance(switchboard, post); + setPerformance(sb, post); } } diff --git a/htroot/Wiki.java b/htroot/Wiki.java index f78770229..00b4ac197 100644 --- a/htroot/Wiki.java +++ b/htroot/Wiki.java @@ -59,7 +59,6 @@ import de.anomic.http.httpHeader; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
@@ -75,48 +74,48 @@ public class Wiki { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) throws IOException {
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if (post == null) {
post = new serverObjects();
post.put("page", "start");
}
- final boolean authenticated = switchboard.adminAuthenticated(header) >= 2;
+ final boolean authenticated = sb.adminAuthenticated(header) >= 2;
final int display = ((post == null) || (!authenticated)) ? 0 : post.getInt("display", 0);
prop.put("display", display);
- String access = switchboard.getConfig("WikiAccess", "admin");
+ String access = sb.getConfig("WikiAccess", "admin");
String pagename = post.get("page", "start");
String ip = post.get(httpHeader.CONNECTION_PROP_CLIENTIP, "127.0.0.1");
String author = post.get("author", "anonymous");
if (author.equals("anonymous")) {
author = wikiBoard.guessAuthor(ip);
if (author == null) {
- if (de.anomic.yacy.yacyCore.seedDB.mySeed() == null) author = "anonymous";
- else author = de.anomic.yacy.yacyCore.seedDB.mySeed().get("Name", "anonymous");
+ if (sb.wordIndex.seedDB.mySeed() == null) author = "anonymous";
+ else author = sb.wordIndex.seedDB.mySeed().get("Name", "anonymous");
}
}
if (post.containsKey("access")) {
// only the administrator may change the access right
- if (!switchboard.verifyAuthentication(header, true)) {
+ if (!sb.verifyAuthentication(header, true)) {
// check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
}
access = post.get("access", "admin");
- switchboard.setConfig("WikiAccess", access);
+ sb.setConfig("WikiAccess", access);
}
if (access.equals("admin")) prop.put("mode_access", "0");
if (access.equals("all")) prop.put("mode_access", "1");
- wikiBoard.entry page = switchboard.wikiDB.read(pagename);
+ wikiBoard.entry page = sb.wikiDB.read(pagename);
if (post.containsKey("submit")) {
- if ((access.equals("admin") && (!switchboard.verifyAuthentication(header, true)))) {
+ if ((access.equals("admin") && (!sb.verifyAuthentication(header, true)))) {
// check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
@@ -129,20 +128,20 @@ public class Wiki { } catch (UnsupportedEncodingException e) {
content = post.get("content", "").getBytes();
}
- wikiBoard.entry newEntry = switchboard.wikiDB.newEntry(pagename, author, ip, post.get("reason", "edit"), content);
- switchboard.wikiDB.write(newEntry);
+ wikiBoard.entry newEntry = sb.wikiDB.newEntry(pagename, author, ip, post.get("reason", "edit"), content);
+ sb.wikiDB.write(newEntry);
// create a news message
HashMap<String, String> map = new HashMap<String, String>();
map.put("page", pagename);
map.put("author", author.replace(',', ' '));
if (post.get("content", "").trim().length() > 0 && !page.page().equals(content))
- yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_WIKI_UPDATE, map));
+ sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_WIKI_UPDATE, map));
page = newEntry;
prop.put("LOCATION", "/Wiki.html?page=" + pagename);
}
if (post.containsKey("edit")) {
- if ((access.equals("admin") && (!switchboard.verifyAuthentication(header, true)))) {
+ if ((access.equals("admin") && (!sb.verifyAuthentication(header, true)))) {
// check access right for admin
prop.put("AUTHENTICATE", "admin log-in"); // force log-in
return prop;
@@ -174,12 +173,12 @@ public class Wiki { prop.put("mode", "3"); //Index
String subject;
try {
- Iterator<String> i = switchboard.wikiDB.keys(true);
+ Iterator<String> i = sb.wikiDB.keys(true);
wikiBoard.entry entry;
int count=0;
while (i.hasNext()) {
subject = i.next();
- entry = switchboard.wikiDB.read(subject);
+ entry = sb.wikiDB.read(subject);
prop.putHTML("mode_pages_"+count+"_name",wikiBoard.webalize(subject));
prop.putHTML("mode_pages_"+count+"_subject", subject);
prop.put("mode_pages_"+count+"_date", dateString(entry.date()));
@@ -201,14 +200,14 @@ public class Wiki { prop.putHTML("mode_error_page", pagename);
try {
- Iterator<String> it = switchboard.wikiDB.keysBkp(true);
+ Iterator<String> it = sb.wikiDB.keysBkp(true);
wikiBoard.entry entry;
wikiBoard.entry oentry = null;
wikiBoard.entry nentry = null;
int count = 0;
boolean oldselected = false, newselected = false;
while (it.hasNext()) {
- entry = switchboard.wikiDB.readBkp(it.next());
+ entry = sb.wikiDB.readBkp(it.next());
prop.put("mode_error_versions_" + count + "_date", wikiBoard.dateString(entry.date()));
prop.put("mode_error_versions_" + count + "_fdate", dateString(entry.date()));
if (wikiBoard.dateString(entry.date()).equals(post.get("old", null))) {
@@ -235,7 +234,7 @@ public class Wiki { prop.put("mode_error_versions", count);
}
- entry = switchboard.wikiDB.read(pagename);
+ entry = sb.wikiDB.read(pagename);
if (entry != null) {
prop.put("mode_error_curdate", wikiBoard.dateString(entry.date()));
prop.put("mode_error_curfdate", dateString(entry.date()));
diff --git a/htroot/YaCySearchPluginFF.java b/htroot/YaCySearchPluginFF.java index fcbe4caf0..0f50c64c0 100644 --- a/htroot/YaCySearchPluginFF.java +++ b/htroot/YaCySearchPluginFF.java @@ -44,9 +44,9 @@ //if the shell's current path is HTROOT
import de.anomic.http.httpHeader;
+import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
public class YaCySearchPluginFF {
@@ -57,7 +57,7 @@ public class YaCySearchPluginFF { * @return the rewrite-properties for the template
*/
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
-
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
// getting the http host header
@@ -72,7 +72,7 @@ public class YaCySearchPluginFF { prop.put("host", host);
prop.put("port", port);
- prop.putHTML("name", yacyCore.seedDB.mySeed().getName());
+ prop.putHTML("name", sb.wordIndex.seedDB.mySeed().getName());
return prop;
}
diff --git a/htroot/index.java b/htroot/index.java index 4514204e3..be7a6debc 100644 --- a/htroot/index.java +++ b/htroot/index.java @@ -34,7 +34,6 @@ import de.anomic.plasma.plasmaSearchQuery; import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
public class index {
@@ -80,7 +79,7 @@ public class index { if (cds.equals("app")) contentdom = plasmaSearchQuery.CONTENTDOM_APP;
//long mylinks = 0;
- prop.putNum("links", yacyCore.seedDB.mySeed().getLinkCount());
+ prop.putNum("links", sb.wordIndex.seedDB.mySeed().getLinkCount());
// we create empty entries for template strings
String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", "");
diff --git a/htroot/opensearchdescription.java b/htroot/opensearchdescription.java index 02f0b9c97..ca6dfabac 100644 --- a/htroot/opensearchdescription.java +++ b/htroot/opensearchdescription.java @@ -25,15 +25,15 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import de.anomic.http.httpHeader; +import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import de.anomic.yacy.yacyCore; public class opensearchdescription { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) { - + plasmaSwitchboard sb = (plasmaSwitchboard) env; // generate message content for open search description String promoteSearchPageGreeting = env.getConfig("promoteSearchPageGreeting", ""); if (env.getConfigBool("promoteSearchPageGreeting.useNetworkName", false)) promoteSearchPageGreeting = env.getConfig("network.unit.description", ""); @@ -45,7 +45,7 @@ public class opensearchdescription { final serverObjects prop = new serverObjects(); prop.putHTML("thisaddress", thisaddress, true); prop.putHTML("SearchPageGreeting", promoteSearchPageGreeting, true); - prop.putHTML("clientname", yacyCore.seedDB.mySeed().getName(), true); + prop.putHTML("clientname", sb.wordIndex.seedDB.mySeed().getName(), true); // return rewrite properties return prop; diff --git a/htroot/rct_p.java b/htroot/rct_p.java index d79c2526a..94b79eb3c 100644 --- a/htroot/rct_p.java +++ b/htroot/rct_p.java @@ -52,8 +52,8 @@ public class rct_p { if (post != null) { if (post.containsKey("retrieve")) { String peerhash = post.get("peer", null); - yacySeed seed = (peerhash == null) ? null : yacyCore.seedDB.getConnected(peerhash); - RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(seed, 10); + yacySeed seed = (peerhash == null) ? null : sb.wordIndex.seedDB.getConnected(peerhash); + RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(sb.wordIndex.seedDB, seed, 10); if (feed != null) { for (RSSMessage item: feed) { //System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ", pubDate=" + item.getPubDate()); @@ -95,17 +95,17 @@ public class rct_p { } } - listHosts(prop); + listHosts(sb, prop); // return rewrite properties return prop; } - private static void listHosts(serverObjects prop) { + private static void listHosts(plasmaSwitchboard sb, serverObjects prop) { // list known hosts yacySeed seed; int hc = 0; - if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) { + if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0) { Iterator<yacySeed> e = yacyCore.dhtAgent.getProvidesRemoteCrawlURLs(); while (e.hasNext()) { seed = e.next(); diff --git a/htroot/sharedBlacklist_p.java b/htroot/sharedBlacklist_p.java index 7e40081fa..26704e693 100644 --- a/htroot/sharedBlacklist_p.java +++ b/htroot/sharedBlacklist_p.java @@ -63,7 +63,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.tools.nxTools;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@@ -77,6 +76,7 @@ public class sharedBlacklist_p { public static final int STATUS_WRONG_INVOCATION = 5;
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
// return variable that accumulates replacements
serverObjects prop = new serverObjects();
@@ -104,8 +104,8 @@ public class sharedBlacklist_p { // generate the download URL
String downloadURL = null;
- if( yacyCore.seedDB != null ){ //no nullpointer error..
- yacySeed seed = yacyCore.seedDB.getConnected(Hash);
+ if( sb.wordIndex.seedDB != null ){ //no nullpointer error..
+ yacySeed seed = sb.wordIndex.seedDB.getConnected(Hash);
if (seed != null) {
String IP = seed.get(yacySeed.IP, "127.0.0.1");
String Port = seed.get(yacySeed.PORT, "8080");
diff --git a/htroot/www/welcome.java b/htroot/www/welcome.java index bd2337025..5b7941a83 100644 --- a/htroot/www/welcome.java +++ b/htroot/www/welcome.java @@ -52,6 +52,7 @@ import java.net.InetAddress; import java.net.UnknownHostException;
import de.anomic.http.httpHeader;
+import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverDomains;
import de.anomic.server.serverObjects;
@@ -62,6 +63,8 @@ import de.anomic.yacy.yacySeed; public class welcome {
public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
+
// return variable that accumulates replacements
final serverObjects prop = new serverObjects();
@@ -70,7 +73,7 @@ public class welcome { prop.putHTML("peername", env.getConfig("peerName", "<nameless>"));
prop.putHTML("peerdomain", env.getConfig("peerName", "<nameless>").toLowerCase());
- prop.putHTML("peeraddress", yacyCore.seedDB.mySeed().getPublicAddress());
+ prop.putHTML("peeraddress", sb.wordIndex.seedDB.mySeed().getPublicAddress());
prop.put("hostname", serverDomains.myPublicIP());
try{
prop.put("hostip", InetAddress.getByName(serverDomains.myPublicIP()).getHostAddress());
@@ -80,7 +83,7 @@ public class welcome { prop.put("port", serverCore.getPortNr(env.getConfig("port","8080")));
prop.put("clientip", (String) header.get(httpHeader.CONNECTION_PROP_CLIENTIP, ""));
- final String peertype = (yacyCore.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_JUNIOR : yacyCore.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
+ final String peertype = (sb.wordIndex.seedDB.mySeed() == null) ? yacySeed.PEERTYPE_JUNIOR : sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN);
final boolean senior = (peertype.equals(yacySeed.PEERTYPE_SENIOR)) || (peertype.equals(yacySeed.PEERTYPE_PRINCIPAL));
if (senior) { prop.put("couldcan", "can"); } else { prop.put("couldcan", "could"); }
if (senior) { prop.put("seniorinfo", "This peer runs in senior mode which means that your peer can be accessed using the addresses shown above."); } else { prop.putHTML("seniorinfo", "<b>Nobody can access your peer from the outside of your intranet. You must open your firewall and/or set a 'virtual server' in the settings of your router to enable access to the addresses as shown below.</b>"); }
diff --git a/htroot/xml/queues_p.java b/htroot/xml/queues_p.java index 23aaf1b46..9fa74a621 100644 --- a/htroot/xml/queues_p.java +++ b/htroot/xml/queues_p.java @@ -60,7 +60,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.plasma.plasmaSwitchboardQueue;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
public class queues_p {
@@ -119,7 +118,7 @@ public class queues_p { if ((pcentry != null) && (pcentry.url() != null)) {
long entrySize = pcentry.size();
totalSize += entrySize;
- initiator = yacyCore.seedDB.getConnected(pcentry.initiator());
+ initiator = sb.wordIndex.seedDB.getConnected(pcentry.initiator());
prop.put("list-indexing_"+i+"_profile", (pcentry.profile() != null) ? pcentry.profile().name() : "deleted");
prop.put("list-indexing_"+i+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("list-indexing_"+i+"_depth", pcentry.depth());
@@ -146,7 +145,7 @@ public class queues_p { for (int i = 0; i < w.length; i++) {
if (w[i] == null) continue;
prop.put("list-loader_"+count+"_profile", w[i].profileHandle());
- initiator = yacyCore.seedDB.getConnected(w[i].initiator());
+ initiator = sb.wordIndex.seedDB.getConnected(w[i].initiator());
prop.put("list-loader_"+count+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put("list-loader_"+count+"_depth", w[i].depth());
prop.putHTML("list-loader_"+count+"_url", w[i].url().toString(), true);
@@ -159,7 +158,7 @@ public class queues_p { prop.putNum("localCrawlSize", Integer.toString(sb.getThread(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL).getJobCount()));
prop.put("localCrawlState", sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
int stackSize = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
- addNTable(prop, "list-local", sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
+ addNTable(sb, prop, "list-local", sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
//global crawl queue prop.putNum("limitCrawlSize", Integer.toString(sb.crawlQueues.limitCrawlJobSize()));
@@ -174,7 +173,7 @@ public class queues_p { if (stackSize == 0) {
prop.put("list-remote", "0");
} else {
- addNTable(prop, "list-remote", sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, Math.min(10, stackSize)));
+ addNTable(sb, prop, "list-remote", sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, Math.min(10, stackSize)));
}
// return rewrite properties
@@ -182,7 +181,7 @@ public class queues_p { }
- public static final void addNTable(serverObjects prop, String tableName, plasmaCrawlEntry[] crawlerList) {
+ public static final void addNTable(plasmaSwitchboard sb, serverObjects prop, String tableName, plasmaCrawlEntry[] crawlerList) {
int showNum = 0;
plasmaCrawlEntry urle;
@@ -190,7 +189,7 @@ public class queues_p { for (int i = 0; i < crawlerList.length; i++) {
urle = crawlerList[i];
if ((urle != null) && (urle.url() != null)) {
- initiator = yacyCore.seedDB.getConnected(urle.initiator());
+ initiator = sb.wordIndex.seedDB.getConnected(urle.initiator());
prop.put(tableName + "_" + showNum + "_profile", urle.profileHandle());
prop.put(tableName + "_" + showNum + "_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
prop.put(tableName + "_" + showNum + "_depth", urle.depth());
diff --git a/htroot/xml/status_p.java b/htroot/xml/status_p.java index d09337353..11258d3eb 100644 --- a/htroot/xml/status_p.java +++ b/htroot/xml/status_p.java @@ -54,19 +54,19 @@ public class status_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if (post == null || !post.containsKey("html"))
prop.setLocalized(false);
prop.put("rejected", "0");
yacyCore.peerActions.updateMySeed();
- final int cacheOutSize = switchboard.wordIndex.dhtOutCacheSize();
- final long cacheMaxSize = switchboard.getConfigLong(plasmaSwitchboard.WORDCACHE_MAX_COUNT, 10000);
- prop.putNum("ppm", yacyCore.seedDB.mySeed().getPPM());
- prop.putNum("qpm", yacyCore.seedDB.mySeed().getQPM());
- prop.putNum("wordCacheSize", switchboard.wordIndex.dhtOutCacheSize() + switchboard.wordIndex.dhtInCacheSize());
+ final int cacheOutSize = sb.wordIndex.dhtOutCacheSize();
+ final long cacheMaxSize = sb.getConfigLong(plasmaSwitchboard.WORDCACHE_MAX_COUNT, 10000);
+ prop.putNum("ppm", sb.wordIndex.seedDB.mySeed().getPPM());
+ prop.putNum("qpm", sb.wordIndex.seedDB.mySeed().getQPM());
+ prop.putNum("wordCacheSize", sb.wordIndex.dhtOutCacheSize() + sb.wordIndex.dhtInCacheSize());
prop.putNum("wordCacheWSize", cacheOutSize);
- prop.putNum("wordCacheKSize", switchboard.wordIndex.dhtInCacheSize());
+ prop.putNum("wordCacheKSize", sb.wordIndex.dhtInCacheSize());
prop.putNum("wordCacheMaxSize", cacheMaxSize);
prop.put("wordCacheWCount", cacheOutSize);
prop.put("wordCacheMaxCount", cacheMaxSize);
diff --git a/htroot/yacy/crawlReceipt.java b/htroot/yacy/crawlReceipt.java index 22eb1043c..2fe45d3aa 100644 --- a/htroot/yacy/crawlReceipt.java +++ b/htroot/yacy/crawlReceipt.java @@ -56,7 +56,6 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.crypt;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
@@ -69,12 +68,12 @@ public final class crawlReceipt { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch<?> env) {
// return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
if ((post == null) || (env == null)) return prop;
if (!yacyNetwork.authentifyRequest(post, env)) return prop;
- serverLog log = switchboard.getLog();
+ serverLog log = sb.getLog();
//int proxyPrefetchDepth = Integer.parseInt(env.getConfig("proxyPrefetchDepth", "0"));
//int crawlingDepth = Integer.parseInt(env.getConfig("crawlingDepth", "0"));
@@ -111,10 +110,10 @@ public final class crawlReceipt { */
- final yacySeed otherPeer = yacyCore.seedDB.get(iam);
+ final yacySeed otherPeer = sb.wordIndex.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
- if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
+ if ((sb.wordIndex.seedDB.mySeed() == null) || (!(sb.wordIndex.seedDB.mySeed().hash.equals(youare)))) {
// no yacy connection / unknown peers
prop.put("delay", "3600");
return prop;
@@ -126,7 +125,7 @@ public final class crawlReceipt { return prop;
}
- if ((switchboard.isRobinsonMode()) && (!switchboard.isInMyCluster(otherPeer))) {
+ if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(otherPeer))) {
// we reject urls that are from outside our cluster
prop.put("delay", "9999");
}
@@ -147,7 +146,7 @@ public final class crawlReceipt { }
// check if the entry is in our network domain
- String urlRejectReason = switchboard.acceptURL(comp.url());
+ String urlRejectReason = sb.acceptURL(comp.url());
if (urlRejectReason != null) {
log.logWarning("crawlReceipt: RECEIVED wrong RECEIPT (" + urlRejectReason + ") for hash " + entry.hash() + " from peer " + iam + "\n\tURL properties: "+ propStr);
prop.put("delay", "9999");
@@ -156,9 +155,9 @@ public final class crawlReceipt { if (result.equals("fill")) try {
// put new entry into database
- switchboard.wordIndex.putURL(entry);
- switchboard.crawlResults.stack(entry, youare, iam, 1);
- switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
+ sb.wordIndex.putURL(entry);
+ sb.crawlResults.stack(entry, youare, iam, 1);
+ sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
log.logInfo("crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + entry.hash() + ":" + comp.url().toNormalform(false, true));
// ready for more
@@ -170,10 +169,15 @@ public final class crawlReceipt { return prop;
}
- switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
- plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason);
+ sb.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
+ plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(
+ entry.toBalancerEntry(iam),
+ youare,
+ null,
+ 0,
+ result + ":" + reason);
ee.store();
- switchboard.crawlQueues.errorURL.push(ee);
+ sb.crawlQueues.errorURL.push(ee);
//switchboard.noticeURL.remove(receivedUrlhash);
prop.put("delay", "3600");
return prop;
diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 54ca87d8e..29ee8b622 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -144,7 +144,7 @@ public final class hello { boolean isNotLocal = true;
// we are only allowed to connect to the client IP address if it's not our own address
- if (serverCore.portForwardingEnabled || serverCore.useStaticIP) {
+ if (serverCore.useStaticIP) {
isNotLocal = !ias.isSiteLocalAddress();
}
if (isNotLocal) {
@@ -194,15 +194,15 @@ public final class hello { serverCore.checkInterruption();
final StringBuffer seeds = new StringBuffer(768);
// attach some more seeds, as requested
- if ((yacyCore.seedDB != null) && (yacyCore.seedDB.sizeConnected() > 0)) {
- if (count > yacyCore.seedDB.sizeConnected()) { count = yacyCore.seedDB.sizeConnected(); }
+ if ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.sizeConnected() > 0)) {
+ if (count > sb.wordIndex.seedDB.sizeConnected()) { count = sb.wordIndex.seedDB.sizeConnected(); }
if (count > 100) { count = 100; }
// latest seeds
- final Map<String, yacySeed> ySeeds = yacyCore.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation
+ final Map<String, yacySeed> ySeeds = sb.wordIndex.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation
// attach also my own seed
- seeds.append("seed0=").append(yacyCore.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
+ seeds.append("seed0=").append(sb.wordIndex.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
count = 1;
// attach other seeds
@@ -222,7 +222,7 @@ public final class hello { }
} else {
// attach also my own seed
- seeds.append("seed0=").append(yacyCore.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
+ seeds.append("seed0=").append(sb.wordIndex.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
}
prop.put("seedlist", seeds.toString());
diff --git a/htroot/yacy/list.java b/htroot/yacy/list.java index 1d071b23c..601d0a652 100644 --- a/htroot/yacy/list.java +++ b/htroot/yacy/list.java @@ -34,7 +34,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNetwork; import de.anomic.yacy.yacySeed; @@ -55,7 +54,7 @@ public final class list { String otherPeerName = null; if (post.containsKey("iam")) { - yacySeed bla = yacyCore.seedDB.get(post.get("iam", "")); + yacySeed bla = sb.wordIndex.seedDB.get(post.get("iam", "")); if (bla != null) otherPeerName = bla.getName(); } if (otherPeerName == null) otherPeerName = (String)header.get(httpHeader.CONNECTION_PROP_CLIENTIP); diff --git a/htroot/yacy/message.java b/htroot/yacy/message.java index e7a9297fc..35dc6b340 100644 --- a/htroot/yacy/message.java +++ b/htroot/yacy/message.java @@ -92,7 +92,7 @@ public final class message { String youare = post.get("youare", ""); // seed hash of the target peer, needed for network stability
// check if we are the right target and requester has correct information about this peer
- if ((yacyCore.seedDB.mySeed() == null) || (!(yacyCore.seedDB.mySeed().hash.equals(youare)))) {
+ if ((sb.wordIndex.seedDB.mySeed() == null) || (!(sb.wordIndex.seedDB.mySeed().hash.equals(youare)))) {
// this request has a wrong target
prop.put("response", "-1"); // request rejected
return prop;
@@ -152,10 +152,10 @@ public final class message { sb.messageDB.write(msgEntry = sb.messageDB.newEntry(
"remote",
otherSeed.get(yacySeed.NAME, "anonymous"), otherSeed.hash,
- yacyCore.seedDB.mySeed().getName(), yacyCore.seedDB.mySeed().hash,
+ sb.wordIndex.seedDB.mySeed().getName(), sb.wordIndex.seedDB.mySeed().hash,
subject, mb));
- messageForwardingViaEmail(env, msgEntry);
+ messageForwardingViaEmail(sb, msgEntry);
// finally write notification
File notifierSource = new File(sb.getRootPath(), sb.getConfig("htRootPath","htroot") + "/env/grafics/message.gif");
@@ -181,15 +181,15 @@ public final class message { #[message]#
*/
- private static void messageForwardingViaEmail(serverSwitch<?> env, messageBoard.entry msgEntry) {
+ private static void messageForwardingViaEmail(plasmaSwitchboard sb, messageBoard.entry msgEntry) {
try {
- if (!Boolean.valueOf(env.getConfig("msgForwardingEnabled","false")).booleanValue()) return;
+ if (!Boolean.valueOf(sb.getConfig("msgForwardingEnabled","false")).booleanValue()) return;
// getting the recipient address
- String sendMailTo = env.getConfig("msgForwardingTo","root@localhost").trim();
+ String sendMailTo = sb.getConfig("msgForwardingTo","root@localhost").trim();
// getting the sendmail configuration
- String sendMailStr = env.getConfig("msgForwardingCmd","/usr/bin/sendmail")+" "+sendMailTo;
+ String sendMailStr = sb.getConfig("msgForwardingCmd","/usr/bin/sendmail")+" "+sendMailTo;
String[] sendMail = sendMailStr.trim().split(" ");
// building the message text
@@ -198,7 +198,7 @@ public final class message { .append(sendMailTo)
.append("\nFrom: ")
.append("yacy@")
- .append(yacyCore.seedDB.mySeed().getName())
+ .append(sb.wordIndex.seedDB.mySeed().getName())
.append("\nSubject: [YaCy] ")
.append(msgEntry.subject().replace('\n', ' '))
.append("\nDate: ")
diff --git a/htroot/yacy/query.java b/htroot/yacy/query.java index 2872e1af5..82a63ec6f 100644 --- a/htroot/yacy/query.java +++ b/htroot/yacy/query.java @@ -51,7 +51,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverDate;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNetwork;
public final class query {
@@ -88,7 +87,7 @@ public final class query { prop.put("mytime", serverDate.formatShortSecond());
// check if we are the right target and requester has correct information about this peer
- if (yacyCore.seedDB.mySeed() == null || !yacyCore.seedDB.mySeed().hash.equals(youare)) {
+ if (sb.wordIndex.seedDB.mySeed() == null || !sb.wordIndex.seedDB.mySeed().hash.equals(youare)) {
// this request has a wrong target
prop.put("response", "-1"); // request rejected
return prop;
diff --git a/htroot/yacy/search.java b/htroot/yacy/search.java index 8418250c5..be97e7146 100644 --- a/htroot/yacy/search.java +++ b/htroot/yacy/search.java @@ -128,7 +128,7 @@ public final class search { // store accessing peer
yacySeed remoteSeed = yacySeed.genRemoteSeed(oseed, key, true);
- if (yacyCore.seedDB == null) {
+ if (sb.wordIndex.seedDB == null) {
yacyCore.log.logSevere("yacy.search: seed cache not initialized");
} else {
yacyCore.peerActions.peerArrival(remoteSeed, true);
@@ -285,7 +285,7 @@ public final class search { prop.put("fwrec", ""); // peers that would have helped to construct this result (recommendations)
// prepare search statistics
- theQuery.remotepeer = yacyCore.seedDB.lookupByIP(natLib.getInetAddress(client), true, false, false);
+ theQuery.remotepeer = sb.wordIndex.seedDB.lookupByIP(natLib.getInetAddress(client), true, false, false);
theQuery.resultcount = (theSearch == null) ? 0 : theSearch.getRankingResult().getLocalResourceSize() + theSearch.getRankingResult().getRemoteResourceSize();
theQuery.searchtime = System.currentTimeMillis() - timestamp;
theQuery.urlretrievaltime = (theSearch == null) ? 0 : theSearch.getURLRetrievalTime();
@@ -306,8 +306,8 @@ public final class search { prop.put("searchtime", System.currentTimeMillis() - timestamp);
final int links = Integer.parseInt(prop.get("linkcount","0"));
- yacyCore.seedDB.mySeed().incSI(links);
- yacyCore.seedDB.mySeed().incSU(links);
+ sb.wordIndex.seedDB.mySeed().incSI(links);
+ sb.wordIndex.seedDB.mySeed().incSU(links);
return prop;
}
diff --git a/htroot/yacy/transfer.java b/htroot/yacy/transfer.java index 23f79df15..bcfd077f5 100644 --- a/htroot/yacy/transfer.java +++ b/htroot/yacy/transfer.java @@ -54,7 +54,6 @@ import de.anomic.server.serverCodings; import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNetwork;
import de.anomic.yacy.yacySeed;
@@ -86,7 +85,7 @@ public final class transfer { return prop;
}
- yacySeed otherseed = yacyCore.seedDB.get(otherpeer);
+ yacySeed otherseed = sb.wordIndex.seedDB.get(otherpeer);
if ((otherseed == null) || (filename.indexOf("..") >= 0)) {
// reject unknown peers: this does not appear fair, but anonymous senders are dangerous
// reject paths that contain '..' because they are dangerous
@@ -106,7 +105,7 @@ public final class transfer { String access = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(otherpeer + ":" + filename)) + ":" + kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw("" + System.currentTimeMillis()));
prop.put("response", "ok");
prop.put("process_access", access);
- prop.put("process_address", yacyCore.seedDB.mySeed().getPublicAddress());
+ prop.put("process_address", sb.wordIndex.seedDB.mySeed().getPublicAddress());
prop.put("process_protocol", "http");
prop.put("process_path", ""); // currently empty; the store process will find a path
prop.put("process_maxsize", "-1"); // if response is too big we return the size of the file
diff --git a/htroot/yacy/transferRWI.java b/htroot/yacy/transferRWI.java index 93abde3b2..297637c7e 100644 --- a/htroot/yacy/transferRWI.java +++ b/htroot/yacy/transferRWI.java @@ -86,7 +86,7 @@ public final class transferRWI { boolean blockBlacklist = sb.getConfig("indexReceiveBlockBlacklist", "false").equals("true");
boolean checkLimit = sb.getConfigBool("indexDistribution.transferRWIReceiptLimitEnabled", true);
final long cachelimit = sb.getConfigLong("indexDistribution.dhtReceiptLimit", 10000);
- final yacySeed otherPeer = yacyCore.seedDB.get(iam);
+ final yacySeed otherPeer = sb.wordIndex.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
// response values
@@ -94,8 +94,8 @@ public final class transferRWI { StringBuffer unknownURLs = new StringBuffer();
int pause = 10000;
- if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
- sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
+ if ((youare == null) || (!youare.equals(sb.wordIndex.seedDB.mySeed().hash))) {
+ sb.getLog().logInfo("Rejecting RWIs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.wordIndex.seedDB.mySeed().hash);
result = "wrong_target";
pause = 0;
} else if ((!granted) || (sb.isRobinsonMode())) {
@@ -184,7 +184,7 @@ public final class transferRWI { }
received++;
}
- yacyCore.seedDB.mySeed().incRI(received);
+ sb.wordIndex.seedDB.mySeed().incRI(received);
// finally compose the unknownURL hash list
final Iterator<String> it = unknownURL.iterator();
@@ -196,7 +196,7 @@ public final class transferRWI { if ((wordhashes.length == 0) || (received == 0)) {
sb.getLog().logInfo("Received 0 RWIs from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + " URLs, blocked " + blocked + " RWIs");
} else {
- final double avdist = (yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0;
+ final double avdist = (yacyDHTAction.dhtDistance(sb.wordIndex.seedDB.mySeed().hash, wordhashes[0]) + yacyDHTAction.dhtDistance(sb.wordIndex.seedDB.mySeed().hash, wordhashes[received - 1])) / 2.0;
sb.getLog().logInfo("Received " + received + " Entries " + wordc + " Words [" + wordhashes[0] + " .. " + wordhashes[received - 1] + "]/" + avdist + " from " + otherPeerName + ", processed in " + (System.currentTimeMillis() - startProcess) + " milliseconds, requesting " + unknownURL.size() + "/" + receivedURL + " URLs, blocked " + blocked + " RWIs");
}
result = "ok";
diff --git a/htroot/yacy/transferURL.java b/htroot/yacy/transferURL.java index 8518aaa02..d88f6303e 100644 --- a/htroot/yacy/transferURL.java +++ b/htroot/yacy/transferURL.java @@ -86,11 +86,11 @@ public final class transferURL { String result = "";
String doublevalues = "0";
- final yacySeed otherPeer = yacyCore.seedDB.get(iam);
+ final yacySeed otherPeer = sb.wordIndex.seedDB.get(iam);
final String otherPeerName = iam + ":" + ((otherPeer == null) ? "NULL" : (otherPeer.getName() + "/" + otherPeer.getVersion()));
- if ((youare == null) || (!youare.equals(yacyCore.seedDB.mySeed().hash))) {
- sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + yacyCore.seedDB.mySeed().hash);
+ if ((youare == null) || (!youare.equals(sb.wordIndex.seedDB.mySeed().hash))) {
+ sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Wrong target. Wanted peer=" + youare + ", iam=" + sb.wordIndex.seedDB.mySeed().hash);
result = "wrong_target";
} else if ((!granted) || (sb.isRobinsonMode())) {
sb.getLog().logInfo("Rejecting URLs from peer " + otherPeerName + ". Not granted.");
@@ -164,7 +164,7 @@ public final class transferURL { }
}
- yacyCore.seedDB.mySeed().incRU(received);
+ sb.wordIndex.seedDB.mySeed().incRU(received);
// return rewrite properties
final int more = sb.wordIndex.countURL() - sizeBefore;
diff --git a/htroot/yacy/ui/result.java b/htroot/yacy/ui/result.java index 38601b547..e76facafc 100644 --- a/htroot/yacy/ui/result.java +++ b/htroot/yacy/ui/result.java @@ -40,7 +40,6 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.logging.serverLog; import de.anomic.tools.yFormatter; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyURL; public class result { @@ -161,7 +160,7 @@ public class result { } // prepare search properties - final boolean yacyonline = ((yacyCore.seedDB != null) && (yacyCore.seedDB.mySeed() != null) && (yacyCore.seedDB.mySeed().getPublicAddress() != null)); + final boolean yacyonline = ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.mySeed() != null) && (sb.wordIndex.seedDB.mySeed().getPublicAddress() != null)); final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false)); // do the search diff --git a/htroot/yacy/ui/ymarks.java b/htroot/yacy/ui/ymarks.java index 6f56afc1c..b58abd4fb 100644 --- a/htroot/yacy/ui/ymarks.java +++ b/htroot/yacy/ui/ymarks.java @@ -69,7 +69,6 @@ import de.anomic.server.serverDate; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; import de.anomic.server.logging.serverLog; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacyNewsRecord; import de.anomic.yacy.yacyURL; @@ -78,7 +77,6 @@ import de.anomic.yacy.yacyURL; public class ymarks { private static final serverObjects prop = new serverObjects(); - private static plasmaSwitchboard switchboard = null; private static userDB.Entry user = null; private static boolean isAdmin = false; @@ -96,9 +94,9 @@ public class ymarks { String username=""; prop.clear(); - switchboard = (plasmaSwitchboard) env; - user = switchboard.userDB.getUser(header); - isAdmin=(switchboard.verifyAuthentication(header, true) || user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT)); + plasmaSwitchboard sb = (plasmaSwitchboard) env; + user = sb.userDB.getUser(header); + isAdmin=(sb.verifyAuthentication(header, true) || user!= null && user.hasRight(userDB.Entry.BOOKMARK_RIGHT)); // set user name if(user != null) username=user.getUserName(); @@ -112,7 +110,7 @@ public class ymarks { */ // set peer address - final String address = yacyCore.seedDB.mySeed().getPublicAddress(); + final String address = sb.wordIndex.seedDB.mySeed().getPublicAddress(); prop.put("address", address); //defaultvalues @@ -157,7 +155,7 @@ public class ymarks { tagsString="unsorted"; //default tag } Set<String> tags=listManager.string2set(bookmarksDB.cleanTagsString(tagsString)); - bookmarksDB.Bookmark bookmark = switchboard.bookmarksDB.createBookmark(url, username); + bookmarksDB.Bookmark bookmark = sb.bookmarksDB.createBookmark(url, username); if(bookmark != null){ bookmark.setProperty(bookmarksDB.Bookmark.BOOKMARK_TITLE, title); bookmark.setProperty(bookmarksDB.Bookmark.BOOKMARK_DESCRIPTION, description); @@ -166,7 +164,7 @@ public class ymarks { } if(((String) post.get("public")).equals("public")){ bookmark.setPublic(true); - publishNews(url, title, description, tagsString); + publishNews(sb, url, title, description, tagsString); }else{ bookmark.setPublic(false); } @@ -176,7 +174,7 @@ public class ymarks { bookmark.setFeed(false); } bookmark.setTags(tags, true); - switchboard.bookmarksDB.saveBookmark(bookmark); + sb.bookmarksDB.saveBookmark(bookmark); //}else{ //ERROR } @@ -193,10 +191,10 @@ public class ymarks { prop.put("mode_public", "0"); prop.put("mode_feed", "0"); } else { - bookmarksDB.Bookmark bookmark = switchboard.bookmarksDB.getBookmark(urlHash); + bookmarksDB.Bookmark bookmark = sb.bookmarksDB.getBookmark(urlHash); if (bookmark == null) { // try to get the bookmark from the LURL database - indexURLReference urlentry = switchboard.wordIndex.getURL(urlHash, null, 0); + indexURLReference urlentry = sb.wordIndex.getURL(urlHash, null, 0); plasmaParserDocument document = null; if (urlentry != null) { indexURLReference.Components comp = urlentry.comp(); @@ -244,7 +242,7 @@ public class ymarks { serverLog.logInfo("BOOKMARKS", "I try to import bookmarks from HTML-file"); try { File file=new File((String)post.get("htmlfile")); - switchboard.bookmarksDB.importFromBookmarks(new yacyURL(file) , post.get("htmlfile$file"), tags, isPublic); + sb.bookmarksDB.importFromBookmarks(new yacyURL(file) , post.get("htmlfile$file"), tags, isPublic); } catch (MalformedURLException e) {} serverLog.logInfo("BOOKMARKS", "success!!"); }else if(post.containsKey("xmlfile")){ @@ -252,10 +250,10 @@ public class ymarks { if(((String) post.get("public")).equals("public")){ isPublic=true; } - switchboard.bookmarksDB.importFromXML(post.get("xmlfile$file"), isPublic); + sb.bookmarksDB.importFromXML(post.get("xmlfile$file"), isPublic); }else if(post.containsKey("delete")){ String urlHash=(String) post.get("delete"); - switchboard.bookmarksDB.removeBookmark(urlHash); + sb.bookmarksDB.removeBookmark(urlHash); } if(post.containsKey("tag")){ tagName=(String) post.get("tag"); @@ -272,8 +270,8 @@ public class ymarks { //----------------------- // create tag list //----------------------- - printTagList("taglist", tagName, SORT_SIZE, 25, false); - printTagList("optlist", tagName, SORT_ALPHA, SHOW_ALL, true); + printTagList(sb, "taglist", tagName, SORT_SIZE, 25, false); + printTagList(sb, "optlist", tagName, SORT_ALPHA, SHOW_ALL, true); //----------------------- // create bookmark list @@ -285,15 +283,15 @@ public class ymarks { Iterator<String> tagsIt; int tagCount; - prop.put("num-bookmarks", switchboard.bookmarksDB.bookmarksSize()); + prop.put("num-bookmarks", sb.bookmarksDB.bookmarksSize()); count=0; if(!tagName.equals("")){ prop.put("selected", ""); - it=switchboard.bookmarksDB.getBookmarksIterator(tagName, isAdmin); + it=sb.bookmarksDB.getBookmarksIterator(tagName, isAdmin); }else{ prop.put("selected", " selected=\"selected\""); - it=switchboard.bookmarksDB.getBookmarksIterator(isAdmin); + it=sb.bookmarksDB.getBookmarksIterator(isAdmin); } //skip the first entries (display next page) @@ -305,7 +303,7 @@ public class ymarks { count=0; while(count<max_count && it.hasNext()){ - bookmark=switchboard.bookmarksDB.getBookmark((String)it.next()); + bookmark=sb.bookmarksDB.getBookmark((String)it.next()); if(bookmark!=null){ if(bookmark.getFeed() && isAdmin) prop.put("bookmarks_"+count+"_link", "/FeedReader_p.html?url="+bookmark.getUrl()); @@ -359,22 +357,22 @@ public class ymarks { //----------------------- count = 0; - count = recurseFolders(switchboard.bookmarksDB.getFolderList(isAdmin),"/",0,true,""); + count = recurseFolders(sb, sb.bookmarksDB.getFolderList(isAdmin),"/",0,true,""); prop.put("folderlist", count); return prop; // return from serverObjects respond() } - private static void printTagList(String id, String tagName, int comp, int max, boolean opt){ + private static void printTagList(plasmaSwitchboard sb, String id, String tagName, int comp, int max, boolean opt){ int count=0; bookmarksDB.Tag tag; Iterator<Tag> it = null; if (tagName.equals("")) { - it = switchboard.bookmarksDB.getTagIterator(isAdmin, comp, max); + it = sb.bookmarksDB.getTagIterator(isAdmin, comp, max); } else { - it = switchboard.bookmarksDB.getTagIterator(tagName, isAdmin, comp, max); + it = sb.bookmarksDB.getTagIterator(tagName, isAdmin, comp, max); } while(it.hasNext()){ tag=(Tag) it.next(); @@ -398,7 +396,7 @@ public class ymarks { prop.put(id, count); } - private static int recurseFolders(Iterator<String> it, String root, int count, boolean next, String prev){ + private static int recurseFolders(plasmaSwitchboard sb, Iterator<String> it, String root, int count, boolean next, String prev){ String fn=""; bookmarksDB.Bookmark bookmark; @@ -418,33 +416,33 @@ public class ymarks { if(fn.startsWith(root)){ prop.put("folderlist_"+count+"_folder", "<li>"+fn.replaceFirst(root+"/*","")+"<ul class=\"folder\">"); count++; - Iterator<String> bit=switchboard.bookmarksDB.getBookmarksIterator(fn, isAdmin); + Iterator<String> bit=sb.bookmarksDB.getBookmarksIterator(fn, isAdmin); while(bit.hasNext()){ - bookmark=switchboard.bookmarksDB.getBookmark((String)bit.next()); + bookmark=sb.bookmarksDB.getBookmark((String)bit.next()); prop.put("folderlist_"+count+"_folder", "<li><a href=\""+bookmark.getUrl()+"\" title=\""+bookmark.getDescription()+"\">"+ bookmark.getTitle()+"</a></li>"); count++; } if(it.hasNext()){ - count = recurseFolders(it, fn, count, true, fn); + count = recurseFolders(sb, it, fn, count, true, fn); } } else { prop.put("folderlist_"+count+"_folder", "</ul></li>"); count++; root = root.replaceAll("(/.[^/]*$)", ""); if(root.equals("")) root = "/"; - count = recurseFolders(it, root, count, false, fn); + count = recurseFolders(sb, it, root, count, false, fn); } return count; } - private static void publishNews(String url, String title, String description, String tagsString) { + private static void publishNews(plasmaSwitchboard sb, String url, String title, String description, String tagsString) { // create a news message HashMap<String, String> map = new HashMap<String, String>(); map.put("url", url.replace(',', '|')); map.put("title", title.replace(',', ' ')); map.put("description", description.replace(',', ' ')); map.put("tags", tagsString.replace(',', ' ')); - yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); + sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_BOOKMARK_ADD, map)); } } diff --git a/htroot/yacy/urls.java b/htroot/yacy/urls.java index 1ccf75b3e..6778ea9c9 100644 --- a/htroot/yacy/urls.java +++ b/htroot/yacy/urls.java @@ -25,6 +25,7 @@ // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA import java.io.IOException; +import java.util.Date; import de.anomic.http.httpHeader; import de.anomic.index.indexURLReference; @@ -34,7 +35,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverDate; import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNetwork; import de.anomic.yacy.yacyURL; @@ -45,7 +45,7 @@ public class urls { // insert default values serverObjects prop = new serverObjects(); - prop.put("iam", yacyCore.seedDB.mySeed().hash); + prop.put("iam", sb.wordIndex.seedDB.mySeed().hash); prop.put("response", "rejected - insufficient call parameters"); prop.put("channel_title", ""); prop.put("channel_description", ""); @@ -69,10 +69,20 @@ public class urls { break; } if (entry == null) break; + // find referrer, if there is one referrer = sb.getURL(entry.referrerhash()); + // place url to notice-url db - sb.crawlQueues.delegatedURL.push(sb.crawlQueues.delegatedURL.newEntry(entry.url(), "client=____________")); + sb.crawlQueues.delegatedURL.push( + sb.crawlQueues.delegatedURL.newEntry( + entry, + sb.wordIndex.seedDB.mySeed().hash, + new Date(), + 0, + "client=____________") + ); + // create RSS entry prop.put("item_" + c + "_title", ""); prop.putHTML("item_" + c + "_link", entry.url().toNormalform(true, false)); diff --git a/htroot/yacy/user/ysearch.java b/htroot/yacy/user/ysearch.java index 21c414d04..fac2fb88e 100644 --- a/htroot/yacy/user/ysearch.java +++ b/htroot/yacy/user/ysearch.java @@ -40,7 +40,6 @@ import de.anomic.server.serverObjects; import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.yFormatter;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyURL;
public class ysearch {
@@ -161,7 +160,7 @@ public class ysearch { }
// prepare search properties
- final boolean yacyonline = ((yacyCore.seedDB != null) && (yacyCore.seedDB.mySeed() != null) && (yacyCore.seedDB.mySeed().getPublicAddress() != null));
+ final boolean yacyonline = ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.mySeed() != null) && (sb.wordIndex.seedDB.mySeed().getPublicAddress() != null));
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
// do the search
diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index 108f392f2..952c1b136 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -68,7 +68,6 @@ import de.anomic.server.serverProfiling; import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.tools.yFormatter;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
import de.anomic.yacy.yacyURL;
@@ -208,7 +207,7 @@ public class yacysearch { map.put("urlhash", delHash);
map.put("vote", "negative");
map.put("refid", "");
- yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
+ sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_VOTE_ADD, map));
}
// if a plus-button was hit, create new voting message
@@ -231,14 +230,14 @@ public class yacysearch { map.put("description", ((document == null) ? comp.dc_title() : document.dc_title()).replace(',', ' '));
map.put("author", ((document == null) ? "" : document.dc_creator()));
map.put("tags", ((document == null) ? "" : document.dc_subject(' ')));
- yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_SURFTIPP_ADD, map));
+ sb.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(sb.wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_SURFTIPP_ADD, map));
document.close();
}
}
}
// prepare search properties
- final boolean yacyonline = ((yacyCore.seedDB != null) && (yacyCore.seedDB.mySeed() != null) && (yacyCore.seedDB.mySeed().getPublicAddress() != null));
+ final boolean yacyonline = ((sb.wordIndex.seedDB != null) && (sb.wordIndex.seedDB.mySeed() != null) && (sb.wordIndex.seedDB.mySeed().getPublicAddress() != null));
final boolean globalsearch = (global) && (yacyonline) && (sb.getConfigBool(plasmaSwitchboard.INDEX_RECEIVE_ALLOW, false));
// do the search
diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index 144b95d91..b2e39bcdf 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -47,7 +47,6 @@ import de.anomic.server.serverSwitch; import de.anomic.tools.crypt; import de.anomic.tools.nxTools; import de.anomic.tools.yFormatter; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyNewsPool; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyURL; @@ -192,7 +191,7 @@ public class yacysearchitem { prop.put("content", theQuery.contentdom + 1); // switch on specific content prop.put("content_authorized", authenticated ? "1" : "0"); - prop.put("content_authorized_recommend", (yacyCore.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? "1" : "0"); + prop.put("content_authorized_recommend", (sb.wordIndex.newsPool.getSpecific(yacyNewsPool.OUTGOING_DB, yacyNewsPool.CATEGORY_SURFTIPP_ADD, "url", result.urlstring()) == null) ? "1" : "0"); prop.put("content_authorized_recommend_deletelink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&deleteref=" + result.hash() + "&urlmaskfilter=.*"); prop.put("content_authorized_recommend_recommendlink", "/yacysearch.html?search=" + theQuery.queryString + "&Enter=Search&count=" + theQuery.displayResults() + "&offset=" + (theQuery.neededResults() - theQuery.displayResults()) + "&order=" + crypt.simpleEncode(theQuery.ranking.toExternalString()) + "&resource=local&time=3&recommendref=" + result.hash() + "&urlmaskfilter=.*"); prop.put("content_authorized_urlhash", result.hash()); diff --git a/source/de/anomic/data/SitemapParser.java b/source/de/anomic/data/SitemapParser.java index 724db0a07..638606966 100644 --- a/source/de/anomic/data/SitemapParser.java +++ b/source/de/anomic/data/SitemapParser.java @@ -62,12 +62,12 @@ import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.JakartaCommonsHttpResponse; import de.anomic.http.httpdByteCountInputStream; import de.anomic.index.indexURLReference; +import de.anomic.plasma.plasmaCrawlEntry; import de.anomic.plasma.plasmaCrawlProfile; import de.anomic.plasma.plasmaCrawlZURL; import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverDate; import de.anomic.server.logging.serverLog; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacyURL; /** @@ -298,7 +298,7 @@ public class SitemapParser extends DefaultHandler { String error = null; error = this.switchboard.crawlStacker.stackCrawl(url, null, // this.siteMapURL.toString(), - yacyCore.seedDB.mySeed().hash, this.nextURL, new Date(), + this.switchboard.wordIndex.seedDB.mySeed().hash, this.nextURL, new Date(), 0, this.crawlingProfile); if (error != null) { @@ -306,8 +306,21 @@ public class SitemapParser extends DefaultHandler { this.logger.logInfo("The URL '" + this.nextURL + "' can not be crawled. Reason: " + error); // insert URL into the error DB - plasmaCrawlZURL.Entry ee = this.switchboard.crawlQueues.errorURL.newEntry(new yacyURL(this.nextURL, - null), error); + plasmaCrawlZURL.Entry ee = this.switchboard.crawlQueues.errorURL.newEntry( + new plasmaCrawlEntry( + switchboard.wordIndex.seedDB.mySeed().hash, + new yacyURL(this.nextURL, null), + "", + "", + new Date(), + null, + 0, + 0, + 0), + this.switchboard.wordIndex.seedDB.mySeed().hash, + new Date(), + 1, + error); ee.store(); this.switchboard.crawlQueues.errorURL.push(ee); } catch (MalformedURLException e) {/* ignore this */ diff --git a/source/de/anomic/data/wiki/abstractWikiParser.java b/source/de/anomic/data/wiki/abstractWikiParser.java index 6d7ccc560..965ab91b2 100644 --- a/source/de/anomic/data/wiki/abstractWikiParser.java +++ b/source/de/anomic/data/wiki/abstractWikiParser.java @@ -8,7 +8,6 @@ import java.io.StringReader; import java.io.UnsupportedEncodingException; import de.anomic.plasma.plasmaSwitchboard; -import de.anomic.yacy.yacyCore; public abstract class abstractWikiParser implements wikiParser { @@ -29,7 +28,7 @@ public abstract class abstractWikiParser implements wikiParser { return transform( new BufferedReader(new StringReader(content)), content.length(), - yacyCore.seedDB.mySeed().getPublicAddress(), + sb.wordIndex.seedDB.mySeed().getPublicAddress(), sb); } catch (IOException e) { return "internal error: " + e.getMessage(); @@ -75,7 +74,7 @@ public abstract class abstractWikiParser implements wikiParser { return transform( new BufferedReader(new InputStreamReader(bais, encoding)), content.length, - yacyCore.seedDB.mySeed().getPublicAddress(), + sb.wordIndex.seedDB.mySeed().getPublicAddress(), switchboard); } catch (IOException e) { return "internal error: " + e.getMessage(); diff --git a/source/de/anomic/data/wiki/knwikiParser.java b/source/de/anomic/data/wiki/knwikiParser.java index ad89f6c6e..71e7ecb91 100644 --- a/source/de/anomic/data/wiki/knwikiParser.java +++ b/source/de/anomic/data/wiki/knwikiParser.java @@ -58,7 +58,6 @@ import de.anomic.data.wiki.tokens.SimpleToken; import de.anomic.data.wiki.tokens.TableToken; import de.anomic.data.wiki.tokens.Token; import de.anomic.plasma.plasmaSwitchboard; -import de.anomic.yacy.yacyCore; public class knwikiParser implements wikiParser { @@ -146,7 +145,7 @@ public class knwikiParser implements wikiParser { tokens = new Token[] { new SimpleToken('=', '=', new String[][] { null, { "h2" }, { "h3" }, { "h4" } }, true), new SimpleToken('\'', '\'', new String[][] { null, { "i" }, { "b" }, null, { "b", "i" } }, false), - new LinkToken((publicAddress == null) ? yacyCore.seedDB.mySeed().getPublicAddress() : publicAddress, "Wiki.html?page=", sb), + new LinkToken((publicAddress == null) ? sb.wordIndex.seedDB.mySeed().getPublicAddress() : publicAddress, "Wiki.html?page=", sb), new ListToken('*', "ul"), new ListToken('#', "ol"), new ListToken(':', "blockquote", null), diff --git a/source/de/anomic/http/httpd.java b/source/de/anomic/http/httpd.java index 1540ca4b8..289b888be 100644 --- a/source/de/anomic/http/httpd.java +++ b/source/de/anomic/http/httpd.java @@ -79,8 +79,6 @@ import de.anomic.server.serverHandler; import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
-import de.anomic.yacy.yacyCore;
-import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyURL;
@@ -102,6 +100,7 @@ public final class httpd implements serverHandler { public static final int ERRORCASE_MESSAGE = 4;
public static final int ERRORCASE_FILE = 5;
+ public static httpdAlternativeDomainNames alternativeResolver = null;
/**
* A hashset containing extensions that indicate content that should not be transported
@@ -363,8 +362,10 @@ public final class httpd implements serverHandler { // user = addressed peer name
// pw = addressed peer hash (b64-hash)
String auth = (String) header.get(httpHeader.PROXY_AUTHORIZATION,"xxxxxx");
- String test = kelondroBase64Order.standardCoder.encodeString(yacyCore.seedDB.mySeed().getName() + ":" + yacyCore.seedDB.mySeed().hash);
- if (!test.equals(auth.trim().substring(6))) return false;
+ if (alternativeResolver != null) {
+ String test = kelondroBase64Order.standardCoder.encodeString(alternativeResolver.myName() + ":" + alternativeResolver.myID());
+ if (!test.equals(auth.trim().substring(6))) return false;
+ }
// the accessing client must use a yacy user-agent
if (!(((String) header.get(httpHeader.USER_AGENT,"")).startsWith("yacy"))) return false;
@@ -1184,21 +1185,19 @@ public final class httpd implements serverHandler { tp.put("port", serverCore.getPortNr(switchboard.getConfig("port", "8080")));
} else {
tp.put("host", serverDomains.myPublicIP());
- tp.put("port", (serverCore.portForwardingEnabled && (serverCore.portForwarding != null))
- ? Integer.toString(serverCore.portForwarding.getPort())
- : Integer.toString(serverCore.getPortNr(switchboard.getConfig("port", "8080"))));
+ tp.put("port", Integer.toString(serverCore.getPortNr(switchboard.getConfig("port", "8080"))));
}
// if peer has public address it will be used
- if (yacyCore.seedDB.mySeed().getPublicAddress() != null) {
- tp.put("extAddress", yacyCore.seedDB.mySeed().getPublicAddress());
+ if (alternativeResolver != null) {
+ tp.put("extAddress", alternativeResolver.myIP() + ":" + alternativeResolver.myPort());
}
// otherwise the local ip address will be used
else {
tp.put("extAddress", tp.get("host", "127.0.0.1") + ":" + tp.get("port", "8080"));
}
- tp.put("peerName", yacyCore.seedDB.mySeed().getName());
+ tp.put("peerName", (alternativeResolver == null) ? "" : alternativeResolver.myName());
tp.put("errorMessageType", errorcase);
tp.put("httpStatus", Integer.toString(httpStatusCode) + " " + httpStatusText);
tp.put("requestMethod", conProp.getProperty(httpHeader.CONNECTION_PROP_METHOD));
@@ -1496,34 +1495,14 @@ public final class httpd implements serverHandler { // httpHeader.CONNECTION_PROP_PROXY_RESPOND_STATUS
}
- public static boolean isThisHostPortForwardingIP(String hostName) {
- if ((hostName == null) || (hostName.length() == 0)) return false;
- if ((!serverCore.portForwardingEnabled) || (serverCore.portForwarding == null)) return false;
-
- boolean isThisHostIP = false;
- try {
- //InetAddress hostAddress = InetAddress.getByName(hostName);
- InetAddress hostAddress = serverDomains.dnsResolve(hostName);
- //InetAddress forwardingAddress = InetAddress.getByName(serverCore.portForwarding.getHost());
- InetAddress forwardingAddress = serverDomains.dnsResolve(serverCore.portForwarding.getHost());
-
- if ((hostAddress==null)||(forwardingAddress==null)) return false;
- if (hostAddress.equals(forwardingAddress)) return true;
- } catch (Exception e) {}
- return isThisHostIP;
- }
-
public static boolean isThisSeedIP(String hostName) {
if ((hostName == null) || (hostName.length() == 0)) return false;
// getting ip address and port of this seed
- yacySeed thisSeed = yacyCore.seedDB.mySeed();
- String thisSeedIP = thisSeed.get(yacySeed.IP,null);
- String thisSeedPort = thisSeed.get(yacySeed.PORT,null);
+ if (alternativeResolver == null) return false;
// resolve ip addresses
- if (thisSeedIP == null || thisSeedPort == null) return false;
- InetAddress seedInetAddress = serverDomains.dnsResolve(thisSeedIP);
+ InetAddress seedInetAddress = serverDomains.dnsResolve(alternativeResolver.myIP());
InetAddress hostInetAddress = serverDomains.dnsResolve(hostName);
if (seedInetAddress == null || hostInetAddress == null) return false;
@@ -1580,7 +1559,8 @@ public final class httpd implements serverHandler { final Integer dstPort = (idx != -1) ? Integer.valueOf(hostName.substring(idx+1).trim()) : new Integer(80);
// if the hostname endswith thisPeerName.yacy ...
- if (dstHost.endsWith(yacyCore.seedDB.mySeed().getName() + ".yacy")) {
+ String alternativeAddress = (alternativeResolver == null) ? null : alternativeResolver.myAlternativeAddress();
+ if ((alternativeAddress != null) && (dstHost.endsWith(alternativeAddress))) {
return true;
/*
* If the port number is equal to the yacy port and the IP address is an address of this host ...
@@ -1597,11 +1577,6 @@ public final class httpd implements serverHandler { )
) {
return true;
- } else if ((serverCore.portForwardingEnabled) &&
- (serverCore.portForwarding != null) &&
- (dstPort.intValue() == serverCore.portForwarding.getPort()) &&
- isThisHostPortForwardingIP(dstHost)) {
- return true;
}
} catch (Exception e) {}
return false;
diff --git a/source/de/anomic/http/httpdAlternativeDomainNames.java b/source/de/anomic/http/httpdAlternativeDomainNames.java new file mode 100644 index 000000000..9324511ad --- /dev/null +++ b/source/de/anomic/http/httpdAlternativeDomainNames.java @@ -0,0 +1,68 @@ +// httpdAlternativeDomainNames.java +// (C) 2008 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 05.05.2008 on http://yacy.net +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.http; + +public interface httpdAlternativeDomainNames { + + /** + * for a given domain name, return a new address + * the new address may also be a combination of a standard domain name or an IP with ':' and a port number + * @param name: a domain name + * @return an alternative name + */ + public String resolve(String name); + + /** + * while other servers may have alternative addresses, this server may also have an alternative + * address + * @return the alternative address of this server which other servers may can resolve + */ + public String myAlternativeAddress(); + + /** + * return the IP as string of my server address + * @return IP as string of this server + */ + public String myIP(); + + /** + * return the port of my server address + * @return port number of this server + */ + public int myPort(); + + /** + * return a name of this server. this may be any string and there is no need that it must be unique + * @return + */ + public String myName(); + + /** + * return an unique ID of this server + * @return + */ + public String myID(); + +} diff --git a/source/de/anomic/http/httpdFileHandler.java b/source/de/anomic/http/httpdFileHandler.java index b8b9e6e83..05d3cf288 100644 --- a/source/de/anomic/http/httpdFileHandler.java +++ b/source/de/anomic/http/httpdFileHandler.java @@ -83,7 +83,6 @@ import java.io.FileInputStream; import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
-import java.io.PushbackInputStream;
import java.io.UnsupportedEncodingException;
import java.lang.ref.SoftReference;
import java.lang.reflect.InvocationTargetException;
diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 59eb4f470..04edbb003 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -1,14 +1,13 @@ // httpdProxyHandler.java
-// -----------------------
-// part of YACY
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2004
+// (C) 2004 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// first published 2004 on http://yacy.net
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
+// LICENSE
+//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@@ -22,25 +21,6 @@ // You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
// Contributions:
// [AS] Alexander Schier: Blacklist (404 response for AGIS hosts)
@@ -53,7 +33,7 @@ or a host is given in the header host field of an HTTP/1.0 / HTTP/1.1
command.
Transparency is maintained, whenever appropriate. We change header
- atributes if necessary for the indexing mechanism; i.e. we do not
+ attributes if necessary for the indexing mechanism; i.e. we do not
support gzip-ed encoding. We also do not support unrealistic
'expires' values that would force a cache to be flushed immediately
pragma non-cache attributes are supported
@@ -507,7 +487,7 @@ public final class httpdProxyHandler { }
// resolve yacy and yacyh domains
- String yAddress = yacyCore.seedDB.resolveYacyAddress(host);
+ String yAddress = (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.resolve(host);
// re-calc the url path
String remotePath = (args == null) ? path : (path + "?" + args); // with leading '/'
@@ -975,7 +955,7 @@ public final class httpdProxyHandler { return;
}
- // set another userAgent, if not yellowlisted
+ // set another userAgent, if not yellow-listed
if (!(yellowList.contains(domain(hostlow)))) {
// change the User-Agent
requestHeader.put(httpHeader.USER_AGENT, generateUserAgent(requestHeader));
@@ -984,7 +964,7 @@ public final class httpdProxyHandler { addXForwardedForHeader(conProp, requestHeader);
// resolve yacy and yacyh domains
- String yAddress = yacyCore.seedDB.resolveYacyAddress(host);
+ String yAddress = (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.resolve(host);
// attach possible yacy-sublevel-domain
if ((yAddress != null) && ((pos = yAddress.indexOf("/")) >= 0)) remotePath = yAddress.substring(pos) + remotePath;
@@ -1084,7 +1064,7 @@ public final class httpdProxyHandler { addXForwardedForHeader(conProp, requestHeader);
// resolve yacy and yacyh domains
- String yAddress = yacyCore.seedDB.resolveYacyAddress(host);
+ String yAddress = (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.resolve(host);
// re-calc the url path
String remotePath = (args == null) ? path : (path + "?" + args);
@@ -1658,20 +1638,23 @@ public final class httpdProxyHandler { private static void setViaHeader(httpHeader header, String httpVer) {
if (!switchboard.getConfigBool("proxy.sendViaHeader", true)) return;
-
- // getting header set by other proxies in the chain
- StringBuffer viaValue = new StringBuffer();
- if (header.containsKey(httpHeader.VIA)) viaValue.append((String)header.get(httpHeader.VIA));
- if (viaValue.length() > 0) viaValue.append(", ");
-
- // appending info about this peer
- viaValue
- .append(httpVer).append(" ")
- .append(yacyCore.seedDB.mySeed().getName()).append(".yacy ")
- .append("(YaCy ").append(switchboard.getConfig("vString", "0.0")).append(")");
-
- // storing header back
- header.put(httpHeader.VIA, new String(viaValue));
+ String myAddress = (httpd.alternativeResolver == null) ? null : httpd.alternativeResolver.myAlternativeAddress();
+ if (myAddress != null) {
+
+ // getting header set by other proxies in the chain
+ StringBuffer viaValue = new StringBuffer();
+ if (header.containsKey(httpHeader.VIA)) viaValue.append((String)header.get(httpHeader.VIA));
+ if (viaValue.length() > 0) viaValue.append(", ");
+
+ // appending info about this peer
+ viaValue
+ .append(httpVer).append(" ")
+ .append(myAddress).append(" ")
+ .append("(YaCy ").append(switchboard.getConfig("vString", "0.0")).append(")");
+
+ // storing header back
+ header.put(httpHeader.VIA, new String(viaValue));
+ }
}
/**
diff --git a/source/de/anomic/index/indexURLReference.java b/source/de/anomic/index/indexURLReference.java index 6e2bf4893..db9bb518c 100644 --- a/source/de/anomic/index/indexURLReference.java +++ b/source/de/anomic/index/indexURLReference.java @@ -191,7 +191,7 @@ public class indexURLReference { this.ranking = ranking; } - public indexURLReference(Properties prop){ + public indexURLReference(Properties prop) { // generates an plasmaLURLEntry using the properties from the argument // the property names must correspond to the one from toString //System.out.println("DEBUG-ENTRY: prop=" + prop.toString()); @@ -224,7 +224,7 @@ public class indexURLReference { } catch (ParseException e) { encodeDate(col_fresh, new Date()); } - this.entry.setCol(col_referrer, prop.getProperty("referrer", yacyURL.dummyHash).getBytes()); + this.entry.setCol(col_referrer, prop.getProperty("referrer", "").getBytes()); this.entry.setCol(col_md5, serverCodings.decodeHex(prop.getProperty("md5", ""))); this.entry.setCol(col_size, Integer.parseInt(prop.getProperty("size", "0"))); this.entry.setCol(col_wc, Integer.parseInt(prop.getProperty("wc", "0"))); @@ -434,9 +434,9 @@ public class indexURLReference { //return "{" + core + ",snippet=" + crypt.simpleEncode(snippet) + "}"; } - public plasmaCrawlEntry toBalancerEntry() { + public plasmaCrawlEntry toBalancerEntry(String initiatorHash) { return new plasmaCrawlEntry( - null, + initiatorHash, comp().url(), referrerHash(), comp().dc_title(), diff --git a/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java b/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java index dc2d8c99b..055dc6db8 100644 --- a/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java +++ b/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java @@ -106,7 +106,7 @@ public class plasmaCrawlQueues { } public yacyURL getURL(String urlhash) { - if (urlhash.equals(yacyURL.dummyHash)) return null; + assert urlhash != null; plasmaCrawlEntry ne = noticeURL.get(urlhash); if (ne != null) return ne.url(); plasmaCrawlZURL.Entry ee = delegatedURL.getEntry(urlhash); @@ -220,7 +220,7 @@ public class plasmaCrawlQueues { } if (this.log.isFine()) log.logFine("LOCALCRAWL: URL=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter=" + profile.generalFilter() - + ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false"))); + + ", permission=" + ((sb.wordIndex.seedDB == null) ? "undefined" : (((sb.wordIndex.seedDB.mySeed().isSenior()) || (sb.wordIndex.seedDB.mySeed().isPrincipal())) ? "true" : "false"))); processLocalCrawling(urlEntry, stats); return true; @@ -234,13 +234,13 @@ public class plasmaCrawlQueues { public boolean remoteCrawlLoaderJob() { // check if we are allowed to crawl urls provided by other peers - if (!yacyCore.seedDB.mySeed().getFlagAcceptRemoteCrawl()) { + if (!sb.wordIndex.seedDB.mySeed().getFlagAcceptRemoteCrawl()) { //this.log.logInfo("remoteCrawlLoaderJob: not done, we are not allowed to do that"); return false; } // check if we are a senior peer - if (!yacyCore.seedDB.mySeed().isActive()) { + if (!sb.wordIndex.seedDB.mySeed().isActive()) { //this.log.logInfo("remoteCrawlLoaderJob: not done, this should be a senior or principal peer"); return false; } @@ -266,7 +266,7 @@ public class plasmaCrawlQueues { (coreCrawlJobSize() == 0) && (remoteTriggeredCrawlJobSize() == 0) && (sb.queueSize() < 10)) { - if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) { + if (sb.wordIndex.seedDB != null && sb.wordIndex.seedDB.sizeConnected() > 0) { Iterator<yacySeed> e = yacyCore.dhtAgent.getProvidesRemoteCrawlURLs(); while (e.hasNext()) { seed = e.next(); @@ -285,7 +285,7 @@ public class plasmaCrawlQueues { while ((seed == null) && (remoteCrawlProviderHashes.size() > 0)) { hash = (String) remoteCrawlProviderHashes.remove(remoteCrawlProviderHashes.size() - 1); if (hash == null) continue; - seed = yacyCore.seedDB.get(hash); + seed = sb.wordIndex.seedDB.get(hash); if (seed == null) continue; // check if the peer is inside our cluster if ((sb.isRobinsonMode()) && (!sb.isInMyCluster(seed))) { @@ -296,7 +296,7 @@ public class plasmaCrawlQueues { if (seed == null) return false; // we know a peer which should provide remote crawl entries. load them now. - RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(seed, 20); + RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(sb.wordIndex.seedDB, seed, 20); if (feed == null) return true; // parse the rss yacyURL url, referrer; @@ -408,7 +408,7 @@ public class plasmaCrawlQueues { } if (this.log.isFine()) log.logFine("plasmaSwitchboard.remoteTriggeredCrawlJob: url=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter=" - + profile.generalFilter() + ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false"))); + + profile.generalFilter() + ", permission=" + ((sb.wordIndex.seedDB == null) ? "undefined" : (((sb.wordIndex.seedDB.mySeed().isSenior()) || (sb.wordIndex.seedDB.mySeed().isPrincipal())) ? "true" : "false"))); processLocalCrawling(urlEntry, stats); return true; @@ -440,9 +440,9 @@ public class plasmaCrawlQueues { ) { plasmaCrawlEntry centry = new plasmaCrawlEntry( - yacyCore.seedDB.mySeed().hash, + sb.wordIndex.seedDB.mySeed().hash, url, - null, + "", "", new Date(), (forText) ? @@ -485,7 +485,12 @@ public class plasmaCrawlQueues { this.entry.setStatus("worker-checkingrobots"); if ((entry.url().getProtocol().equals("http") || entry.url().getProtocol().equals("https")) && robotsParser.isDisallowed(entry.url())) { if (log.isFine()) log.logFine("Crawling of URL '" + entry.url().toString() + "' disallowed by robots.txt."); - plasmaCrawlZURL.Entry eentry = errorURL.newEntry(this.entry.url(), "denied by robots.txt"); + plasmaCrawlZURL.Entry eentry = errorURL.newEntry( + this.entry, + sb.wordIndex.seedDB.mySeed().hash, + new Date(), + 1, + "denied by robots.txt"); eentry.store(); errorURL.push(eentry); } else { @@ -493,7 +498,12 @@ public class plasmaCrawlQueues { this.entry.setStatus("worker-loading"); String result = loader.process(this.entry, plasmaParser.PARSER_MODE_CRAWLER); if (result != null) { - plasmaCrawlZURL.Entry eentry = errorURL.newEntry(this.entry.url(), "cannot load: " + result); + plasmaCrawlZURL.Entry eentry = errorURL.newEntry( + this.entry, + sb.wordIndex.seedDB.mySeed().hash, + new Date(), + 1, + "cannot load: " + result); eentry.store(); errorURL.push(eentry); } else { @@ -501,7 +511,12 @@ public class plasmaCrawlQueues { } } } catch (Exception e) { - plasmaCrawlZURL.Entry eentry = errorURL.newEntry(this.entry.url(), e.getMessage() + " - in worker"); + plasmaCrawlZURL.Entry eentry = errorURL.newEntry( + this.entry, + sb.wordIndex.seedDB.mySeed().hash, + new Date(), + 1, + e.getMessage() + " - in worker"); eentry.store(); errorURL.push(eentry); e.printStackTrace(); diff --git a/source/de/anomic/plasma/crawler/plasmaFTPLoader.java b/source/de/anomic/plasma/crawler/plasmaFTPLoader.java index c783c413a..e63610105 100644 --- a/source/de/anomic/plasma/crawler/plasmaFTPLoader.java +++ b/source/de/anomic/plasma/crawler/plasmaFTPLoader.java @@ -157,7 +157,7 @@ public class plasmaFTPLoader { // some error logging
final String detail = (berr.size() > 0) ? "\n Errorlog: " + berr.toString() : "";
log.logWarning("Unable to download URL " + entry.url().toString() + detail);
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR);
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR);
// an error has occured. cleanup
if (cacheFile.exists()) {
@@ -281,7 +281,7 @@ public class plasmaFTPLoader { ftpClient.exec("get \"" + path + "\" \"" + cacheFile.getAbsolutePath() + "\"", false);
} else {
log.logInfo("REJECTED TOO BIG FILE with size " + size + " Bytes for URL " + entry.url().toString());
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1,
+ sb.crawlQueues.errorURL.newEntry(entry, this.sb.wordIndex.seedDB.mySeed().hash, new Date(), 1,
plasmaCrawlEURL.DENIED_FILESIZE_LIMIT_EXCEEDED);
throw new Exception("file size exceeds limit");
}
diff --git a/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java b/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java index efe6228e7..62fd718c2 100644 --- a/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java +++ b/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java @@ -139,7 +139,7 @@ public final class plasmaHTTPLoader { if (retryCount < 0) {
this.log.logInfo("Redirection counter exceeded for URL " + entry.url().toString() + ". Processing aborted.");
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_COUNTER_EXCEEDED).store();
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_COUNTER_EXCEEDED).store();
return null;
}
@@ -154,7 +154,7 @@ public final class plasmaHTTPLoader { String hostlow = host.toLowerCase();
if (plasmaSwitchboard.urlBlacklist.isListed(indexReferenceBlacklist.BLACKLIST_CRAWLER, hostlow, path)) {
this.log.logInfo("CRAWLER Rejecting URL '" + entry.url().toString() + "'. URL is in blacklist.");
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_URL_IN_BLACKLIST).store();
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_URL_IN_BLACKLIST).store();
return null;
}
@@ -192,7 +192,7 @@ public final class plasmaHTTPLoader { // aborting download if content is to long ...
if (htCache.cacheFile().getAbsolutePath().length() > serverSystem.maxPathLength) {
this.log.logInfo("REJECTED URL " + entry.url().toString() + " because path too long '" + plasmaHTCache.cachePath.getAbsolutePath() + "'");
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_CACHEFILE_PATH_TOO_LONG);
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_CACHEFILE_PATH_TOO_LONG);
return (htCache = null);
}
@@ -202,7 +202,7 @@ public final class plasmaHTTPLoader { this.log.logInfo("REJECTED URL " + entry.url().toString() + " because of an invalid file path ('" +
htCache.cacheFile().getCanonicalPath() + "' does not start with '" +
plasmaHTCache.cachePath.getAbsolutePath() + "').");
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_INVALID_CACHEFILE_PATH);
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_INVALID_CACHEFILE_PATH);
return (htCache = null);
}
@@ -232,7 +232,7 @@ public final class plasmaHTTPLoader { fos = new httpdBoundedSizeOutputStream(fos,this.maxFileSize);
} else if (contentLength > this.maxFileSize) {
this.log.logInfo("REJECTED URL " + entry.url() + " because file size '" + contentLength + "' exceeds max filesize limit of " + this.maxFileSize + " bytes.");
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_FILESIZE_LIMIT_EXCEEDED);
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_FILESIZE_LIMIT_EXCEEDED);
return null;
}
}
@@ -251,7 +251,7 @@ public final class plasmaHTTPLoader { } else {
// if the response has not the right file type then reject file
this.log.logInfo("REJECTED WRONG MIME/EXT TYPE " + res.getResponseHeader().mime() + " for URL " + entry.url().toString());
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT);
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT);
return null;
}
} catch (SocketException e) {
@@ -262,7 +262,7 @@ public final class plasmaHTTPLoader { // and most possible corrupted
if (cacheFile.exists()) cacheFile.delete();
this.log.logSevere("CRAWLER LOADER ERROR1: with URL=" + entry.url().toString() + ": " + e.toString());
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_CONNECTION_ERROR);
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_CONNECTION_ERROR);
htCache = null;
}
} else if (res.getStatusLine().startsWith("30")) {
@@ -273,7 +273,7 @@ public final class plasmaHTTPLoader { if (redirectionUrlString.length() == 0) {
this.log.logWarning("CRAWLER Redirection of URL=" + entry.url().toString() + " aborted. Location header is empty.");
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_HEADER_EMPTY);
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_HEADER_EMPTY);
return null;
}
@@ -287,7 +287,7 @@ public final class plasmaHTTPLoader { // if we are already doing a shutdown we don't need to retry crawling
if (Thread.currentThread().isInterrupted()) {
this.log.logSevere("CRAWLER Retry of URL=" + entry.url().toString() + " aborted because of server shutdown.");
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_SHUTDOWN);
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_SHUTDOWN);
return null;
}
@@ -298,7 +298,7 @@ public final class plasmaHTTPLoader { String dbname = sb.urlExists(urlhash);
if (dbname != null) {
this.log.logWarning("CRAWLER Redirection of URL=" + entry.url().toString() + " ignored. The url appears already in db " + dbname);
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_TO_DOUBLE_CONTENT);
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_TO_DOUBLE_CONTENT);
return null;
}
@@ -312,7 +312,7 @@ public final class plasmaHTTPLoader { this.log.logInfo("REJECTED WRONG STATUS TYPE '" + res.getStatusLine() + "' for URL " + entry.url().toString());
// not processed any further
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_HTTP_STATUSCODE + res.getStatusCode() + ")");
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_HTTP_STATUSCODE + res.getStatusCode() + ")");
}
} finally {
@@ -402,7 +402,7 @@ public final class plasmaHTTPLoader { if (failreason != null) {
// add url into error db
- sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, failreason);
+ sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, failreason);
}
return null;
}
diff --git a/source/de/anomic/plasma/dbImport/plasmaDbImporter.java b/source/de/anomic/plasma/dbImport/plasmaDbImporter.java index d9f17356e..f559a117e 100644 --- a/source/de/anomic/plasma/dbImport/plasmaDbImporter.java +++ b/source/de/anomic/plasma/dbImport/plasmaDbImporter.java @@ -99,7 +99,7 @@ public class plasmaDbImporter extends AbstractImporter implements dbImporter { }
this.log.logFine("Initializing source word index db.");
- this.importWordIndex = new plasmaWordIndex(this.importPrimaryPath, this.importSecondaryPath, sb.getConfig("network.unit.name", ""), this.log);
+ this.importWordIndex = new plasmaWordIndex(sb.getConfig("network.unit.name", ""), this.log, this.importPrimaryPath, this.importSecondaryPath);
this.importStartSize = this.importWordIndex.size();
}
diff --git a/source/de/anomic/plasma/plasmaCrawlEntry.java b/source/de/anomic/plasma/plasmaCrawlEntry.java index efc53bf79..914f77705 100644 --- a/source/de/anomic/plasma/plasmaCrawlEntry.java +++ b/source/de/anomic/plasma/plasmaCrawlEntry.java @@ -34,7 +34,6 @@ import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroBitfield; import de.anomic.kelondro.kelondroNaturalOrder; import de.anomic.kelondro.kelondroRow; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.yacyURL; @@ -79,10 +78,6 @@ public class plasmaCrawlEntry { private String status; private int initialHash; // to provide a object hash that does not change even if the url changes because of redirection - public plasmaCrawlEntry(yacyURL url) { - this(yacyCore.seedDB.mySeed().hash, url, null, null, new Date(), null, 0, 0, 0); - } - /** * @param initiator the hash of the initiator peer * @param url the {@link URL} to crawl @@ -108,10 +103,12 @@ public class plasmaCrawlEntry { // create new entry and store it into database assert appdate != null; assert url != null; - if ((initiator == null) || (initiator.length() == 0)) initiator = yacyURL.dummyHash; + assert initiator != null; + assert initiator.length() > 0; + assert referrerhash != null; this.initiator = initiator; this.url = url; - this.refhash = (referrerhash == null) ? yacyURL.dummyHash : referrerhash; + this.refhash = referrerhash; this.name = (name == null) ? "" : name; this.appdate = (appdate == null) ? 0 : appdate.getTime(); this.profileHandle = profileHandle; // must not be null @@ -137,7 +134,7 @@ public class plasmaCrawlEntry { if (urlstring == null) throw new IOException ("url string is null"); this.initiator = entry.getColString(1, null); this.url = new yacyURL(urlstring, entry.getColString(0, null)); - this.refhash = (entry.empty(3)) ? yacyURL.dummyHash : entry.getColString(3, null); + this.refhash = (entry.empty(3)) ? "" : entry.getColString(3, null); this.name = (entry.empty(4)) ? "" : entry.getColString(4, "UTF-8").trim(); this.appdate = entry.getColLong(5); this.profileHandle = (entry.empty(6)) ? null : entry.getColString(6, null).trim(); diff --git a/source/de/anomic/plasma/plasmaCrawlResultURLs.java b/source/de/anomic/plasma/plasmaCrawlResultURLs.java index 6265dfcd1..89531f2fb 100644 --- a/source/de/anomic/plasma/plasmaCrawlResultURLs.java +++ b/source/de/anomic/plasma/plasmaCrawlResultURLs.java @@ -56,7 +56,6 @@ import java.util.LinkedList; import de.anomic.index.indexURLReference; import de.anomic.yacy.yacySeedDB; -import de.anomic.yacy.yacyURL; public final class plasmaCrawlResultURLs { @@ -81,10 +80,10 @@ public final class plasmaCrawlResultURLs { } public synchronized void stack(indexURLReference e, String initiatorHash, String executorHash, int stackType) { + assert initiatorHash != null; + assert executorHash != null; if (e == null) { return; } try { - if (initiatorHash == null) { initiatorHash = yacyURL.dummyHash; } - if (executorHash == null) { executorHash = yacyURL.dummyHash; } switch (stackType) { case 0: break; case 1: externResultStack.add(e.hash() + initiatorHash + executorHash); break; diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index 68ddafa6a..761ba2072 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -66,7 +66,6 @@ import de.anomic.kelondro.kelondroRowSet; import de.anomic.kelondro.kelondroTree;
import de.anomic.server.serverDomains;
import de.anomic.server.logging.serverLog;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyURL;
public final class plasmaCrawlStacker extends Thread {
@@ -213,7 +212,7 @@ public final class plasmaCrawlStacker extends Thread { // if the url was rejected we store it into the error URL db
if (rejectReason != null) {
- plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(entry, yacyCore.seedDB.mySeed().hash, null, 0, rejectReason);
+ plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(entry, sb.wordIndex.seedDB.mySeed().hash, new Date(), 1, rejectReason);
ee.store();
sb.crawlQueues.errorURL.push(ee);
}
@@ -357,7 +356,7 @@ public final class plasmaCrawlStacker extends Thread { plasmaCrawlEntry entry = new plasmaCrawlEntry(
initiatorHash, // initiator, needed for p2p-feedback
url, // url clear text string
- (referrer == null) ? null : referrer.hash(), // last url in crawling queue
+ (referrer == null) ? "" : referrer.hash(), // last url in crawling queue
name, // load date
loadDate, // the anchor name
(profile == null) ? null : profile.handle(), // profile must not be null!
@@ -481,15 +480,15 @@ public final class plasmaCrawlStacker extends Thread { }
// store information
- boolean local = ((entry.initiator().equals(yacyURL.dummyHash)) || (entry.initiator().equals(yacyCore.seedDB.mySeed().hash)));
+ boolean local = entry.initiator().equals(sb.wordIndex.seedDB.mySeed().hash);
boolean global =
(profile != null) &&
(profile.remoteIndexing()) /* granted */ &&
(entry.depth() == profile.generalDepth()) /* leaf node */ &&
//(initiatorHash.equals(yacyCore.seedDB.mySeed.hash)) /* not proxy */ &&
(
- (yacyCore.seedDB.mySeed().isSenior()) ||
- (yacyCore.seedDB.mySeed().isPrincipal())
+ (sb.wordIndex.seedDB.mySeed().isSenior()) ||
+ (sb.wordIndex.seedDB.mySeed().isPrincipal())
) /* qualified */;
if ((!local)&&(!global)&&(!profile.handle().equals(this.sb.defaultRemoteProfile.handle()))) {
diff --git a/source/de/anomic/plasma/plasmaCrawlZURL.java b/source/de/anomic/plasma/plasmaCrawlZURL.java index b8b35d66f..55af87099 100644 --- a/source/de/anomic/plasma/plasmaCrawlZURL.java +++ b/source/de/anomic/plasma/plasmaCrawlZURL.java @@ -38,7 +38,6 @@ import de.anomic.kelondro.kelondroFlexWidthArray; import de.anomic.kelondro.kelondroIndex; import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroRowSet; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySeedDB; import de.anomic.yacy.yacyURL; @@ -90,17 +89,17 @@ public class plasmaCrawlZURL { } public synchronized Entry newEntry( - plasmaCrawlEntry bentry, String executor, Date workdate, - int workcount, String anycause) { - if ((executor == null) || (executor.length() < yacySeedDB.commonHashLength)) executor = yacyURL.dummyHash; + plasmaCrawlEntry bentry, + String executor, + Date workdate, + int workcount, + String anycause) { + assert executor != null; + assert executor.length() > 0; if (anycause == null) anycause = "unknown"; return new Entry(bentry, executor, workdate, workcount, anycause); } - public synchronized Entry newEntry(yacyURL url, String anycause) { - return new Entry(url, anycause); - } - public boolean remove(String hash) { if (hash == null) return false; try { @@ -155,23 +154,23 @@ public class plasmaCrawlZURL { public class Entry { plasmaCrawlEntry bentry; // the balancer entry - private String executor; // the crawling initiator + private String executor; // the crawling executor private Date workdate; // the time when the url was last time tried to load private int workcount; // number of tryings private String anycause; // string describing reason for load fail private boolean stored; - public Entry(yacyURL url, String reason) { - this(new plasmaCrawlEntry(url), null, new Date(), 0, reason); - } - public Entry( - plasmaCrawlEntry bentry, String executor, Date workdate, - int workcount, String anycause) { + plasmaCrawlEntry bentry, + String executor, + Date workdate, + int workcount, + String anycause) { // create new entry assert bentry != null; + assert executor != null; this.bentry = bentry; - this.executor = (executor == null) ? yacyCore.seedDB.mySeed().hash : executor; + this.executor = executor; this.workdate = (workdate == null) ? new Date() : workdate; this.workcount = workcount; this.anycause = (anycause == null) ? "" : anycause; @@ -179,11 +178,6 @@ public class plasmaCrawlZURL { } public Entry(kelondroRow.Entry entry) throws IOException { - insertEntry(entry); - this.stored = true; - } - - private void insertEntry(kelondroRow.Entry entry) throws IOException { assert (entry != null); this.executor = entry.getColString(1, "UTF-8"); this.workdate = new Date(entry.getColLong(2)); @@ -191,6 +185,7 @@ public class plasmaCrawlZURL { this.anycause = entry.getColString(4, "UTF-8"); this.bentry = new plasmaCrawlEntry(plasmaCrawlEntry.rowdef.newEntry(entry.getColBytes(5))); assert ((new String(entry.getColBytes(0))).equals(bentry.url().hash())); + this.stored = true; return; } diff --git a/source/de/anomic/plasma/plasmaDHTChunk.java b/source/de/anomic/plasma/plasmaDHTChunk.java index 92d0a9f8c..64d358041 100644 --- a/source/de/anomic/plasma/plasmaDHTChunk.java +++ b/source/de/anomic/plasma/plasmaDHTChunk.java @@ -58,7 +58,6 @@ import de.anomic.kelondro.kelondroBase64Order; import de.anomic.kelondro.kelondroException;
import de.anomic.server.serverCodings;
import de.anomic.server.logging.serverLog;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyDHTAction;
import de.anomic.yacy.yacySeedDB;
@@ -132,7 +131,7 @@ public class plasmaDHTChunk { this.log = log;
this.wordIndex = wordIndex;
this.startPointHash = selectTransferStart();
- if (this.log.isFine()) log.logFine("Selected hash " + this.startPointHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, this.startPointHash));
+ if (this.log.isFine()) log.logFine("Selected hash " + this.startPointHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(wordIndex.seedDB.mySeed().hash, this.startPointHash));
selectTransferContainers(this.startPointHash, minCount, maxCount, maxtime);
// count the indexes, can be smaller as expected
@@ -150,7 +149,7 @@ public class plasmaDHTChunk { try {
this.log = log;
this.wordIndex = wordIndex;
- if (this.log.isFine()) log.logFine("Demanded hash " + startHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, this.startPointHash));
+ if (this.log.isFine()) log.logFine("Demanded hash " + startHash + " as start point for index distribution, distance = " + yacyDHTAction.dhtDistance(wordIndex.seedDB.mySeed().hash, this.startPointHash));
selectTransferContainers(startHash, minCount, maxCount, maxtime);
// count the indexes, can be smaller as expected
@@ -167,12 +166,12 @@ public class plasmaDHTChunk { private String selectTransferStart() {
String startPointHash;
// first try to select with increasing probality a good start point
- double minimumDistance = ((double) peerRedundancy) / ((double) yacyCore.seedDB.sizeConnected());
+ double minimumDistance = ((double) peerRedundancy) / ((double) wordIndex.seedDB.sizeConnected());
double d, bestDistance = 0.0;
String bestHash = null;
- for (int i = yacyCore.seedDB.sizeConnected() / 8; i > 0; i--) {
+ for (int i = wordIndex.seedDB.sizeConnected() / 8; i > 0; i--) {
startPointHash = kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(Long.toString(i + System.currentTimeMillis()))).substring(2, 2 + yacySeedDB.commonHashLength);
- d = yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, startPointHash);
+ d = yacyDHTAction.dhtDistance(wordIndex.seedDB.mySeed().hash, startPointHash);
if (d > (minimumDistance + ((double) i / (double) 10))) {
return startPointHash;
}
@@ -183,7 +182,7 @@ public class plasmaDHTChunk { }
// if that fails, take simply the best start point
if (bestHash == null) {
- return yacyCore.seedDB.mySeed().hash.substring(0, 11) + "z";
+ return wordIndex.seedDB.mySeed().hash.substring(0, 11) + "z";
} else {
return bestHash;
}
@@ -219,7 +218,7 @@ public class plasmaDHTChunk { int wholesize;
urlCache = new HashMap<String, indexURLReference>();
- final double maximumDistance = ((double) peerRedundancy * 2) / ((double) yacyCore.seedDB.sizeConnected());
+ final double maximumDistance = ((double) peerRedundancy * 2) / ((double) wordIndex.seedDB.sizeConnected());
final long timeout = (maxtime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxtime;
while (
(maxcount > refcount) &&
diff --git a/source/de/anomic/plasma/plasmaDHTFlush.java b/source/de/anomic/plasma/plasmaDHTFlush.java index 3b23f8f57..e716db3a1 100644 --- a/source/de/anomic/plasma/plasmaDHTFlush.java +++ b/source/de/anomic/plasma/plasmaDHTFlush.java @@ -230,7 +230,7 @@ public class plasmaDHTFlush extends Thread { // handover chunk to transfer worker if ((newDHTChunk.containerSize() > 0) || (newDHTChunk.getStatus() == plasmaDHTChunk.chunkStatus_FILLED)) { - this.worker = new plasmaDHTTransfer(this.log, this.seed, newDHTChunk, this.gzipBody4Transfer, this.timeout4Transfer, 5); + this.worker = new plasmaDHTTransfer(this.log, this.wordIndex.seedDB, this.seed, newDHTChunk, this.gzipBody4Transfer, this.timeout4Transfer, 5); this.worker.setTransferMode(plasmaDHTTransfer.TRANSFER_MODE_FLUSH); this.worker.start(); } diff --git a/source/de/anomic/plasma/plasmaDHTTransfer.java b/source/de/anomic/plasma/plasmaDHTTransfer.java index 2363d0fa3..c3ae49d23 100644 --- a/source/de/anomic/plasma/plasmaDHTTransfer.java +++ b/source/de/anomic/plasma/plasmaDHTTransfer.java @@ -48,6 +48,7 @@ import de.anomic.server.logging.serverLog; import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
+import de.anomic.yacy.yacySeedDB;
public class plasmaDHTTransfer extends Thread {
@@ -72,12 +73,14 @@ public class plasmaDHTTransfer extends Thread { plasmaDHTChunk dhtChunk;
// other fields
+ private yacySeedDB seedDB;
private int maxRetry;
private int transferMode = TRANSFER_MODE_DISTRIBUTION;
serverLog log;
public plasmaDHTTransfer(
- serverLog log,
+ serverLog log,
+ yacySeedDB seedDB,
yacySeed destSeed,
plasmaDHTChunk dhtChunk,
boolean gzipBody,
@@ -86,6 +89,7 @@ public class plasmaDHTTransfer extends Thread { ) {
super(new ThreadGroup("TransferIndexThreadGroup"), "TransferIndexWorker_" + destSeed.getName());
this.log = log;
+ this.seedDB = seedDB;
this.gzipBody4Transfer = gzipBody;
this.timeout4Transfer = timeout;
this.dhtChunk = dhtChunk;
@@ -152,7 +156,7 @@ public class plasmaDHTTransfer extends Thread { // transfering seleted words to remote peer
this.transferStatusMessage = "Running: Transfering chunk to target " + this.seed.hash + "/" + this.seed.getName();
- HashMap<String, Object> result = yacyClient.transferIndex(this.seed, this.dhtChunk.containers(), this.dhtChunk.urlCacheMap(), this.gzipBody4Transfer, this.timeout4Transfer);
+ HashMap<String, Object> result = yacyClient.transferIndex(this.seedDB, this.seed, this.dhtChunk.containers(), this.dhtChunk.urlCacheMap(), this.gzipBody4Transfer, this.timeout4Transfer);
String error = (String) result.get("result");
if (error == null) {
// words successfully transfered
@@ -236,7 +240,7 @@ public class plasmaDHTTransfer extends Thread { return;
// doing a peer ping to the remote seed
- int added = yacyClient.publishMySeed(this.seed.getPublicAddress(), this.seed.hash);
+ int added = yacyClient.publishMySeed(this.seedDB.mySeed(), this.seed.getPublicAddress(), this.seed.hash);
if (added < 0) {
// inc. retry counter
retryCount++;
@@ -246,7 +250,7 @@ public class plasmaDHTTransfer extends Thread { continue;
}
- yacyCore.seedDB.getConnected(this.seed.hash);
+ this.seedDB.getConnected(this.seed.hash);
this.transferStatusMessage = "running";
break;
}
diff --git a/source/de/anomic/plasma/plasmaGrafics.java b/source/de/anomic/plasma/plasmaGrafics.java index 763fc9ba2..1967febbf 100644 --- a/source/de/anomic/plasma/plasmaGrafics.java +++ b/source/de/anomic/plasma/plasmaGrafics.java @@ -54,9 +54,9 @@ import java.awt.image.BufferedImage; import java.util.Date;
import java.util.Iterator;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySearch;
import de.anomic.yacy.yacySeed;
+import de.anomic.yacy.yacySeedDB;
import de.anomic.ymage.ymageMatrix;
import de.anomic.ymage.ymageToolPrint;
@@ -125,7 +125,7 @@ public class plasmaGrafics { private static BufferedImage logo = null; // [MN]
private static long bannerPictureDate = 0; // [MN]
- public static ymageMatrix getSearchEventPicture(String eventID) {
+ public static ymageMatrix getSearchEventPicture(yacySeedDB seedDB, String eventID) {
plasmaSearchEvent event = plasmaSearchEvent.getEvent(eventID);
if (event == null) return null;
yacySearch[] primarySearches = event.getPrimarySearchThreads();
@@ -133,7 +133,7 @@ public class plasmaGrafics { if (primarySearches == null) return null; // this was a local search and there are no threads
// get a copy of a recent network picture
- ymageMatrix eventPicture = getNetworkPicture(120000, plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", "unspecified"), plasmaSwitchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"), COL_BACKGROUND);
+ ymageMatrix eventPicture = getNetworkPicture(seedDB, 120000, plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", "unspecified"), plasmaSwitchboard.getSwitchboard().getConfig("network.unit.description", "unspecified"), COL_BACKGROUND);
//if (eventPicture instanceof ymageMatrix) eventPicture = (ymageMatrix) eventPicture; //new ymageMatrix((ymageMatrix) eventPicture);
// TODO: fix cloning of ymageMatrix pictures
@@ -177,24 +177,24 @@ public class plasmaGrafics { return eventPicture;
}
- public static ymageMatrix getNetworkPicture(long maxAge, String networkName, String networkTitle, String bgcolor) {
- return getNetworkPicture(maxAge, 640, 480, 300, 300, 1000, true, networkName, networkTitle, bgcolor);
+ public static ymageMatrix getNetworkPicture(yacySeedDB seedDB, long maxAge, String networkName, String networkTitle, String bgcolor) {
+ return getNetworkPicture(seedDB, maxAge, 640, 480, 300, 300, 1000, true, networkName, networkTitle, bgcolor);
}
- public static ymageMatrix getNetworkPicture(long maxAge, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle, String bgcolor) {
+ public static ymageMatrix getNetworkPicture(yacySeedDB seedDB, long maxAge, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle, String bgcolor) {
if ((networkPicture == null) || ((System.currentTimeMillis() - networkPictureDate) > maxAge)) {
- drawNetworkPicture(width, height, passiveLimit, potentialLimit, maxCount, corona, networkName, networkTitle, bgcolor);
+ drawNetworkPicture(seedDB, width, height, passiveLimit, potentialLimit, maxCount, corona, networkName, networkTitle, bgcolor);
}
return networkPicture;
}
- private static void drawNetworkPicture(int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle, String bgcolor) {
+ private static void drawNetworkPicture(yacySeedDB seedDB, int width, int height, int passiveLimit, int potentialLimit, int maxCount, boolean corona, String networkName, String networkTitle, String bgcolor) {
int innerradius = Math.min(width, height) / 5;
- int outerradius = innerradius + innerradius * yacyCore.seedDB.sizeConnected() / 100;
+ int outerradius = innerradius + innerradius * seedDB.sizeConnected() / 100;
if (outerradius > innerradius * 2) outerradius = innerradius * 2;
- if (yacyCore.seedDB == null) return; // no other peers known
+ if (seedDB == null) return; // no other peers known
networkPicture = new ymageMatrix(width, height, ymageMatrix.MODE_SUB, bgcolor);
@@ -211,7 +211,7 @@ public class plasmaGrafics { // draw connected senior and principals
int count = 0;
int totalCount = 0;
- Iterator<yacySeed> e = yacyCore.seedDB.seedsConnected(true, false, null, (float) 0.0);
+ Iterator<yacySeed> e = seedDB.seedsConnected(true, false, null, (float) 0.0);
while (e.hasNext() && count < maxCount) {
seed = (yacySeed) e.next();
@@ -224,7 +224,7 @@ public class plasmaGrafics { // draw disconnected senior and principals that have been seen lately
count = 0;
- e = yacyCore.seedDB.seedsSortedDisconnected(false, yacySeed.LASTSEEN);
+ e = seedDB.seedsSortedDisconnected(false, yacySeed.LASTSEEN);
while (e.hasNext() && count < maxCount) {
seed = (yacySeed) e.next();
if (seed != null) {
@@ -238,7 +238,7 @@ public class plasmaGrafics { // draw juniors that have been seen lately
count = 0;
- e = yacyCore.seedDB.seedsSortedPotential(false, yacySeed.LASTSEEN);
+ e = seedDB.seedsSortedPotential(false, yacySeed.LASTSEEN);
while (e.hasNext() && count < maxCount) {
seed = (yacySeed) e.next();
if (seed != null) {
@@ -251,7 +251,7 @@ public class plasmaGrafics { totalCount += count;
// draw my own peer
- drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, yacyCore.seedDB.mySeed(), COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, corona);
+ drawNetworkPicturePeer(networkPicture, width / 2, height / 2 + 20, innerradius, outerradius, seedDB.mySeed(), COL_WE_DOT, COL_WE_LINE, COL_WE_TEXT, corona);
// draw description
networkPicture.setColor(COL_HEADLINE);
diff --git a/source/de/anomic/plasma/plasmaProfiling.java b/source/de/anomic/plasma/plasmaProfiling.java index 1588e019a..d80874dc4 100644 --- a/source/de/anomic/plasma/plasmaProfiling.java +++ b/source/de/anomic/plasma/plasmaProfiling.java @@ -32,6 +32,7 @@ import java.util.Iterator; import de.anomic.server.serverProfiling; import de.anomic.server.serverProfiling.Event; import de.anomic.yacy.yacyCore; +import de.anomic.yacy.yacySeed; import de.anomic.ymage.ymageChart; import de.anomic.ymage.ymageMatrix; @@ -41,11 +42,11 @@ public class plasmaProfiling { public static long lastPPMUpdate = System.currentTimeMillis()- 30000; - public static void updateIndexedPage(plasmaSwitchboardQueue.QueueEntry entry) { + public static void updateIndexedPage(yacySeed mySeed, plasmaSwitchboardQueue.QueueEntry entry) { if (System.currentTimeMillis() - lastPPMUpdate > 30000) { // we don't want to do this too often yacyCore.peerActions.updateMySeed(); - serverProfiling.update("ppm", new Long(yacyCore.seedDB.mySeed().getPPM())); + serverProfiling.update("ppm", new Long(mySeed.getPPM())); lastPPMUpdate = System.currentTimeMillis(); } serverProfiling.update("indexed", entry.url().toNormalform(true, false)); diff --git a/source/de/anomic/plasma/plasmaRankingDistribution.java b/source/de/anomic/plasma/plasmaRankingDistribution.java index 7c72101ca..dd9fa78a8 100644 --- a/source/de/anomic/plasma/plasmaRankingDistribution.java +++ b/source/de/anomic/plasma/plasmaRankingDistribution.java @@ -52,8 +52,8 @@ import java.util.StringTokenizer; import de.anomic.server.serverFileUtils;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyClient;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
+import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyVersion;
public final class plasmaRankingDistribution {
@@ -73,10 +73,12 @@ public final class plasmaRankingDistribution { private int method; // of peer selection
private int percentage; // to select any other peer
private String address[]; // of fixed other peer
+ private yacySeedDB seedDB;
private static Random random = new Random(System.currentTimeMillis());
- public plasmaRankingDistribution(serverLog log, File sourcePath, int method, int percentage, String addresses) {
+ public plasmaRankingDistribution(serverLog log, yacySeedDB seedDB, File sourcePath, int method, int percentage, String addresses) {
this.log = log;
+ this.seedDB = seedDB;
this.sourcePath = sourcePath;
this.method = method;
this.percentage = percentage;
@@ -104,15 +106,15 @@ public final class plasmaRankingDistribution { log.logFine("no ranking distribution: no transfer method given");
return false;
}
- if (yacyCore.seedDB == null) {
+ if (seedDB == null) {
log.logFine("no ranking distribution: seedDB == null");
return false;
}
- if (yacyCore.seedDB.mySeed() == null) {
+ if (seedDB.mySeed() == null) {
log.logFine("no ranking distribution: mySeed == null");
return false;
}
- if (yacyCore.seedDB.mySeed().isVirgin()) {
+ if (seedDB.mySeed().isVirgin()) {
log.logFine("no ranking distribution: status is virgin");
return false;
}
@@ -164,7 +166,7 @@ public final class plasmaRankingDistribution { if (Thread.currentThread().isInterrupted()) throw new InterruptedException("Shutdown in progress");
// get next target
- target = yacyCore.seedDB.anySeedVersion(yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION);
+ target = seedDB.anySeedVersion(yacyVersion.YACY_ACCEPTS_RANKING_TRANSMISSION);
if (target == null) continue;
String targetaddress = target.getPublicAddress();
diff --git a/source/de/anomic/plasma/plasmaSearchAPI.java b/source/de/anomic/plasma/plasmaSearchAPI.java index e84d32753..f6d614bf7 100644 --- a/source/de/anomic/plasma/plasmaSearchAPI.java +++ b/source/de/anomic/plasma/plasmaSearchAPI.java @@ -72,20 +72,16 @@ public class plasmaSearchAPI { yacySeed seed; int hc = 0; prop.put("searchresult_keyhash", startHash); - if (yacyCore.seedDB != null && yacyCore.seedDB.sizeConnected() > 0) { - Iterator<yacySeed> e = yacyCore.dhtAgent.getAcceptRemoteIndexSeeds(startHash); - while (e.hasNext()) { - seed = (yacySeed) e.next(); - if (seed != null) { - prop.put("searchresult_hosts_" + hc + "_hosthash", seed.hash); - prop.putHTML("searchresult_hosts_" + hc + "_hostname", seed.hash + " " + seed.get(yacySeed.NAME, "nameless")); - hc++; - } + Iterator<yacySeed> e = yacyCore.dhtAgent.getAcceptRemoteIndexSeeds(startHash); + while (e.hasNext()) { + seed = (yacySeed) e.next(); + if (seed != null) { + prop.put("searchresult_hosts_" + hc + "_hosthash", seed.hash); + prop.putHTML("searchresult_hosts_" + hc + "_hostname", seed.hash + " " + seed.get(yacySeed.NAME, "nameless")); + hc++; } - prop.put("searchresult_hosts", hc); - } else { - prop.put("searchresult_hosts", "0"); } + prop.put("searchresult_hosts", hc); } public static plasmaSearchRankingProcess genSearchresult(serverObjects prop, plasmaSwitchboard sb, String keyhash, kelondroBitfield filter) { diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index 1436b1c4b..4602d0fda 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -48,7 +48,6 @@ import de.anomic.kelondro.kelondroSortStore; import de.anomic.plasma.plasmaSnippetCache.MediaSnippet;
import de.anomic.server.serverProfiling;
import de.anomic.server.logging.serverLog;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyDHTAction;
import de.anomic.yacy.yacySearch;
import de.anomic.yacy.yacySeed;
@@ -184,7 +183,7 @@ public final class plasmaSearchEvent { IAmaxcounthash = wordhash;
maxcount = container.size();
}
- d = yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed().hash, wordhash);
+ d = yacyDHTAction.dhtDistance(wordIndex.seedDB.mySeed().hash, wordhash);
if (d < mindhtdistance) {
// calculate the word hash that is closest to our dht position
mindhtdistance = d;
@@ -364,7 +363,7 @@ public final class plasmaSearchEvent { } else {
// problems with snippet fetch
registerFailure(page.hash(), "no text snippet for URL " + comp.url());
- plasmaSnippetCache.failConsequences(snippet, query.id(false));
+ if (!wordIndex.seedDB.mySeed().isVirgin()) plasmaSnippetCache.failConsequences(snippet, query.id(false));
return null;
}
} else {
@@ -677,7 +676,7 @@ public final class plasmaSearchEvent { Iterator<Map.Entry<String, String>> i1 = abstractJoin.entrySet().iterator();
Map.Entry<String, String> entry1;
String url, urls, peer, peers;
- String mypeerhash = yacyCore.seedDB.mySeed().hash;
+ String mypeerhash = wordIndex.seedDB.mySeed().hash;
boolean mypeerinvolved = false;
int mypeercount;
while (i1.hasNext()) {
@@ -788,7 +787,7 @@ public final class plasmaSearchEvent { // translate host into current IP
int p = host.indexOf(".");
String hash = yacySeed.hexHash2b64Hash(host.substring(p + 1, host.length() - 6));
- yacySeed seed = yacyCore.seedDB.getConnected(hash);
+ yacySeed seed = wordIndex.seedDB.getConnected(hash);
String filename = urlcomps.url().getFile();
String address = null;
if ((seed == null) || ((address = seed.getPublicAddress()) == null)) {
diff --git a/source/de/anomic/plasma/plasmaSnippetCache.java b/source/de/anomic/plasma/plasmaSnippetCache.java index fba3b2d4d..a6e460c88 100644 --- a/source/de/anomic/plasma/plasmaSnippetCache.java +++ b/source/de/anomic/plasma/plasmaSnippetCache.java @@ -66,7 +66,6 @@ import de.anomic.kelondro.kelondroMSetTools; import de.anomic.plasma.cache.IResourceInfo; import de.anomic.plasma.parser.ParserException; import de.anomic.server.logging.serverLog; -import de.anomic.yacy.yacyCore; import de.anomic.yacy.yacySearch; import de.anomic.yacy.yacyURL; @@ -897,7 +896,6 @@ public class plasmaSnippetCache { public static String failConsequences(TextSnippet snippet, String eventID) { // problems with snippet fetch - if (yacyCore.seedDB.mySeed().isVirgin()) return snippet.getError() + " (no consequences, no network connection)"; // no consequences if we do not have a network connection String urlHash = snippet.getUrl().hash(); String querystring = kelondroMSetTools.setToString(snippet.getRemainingHashes(), ' '); if ((snippet.getErrorCode() == ERROR_SOURCE_LOADING) || diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index f581ff06f..58151ba78 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -932,7 +932,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo // start indexing management
log.logConfig("Starting Indexing Management");
String networkName = getConfig("network.unit.name", "");
- wordIndex = new plasmaWordIndex(indexPrimaryPath, indexSecondaryPath, networkName, log);
+ wordIndex = new plasmaWordIndex(networkName, log, indexPrimaryPath, indexSecondaryPath);
crawlResults = new plasmaCrawlResultURLs();
// start yacy core
@@ -1166,8 +1166,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo CRDist1Target = kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080
**/
rankingOn = getConfig(RANKING_DIST_ON, "true").equals("true") && networkName.equals("freeworld");
- rankingOwnDistribution = new plasmaRankingDistribution(log, new File(rankingPath, getConfig(RANKING_DIST_0_PATH, plasmaRankingDistribution.CR_OWN)), (int) getConfigLong(RANKING_DIST_0_METHOD, plasmaRankingDistribution.METHOD_ANYSENIOR), (int) getConfigLong(RANKING_DIST_0_METHOD, 0), getConfig(RANKING_DIST_0_TARGET, ""));
- rankingOtherDistribution = new plasmaRankingDistribution(log, new File(rankingPath, getConfig(RANKING_DIST_1_PATH, plasmaRankingDistribution.CR_OTHER)), (int) getConfigLong(RANKING_DIST_1_METHOD, plasmaRankingDistribution.METHOD_MIXEDSENIOR), (int) getConfigLong(RANKING_DIST_1_METHOD, 30), getConfig(RANKING_DIST_1_TARGET, "kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080"));
+ rankingOwnDistribution = new plasmaRankingDistribution(log, wordIndex.seedDB, new File(rankingPath, getConfig(RANKING_DIST_0_PATH, plasmaRankingDistribution.CR_OWN)), (int) getConfigLong(RANKING_DIST_0_METHOD, plasmaRankingDistribution.METHOD_ANYSENIOR), (int) getConfigLong(RANKING_DIST_0_METHOD, 0), getConfig(RANKING_DIST_0_TARGET, ""));
+ rankingOtherDistribution = new plasmaRankingDistribution(log, wordIndex.seedDB, new File(rankingPath, getConfig(RANKING_DIST_1_PATH, plasmaRankingDistribution.CR_OTHER)), (int) getConfigLong(RANKING_DIST_1_METHOD, plasmaRankingDistribution.METHOD_MIXEDSENIOR), (int) getConfigLong(RANKING_DIST_1_METHOD, 30), getConfig(RANKING_DIST_1_TARGET, "kaskelix.de:8080,yacy.dyndns.org:8000,suma-lab.de:8080"));
// init facility DB
/*
@@ -1218,9 +1218,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo // init robinson cluster
// before we do that, we wait some time until the seed list is loaded.
- while (((System.currentTimeMillis() - startedSeedListAquisition) < 8000) && (yacyCore.seedDB.sizeConnected() == 0)) try {Thread.sleep(1000);} catch (InterruptedException e) {}
+ while (((System.currentTimeMillis() - startedSeedListAquisition) < 8000) && (this.wordIndex.seedDB.sizeConnected() == 0)) try {Thread.sleep(1000);} catch (InterruptedException e) {}
try {Thread.sleep(1000);} catch (InterruptedException e) {}
- this.clusterhashes = yacyCore.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
+ this.clusterhashes = this.wordIndex.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
// deploy blocking threads
indexingStorageProcessor = new serverProcessor<indexingQueueEntry>(this, "storeDocumentIndex", 1, null);
@@ -1419,7 +1419,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo public yacyURL getURL(String urlhash) {
if (urlhash == null) return null;
- if (urlhash.equals(yacyURL.dummyHash)) return null;
+ if (urlhash.length() == 0) return null;
yacyURL ne = crawlQueues.getURL(urlhash);
if (ne != null) return ne;
indexURLReference le = wordIndex.getURL(urlhash, null, 0);
@@ -1707,7 +1707,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo crawlQueues.close();
log.logConfig("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal to database manager (stand by...)");
wordIndex.close();
- yc.close();
log.logConfig("SWITCHBOARD SHUTDOWN TERMINATED");
}
@@ -1939,20 +1938,20 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo // clean up news
checkInterruption();
try {
- if (this.log.isFine()) log.logFine("Cleaning Incoming News, " + yacyCore.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack");
- if (yacyCore.newsPool.automaticProcess() > 0) hasDoneSomething = true;
+ if (this.log.isFine()) log.logFine("Cleaning Incoming News, " + this.wordIndex.newsPool.size(yacyNewsPool.INCOMING_DB) + " entries on stack");
+ if (this.wordIndex.newsPool.automaticProcess(wordIndex.seedDB) > 0) hasDoneSomething = true;
} catch (IOException e) {}
if (getConfigBool("cleanup.deletionProcessedNews", true)) {
- yacyCore.newsPool.clear(yacyNewsPool.PROCESSED_DB);
+ this.wordIndex.newsPool.clear(yacyNewsPool.PROCESSED_DB);
}
if (getConfigBool("cleanup.deletionPublishedNews", true)) {
- yacyCore.newsPool.clear(yacyNewsPool.PUBLISHED_DB);
+ this.wordIndex.newsPool.clear(yacyNewsPool.PUBLISHED_DB);
}
// clean up seed-dbs
if(getConfigBool("routing.deleteOldSeeds.permission",true)) {
final long deleteOldSeedsTime = getConfigLong("routing.deleteOldSeeds.time",7)*24*3600000;
- Iterator<yacySeed> e = yacyCore.seedDB.seedsSortedDisconnected(true,yacySeed.LASTSEEN);
+ Iterator<yacySeed> e = this.wordIndex.seedDB.seedsSortedDisconnected(true,yacySeed.LASTSEEN);
yacySeed seed = null;
ArrayList<String> deleteQueue = new ArrayList<String>();
checkInterruption();
@@ -1966,9 +1965,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo deleteQueue.add(seed.hash);
}
}
- for(int i=0;i<deleteQueue.size();++i) yacyCore.seedDB.removeDisconnected((String)deleteQueue.get(i));
+ for(int i=0;i<deleteQueue.size();++i) this.wordIndex.seedDB.removeDisconnected((String)deleteQueue.get(i));
deleteQueue.clear();
- e = yacyCore.seedDB.seedsSortedPotential(true,yacySeed.LASTSEEN);
+ e = this.wordIndex.seedDB.seedsSortedPotential(true,yacySeed.LASTSEEN);
checkInterruption();
//clean potential seeds
while(e.hasNext()) {
@@ -1980,7 +1979,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo deleteQueue.add(seed.hash);
}
}
- for (int i = 0; i < deleteQueue.size(); ++i) yacyCore.seedDB.removePotential((String)deleteQueue.get(i));
+ for (int i = 0; i < deleteQueue.size(); ++i) this.wordIndex.seedDB.removePotential((String)deleteQueue.get(i));
}
// check if update is available and
@@ -2003,7 +2002,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo }
// initiate broadcast about peer startup to spread supporter url
- if (yacyCore.newsPool.size(yacyNewsPool.OUTGOING_DB) == 0) {
+ if (this.wordIndex.newsPool.size(yacyNewsPool.OUTGOING_DB) == 0) {
// read profile
final Properties profile = new Properties();
FileInputStream fileIn = null;
@@ -2018,7 +2017,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo if ((homepage != null) && (homepage.length() > 10)) {
Properties news = new Properties();
news.put("homepage", profile.get("homepage"));
- yacyCore.newsPool.publishMyNews(yacyNewsRecord.newRecord(yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news));
+ this.wordIndex.newsPool.publishMyNews(yacyNewsRecord.newRecord(wordIndex.seedDB.mySeed(), yacyNewsPool.CATEGORY_PROFILE_BROADCAST, news));
}
}
/*
@@ -2030,7 +2029,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo kelondroCache.setCacheGrowStati(40 * 1024 * 1024, 20 * 1024 * 1024);
*/
// update the cluster set
- this.clusterhashes = yacyCore.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
+ this.clusterhashes = this.wordIndex.seedDB.clusterHashes(getConfig("cluster.peers.yacydomain", ""));
return hasDoneSomething;
} catch (InterruptedException e) {
@@ -2206,7 +2205,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo checkInterruption();
log.logFine("Not indexed any word in URL " + entry.url() + "; cause: " + noIndexReason);
- addURLtoErrorDB(entry.url(), (referrerURL == null) ? null : referrerURL.hash(), entry.initiator(), dc_title, noIndexReason, new kelondroBitfield());
+ addURLtoErrorDB(entry.url(), (referrerURL == null) ? "" : referrerURL.hash(), entry.initiator(), dc_title, noIndexReason, new kelondroBitfield());
/*
if ((processCase == PROCESSCASE_6_GLOBAL_CRAWLING) && (initiatorPeer != null)) {
if (clusterhashes != null) initiatorPeer.setAlternativeAddress((String) clusterhashes.get(initiatorPeer.hash));
@@ -2267,7 +2266,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo crawlResults.stack(
newEntry, // loaded url db entry
queueEntry.initiator(), // initiator peer hash
- yacyCore.seedDB.mySeed().hash, // executor peer hash
+ this.wordIndex.seedDB.mySeed().hash, // executor peer hash
processCase // process case
);
@@ -2282,7 +2281,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo indexedPages++;
// update profiling info
- plasmaProfiling.updateIndexedPage(queueEntry);
+ plasmaProfiling.updateIndexedPage(wordIndex.seedDB.mySeed(), queueEntry);
// if this was performed for a remote crawl request, notify requester
yacySeed initiatorPeer = queueEntry.initiatorPeer();
@@ -2303,7 +2302,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo this.reference = reference;
}
public void run() {
- yacyClient.crawlReceipt(initiatorPeer, "crawl", "fill", "indexed", reference, "");
+ yacyClient.crawlReceipt(wordIndex.seedDB.mySeed(), initiatorPeer, "crawl", "fill", "indexed", reference, "");
}
}
@@ -2486,16 +2485,16 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo }
public String dhtShallTransfer() {
- if (yacyCore.seedDB == null) {
+ if (this.wordIndex.seedDB == null) {
return "no DHT distribution: seedDB == null";
}
- if (yacyCore.seedDB.mySeed() == null) {
+ if (this.wordIndex.seedDB.mySeed() == null) {
return "no DHT distribution: mySeed == null";
}
- if (yacyCore.seedDB.mySeed().isVirgin()) {
+ if (this.wordIndex.seedDB.mySeed().isVirgin()) {
return "no DHT distribution: status is virgin";
}
- if (yacyCore.seedDB.noDHTActivity()) {
+ if (this.wordIndex.seedDB.noDHTActivity()) {
return "no DHT distribution: network too small";
}
if (!this.getConfigBool("network.unit.dht", true)) {
@@ -2535,7 +2534,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo }
// do the transfer
- int peerCount = Math.max(1, (yacyCore.seedDB.mySeed().isJunior()) ?
+ int peerCount = Math.max(1, (this.wordIndex.seedDB.mySeed().isJunior()) ?
(int) getConfigLong("network.unit.dhtredundancy.junior", 1) :
(int) getConfigLong("network.unit.dhtredundancy.senior", 1)); // set redundancy factor
long starttime = System.currentTimeMillis();
@@ -2575,12 +2574,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo }
public boolean dhtTransferProcess(plasmaDHTChunk dhtChunk, int peerCount) {
- if ((yacyCore.seedDB == null) || (yacyCore.seedDB.sizeConnected() == 0)) return false;
+ if ((this.wordIndex.seedDB == null) || (this.wordIndex.seedDB.sizeConnected() == 0)) return false;
try {
// find a list of DHT-peers
double maxDist = 0.2;
- ArrayList<yacySeed> seeds = yacyCore.dhtAgent.getDHTTargets(log, peerCount, Math.min(8, (int) (yacyCore.seedDB.sizeConnected() * maxDist)), dhtChunk.firstContainer().getWordHash(), dhtChunk.lastContainer().getWordHash(), maxDist);
+ ArrayList<yacySeed> seeds = yacyCore.dhtAgent.getDHTTargets(wordIndex.seedDB, log, peerCount, Math.min(8, (int) (this.wordIndex.seedDB.sizeConnected() * maxDist)), dhtChunk.firstContainer().getWordHash(), dhtChunk.lastContainer().getWordHash(), maxDist);
if (seeds.size() < peerCount) {
log.logWarning("found not enough (" + seeds.size() + ") peers for distribution for dhtchunk [" + dhtChunk.firstContainer().getWordHash() + " .. " + dhtChunk.lastContainer().getWordHash() + "]");
return false;
@@ -2606,7 +2605,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo checkInterruption();
if (seedIter.hasNext()) {
- plasmaDHTTransfer t = new plasmaDHTTransfer(log, (yacySeed)seedIter.next(), dhtChunk,gzipBody,timeout,retries);
+ plasmaDHTTransfer t = new plasmaDHTTransfer(log, wordIndex.seedDB, (yacySeed)seedIter.next(), dhtChunk,gzipBody,timeout,retries);
t.start();
transfer.add(t);
} else {
@@ -2657,6 +2656,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<plasmaSwitchbo String failreason,
kelondroBitfield flags
) {
+ assert initiator != null;
// create a new errorURL DB entry
plasmaCrawlEntry bentry = new plasmaCrawlEntry(
initiator,
diff --git a/source/de/anomic/plasma/plasmaSwitchboardQueue.java b/source/de/anomic/plasma/plasmaSwitchboardQueue.java index 9278577d6..282290f1c 100644 --- a/source/de/anomic/plasma/plasmaSwitchboardQueue.java +++ b/source/de/anomic/plasma/plasmaSwitchboardQueue.java @@ -59,7 +59,6 @@ import de.anomic.kelondro.kelondroRow; import de.anomic.kelondro.kelondroStack;
import de.anomic.plasma.cache.IResourceInfo;
import de.anomic.server.logging.serverLog;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
@@ -68,14 +67,14 @@ public class plasmaSwitchboardQueue { kelondroStack sbQueueStack;
plasmaCrawlProfile profiles;
- plasmaWordIndex index;
+ plasmaWordIndex wordIndex;
private File sbQueueStackPath;
ConcurrentHashMap<String, QueueEntry> queueInProcess;
- public plasmaSwitchboardQueue(plasmaWordIndex index, File sbQueueStackPath, plasmaCrawlProfile profiles) {
+ public plasmaSwitchboardQueue(plasmaWordIndex wordIndex, File sbQueueStackPath, plasmaCrawlProfile profiles) {
this.sbQueueStackPath = sbQueueStackPath;
this.profiles = profiles;
- this.index = index;
+ this.wordIndex = wordIndex;
this.queueInProcess = new ConcurrentHashMap<String, QueueEntry>();
initQueueStack();
@@ -112,12 +111,12 @@ public class plasmaSwitchboardQueue { if (entry == null) return;
sbQueueStack.push(sbQueueStack.row().newEntry(new byte[][]{
entry.url.toString().getBytes(),
- (entry.referrerHash == null) ? yacyURL.dummyHash.getBytes() : entry.referrerHash.getBytes(),
+ (entry.referrerHash == null) ? "".getBytes() : entry.referrerHash.getBytes(),
kelondroBase64Order.enhancedCoder.encodeLong((entry.ifModifiedSince == null) ? 0 : entry.ifModifiedSince.getTime(), 11).getBytes(),
new byte[]{entry.flags},
- (entry.initiator == null) ? yacyURL.dummyHash.getBytes() : entry.initiator.getBytes(),
+ (entry.initiator == null) ? "".getBytes() : entry.initiator.getBytes(),
kelondroBase64Order.enhancedCoder.encodeLong((long) entry.depth, rowdef.width(5)).getBytes(),
- (entry.profileHandle == null) ? yacyURL.dummyHash.getBytes() : entry.profileHandle.getBytes(),
+ (entry.profileHandle == null) ? "".getBytes() : entry.profileHandle.getBytes(),
(entry.anchorName == null) ? "-".getBytes("UTF-8") : entry.anchorName.getBytes("UTF-8")
}));
}
@@ -326,7 +325,7 @@ public class plasmaSwitchboardQueue { }
public boolean proxy() {
- return (initiator == null) || (initiator.equals(yacyURL.dummyHash));
+ return (initiator == null) || (initiator.equals(initiator.length() == 0));
}
public String initiator() {
@@ -334,13 +333,13 @@ public class plasmaSwitchboardQueue { }
public yacySeed initiatorPeer() {
- if ((initiator == null) || (initiator.equals(yacyURL.dummyHash))) return null;
- if (initiator.equals(yacyCore.seedDB.mySeed().hash)) {
+ if ((initiator == null) || (initiator.length() == 0)) return null;
+ if (initiator.equals(wordIndex.seedDB.mySeed().hash)) {
// normal crawling
return null;
} else {
// this was done for remote peer (a global crawl)
- return yacyCore.seedDB.getConnected(initiator);
+ return wordIndex.seedDB.getConnected(initiator);
}
}
@@ -384,8 +383,8 @@ public class plasmaSwitchboardQueue { public yacyURL referrerURL() {
if (referrerURL == null) {
- if ((referrerHash == null) || (referrerHash.equals(yacyURL.dummyHash))) return null;
- indexURLReference entry = index.getURL(referrerHash, null, 0);
+ if ((referrerHash == null) || (referrerHash.equals(initiator.length() == 0))) return null;
+ indexURLReference entry = wordIndex.getURL(referrerHash, null, 0);
if (entry == null) referrerURL = null; else referrerURL = entry.comp().url();
}
return referrerURL;
@@ -408,10 +407,10 @@ public class plasmaSwitchboardQueue { // 5) local prefetch/crawling (initiator is own seedHash)
// 6) local fetching for global crawling (other known or unknwon initiator)
int processCase = plasmaSwitchboard.PROCESSCASE_0_UNKNOWN;
- if ((initiator == null) || (initiator.equals(yacyURL.dummyHash))) {
+ if ((initiator == null) || (initiator.equals(initiator.length() == 0))) {
// proxy-load
processCase = plasmaSwitchboard.PROCESSCASE_4_PROXY_LOAD;
- } else if ((initiator != null) && (initiator.equals(yacyCore.seedDB.mySeed().hash))) {
+ } else if ((initiator != null) && (initiator.equals(wordIndex.seedDB.mySeed().hash))) {
// normal crawling
processCase = plasmaSwitchboard.PROCESSCASE_5_LOCAL_CRAWLING;
} else {
diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index e43201fa0..50621c4df 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -63,6 +63,7 @@ import de.anomic.xml.RSSFeed; import de.anomic.xml.RSSMessage;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyDHTAction;
+import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
@@ -79,8 +80,12 @@ public final class plasmaWordIndex implements indexRI { private final indexCollectionRI collections; // new database structure to replace AssortmentCluster and FileCluster
private serverLog log;
final indexRepositoryReference referenceURL;
+ public yacySeedDB seedDB;
+ public yacyNewsPool newsPool;
- public plasmaWordIndex(File indexPrimaryRoot, File indexSecondaryRoot, String networkName, serverLog log) {
+
+
+ public plasmaWordIndex(String networkName, serverLog log, File indexPrimaryRoot, File indexSecondaryRoot) {
this.log = log;
File indexPrimaryPath = new File(indexPrimaryRoot, networkName);
File indexPrimaryTextLocation = new File(indexPrimaryPath, "TEXT");
@@ -113,6 +118,22 @@ public final class plasmaWordIndex implements indexRI { // create LURL-db
referenceURL = new indexRepositoryReference(indexSecondaryRoot, networkName);
+ // create or init seed cache
+ File networkRoot = new File(indexPrimaryPath, "NETWORK");
+ networkRoot.mkdirs();
+ File mySeedFile = new File(networkRoot, "mySeed.txt");
+ File oldSeedFile = new File(new File(indexPrimaryRoot.getParentFile(), "YACYDB"), "mySeed.txt");
+ if (oldSeedFile.exists()) oldSeedFile.renameTo(mySeedFile);
+ seedDB = new yacySeedDB(
+ new File(networkRoot, "seed.new.db"),
+ new File(networkRoot, "seed.old.db"),
+ new File(networkRoot, "seed.pot.db"),
+ mySeedFile
+ );
+
+ // create or init news database
+ newsPool = new yacyNewsPool(networkRoot);
+
}
public void putURL(indexURLReference entry) throws IOException {
@@ -236,7 +257,7 @@ public final class plasmaWordIndex implements indexRI { public void addEntry(String wordHash, indexRWIRowEntry entry, long updateTime, boolean dhtInCase) {
// set dhtInCase depending on wordHash
- if ((!dhtInCase) && (yacyDHTAction.shallBeOwnWord(wordHash))) dhtInCase = true;
+ if ((!dhtInCase) && (yacyDHTAction.shallBeOwnWord(seedDB, wordHash))) dhtInCase = true;
// add the entry
if (dhtInCase) {
@@ -256,7 +277,7 @@ public final class plasmaWordIndex implements indexRI { assert (entries.row().objectsize == indexRWIRowEntry.urlEntryRow.objectsize);
// set dhtInCase depending on wordHash
- if ((!dhtInCase) && (yacyDHTAction.shallBeOwnWord(entries.getWordHash()))) dhtInCase = true;
+ if ((!dhtInCase) && (yacyDHTAction.shallBeOwnWord(seedDB, entries.getWordHash()))) dhtInCase = true;
// add the entry
if (dhtInCase) {
@@ -508,8 +529,10 @@ public final class plasmaWordIndex implements indexRI { dhtOutCache.close();
collections.close();
referenceURL.close();
+ seedDB.close();
+ newsPool.close();
}
-
+
public indexContainer deleteContainer(String wordHash) {
indexContainer c = new indexContainer(
wordHash,
diff --git a/source/de/anomic/server/portForwarding/sch/serverPortForwardingSch.java b/source/de/anomic/server/portForwarding/sch/serverPortForwardingSch.java deleted file mode 100644 index d95deb75f..000000000 --- a/source/de/anomic/server/portForwarding/sch/serverPortForwardingSch.java +++ /dev/null @@ -1,321 +0,0 @@ -// serverPortForwardingSch.java
-// -------------------------------------
-// part of YACY
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2004
-//
-// This file ist contributed by Martin Thelian
-//
-// $LastChangedDate: 2006-02-20 23:57:42 +0100 (Mo, 20 Feb 2006) $
-// $LastChangedRevision: 1715 $
-// $LastChangedBy: borg-0300 $
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
-
-package de.anomic.server.portForwarding.sch;
-
-import java.io.IOException;
-
-import com.jcraft.jsch.JSch;
-import com.jcraft.jsch.ProxyHTTP;
-import com.jcraft.jsch.Session;
-import com.jcraft.jsch.UIKeyboardInteractive;
-import com.jcraft.jsch.UserInfo;
-
-import de.anomic.server.serverInstantBusyThread;
-import de.anomic.server.serverSwitch;
-import de.anomic.server.logging.serverLog;
-import de.anomic.server.portForwarding.serverPortForwarding;
-import de.anomic.yacy.yacyClient;
-import de.anomic.yacy.yacyCore;
-
-public class serverPortForwardingSch implements serverPortForwarding{
-
- /* ========================================================================
- * Constants needed to read properties from the configuration file
- * ======================================================================== */
- public static final String FORWARDING_HOST = "portForwarding.sch.Host";
- public static final String FORWARDING_HOST_PORT = "portForwarding.sch.HostPort";
- public static final String FORWARDING_HOST_USER = "portForwarding.sch.HostUser";
- public static final String FORWARDING_HOST_PWD = "portForwarding.sch.HostPwd";
-
- public static final String FORWARDING_PORT = "portForwarding.sch.Port";
- public static final String FORWARDING_USE_PROXY = "portForwarding.sch.UseProxy";
-
-
- /* ========================================================================
- * Other object fields
- * ======================================================================== */
- private serverSwitch<?> switchboard;
-
- private String forwardingHost;
- private int forwardingHostPort;
- private String forwardingHostUser;
- private String forwardingHostPwd;
-
- private int forwardingPort;
- private boolean useProxy;
-
- private String remoteProxyHost;
- private int remoteProxyPort;
-
- private String localHost;
- private int localHostPort;
-
- private static Session session = null;
- private static serverInstantBusyThread sessionWatcher = null;
-
- private final serverLog log;
-
- public serverPortForwardingSch() {
- super();
- this.log = new serverLog("PORT_FORWARDING_SCH");
- }
-
- public void init(
- serverSwitch<?> switchboard,
- String localHost,
- int localPort
- ) throws Exception {
- try {
- if (this.log.isFine()) this.log.logFine("Initializing port forwarding via sch ...");
-
- this.switchboard = switchboard;
-
- this.forwardingHost = switchboard.getConfig(FORWARDING_HOST,"localhost");
- this.forwardingHostPort = Integer.valueOf(switchboard.getConfig(FORWARDING_HOST_PORT,"8080")).intValue();
- this.forwardingHostUser = switchboard.getConfig(FORWARDING_HOST_USER,"xxx");
- this.forwardingHostPwd = switchboard.getConfig(FORWARDING_HOST_PWD,"xxx");
-
- this.forwardingPort = Integer.valueOf(switchboard.getConfig(FORWARDING_PORT,"8080")).intValue();
- this.useProxy = Boolean.valueOf(switchboard.getConfig(FORWARDING_USE_PROXY,"false")).booleanValue();
-
- this.localHost = localHost;
- this.localHostPort = localPort;
-
- // load remote proxy data
- this.remoteProxyHost = switchboard.getConfig("remoteProxyHost","");
- try {
- this.remoteProxyPort = Integer.parseInt(switchboard.getConfig("remoteProxyPort","3128"));
- } catch (NumberFormatException e) {
- remoteProxyPort = 3128;
- }
-
- // checking if all needed libs are availalbe
- String javaClassPath = System.getProperty("java.class.path");
- if (javaClassPath.indexOf("jsch") == -1) {
- throw new IllegalStateException("Missing library.");
- }
- } catch (Exception e) {
- this.log.logSevere("Unable to initialize port forwarding.",e);
- throw e;
- }
- }
-
- public String getHost() {
- return this.forwardingHost;
- }
-
- public int getPort() {
- return this.forwardingPort;
- }
-
- public synchronized void connect() throws IOException {
- try{
- if ((session != null) && (session.isConnected()))
- throw new IOException("Session already connected");
-
- this.log.logInfo("Trying to connect to remote port forwarding host " + this.forwardingHostUser + "@" + this.forwardingHost + ":" + this.forwardingHostPort);
-
- JSch jsch=new JSch();
- session=jsch.getSession(this.forwardingHostUser, this.forwardingHost, this.forwardingHostPort);
- session.setPassword(this.forwardingHostPwd);
-
- /*
- * Setting the StrictHostKeyChecking to ignore unknown
- * hosts because of a missing known_hosts file ...
- */
- java.util.Properties config = new java.util.Properties();
- config.put("StrictHostKeyChecking","no");
- session.setConfig(config);
-
- // setting the proxy that should be used
- if (this.useProxy) {
- session.setProxy(new ProxyHTTP(this.remoteProxyHost, this.remoteProxyPort));
- }
-
- // username and password will be given via UserInfo interface.
- UserInfo ui= new MyUserInfo(this.forwardingHostPwd);
- session.setUserInfo(ui);
-
- // trying to connect ...
- session.connect();
-
- // activating remote port forwarding
- session.setPortForwardingR(this.forwardingPort, this.localHost, this.localHostPort);
-
- // using a timer task to control if the session remains open
- if (sessionWatcher == null) {
- this.log.logFine("Deploying port forwarding session watcher thread.");
- this.switchboard.deployThread("portForwardingWatcher", "Remote Port Forwarding Watcher", "this thread is used to detect broken connections and to re-establish it if necessary.", null,
- sessionWatcher = new serverInstantBusyThread(this, "reconnect", null, null), 30000,30000,30000,1000);
- }
-
- this.log.logInfo("Remote port forwarding connection established: " +
- this.forwardingHost+ ":" + this.forwardingPort + " -> " +
- this.localHost + ":" + this.localHostPort);
- }
- catch(Exception e){
- this.log.logSevere("Unable to connect to remote port forwarding host.",e);
- throw new IOException(e.getMessage());
- }
- }
-
- public synchronized boolean reconnect() throws IOException {
- if ((!this.isConnected()) && (!Thread.currentThread().isInterrupted())) {
- if (this.log.isFine()) this.log.logFine("Trying to reconnect to port forwarding host.");
- this.disconnect();
- this.connect();
- return this.isConnected();
- }
- return false;
- }
-
- public synchronized void disconnect() throws IOException {
- if (session == null) throw new IOException("No connection established.");
-
- // terminating port watcher thread
- this.log.logFine("Terminating port forwarding session watcher thread.");
- this.switchboard.terminateThread("portForwardingWatcher",true);
- sessionWatcher = null;
-
- // disconnection the session
- try {
- session.disconnect();
- this.log.logFine("Successfully disconnected from port forwarding host.");
- } catch (Exception e) {
- this.log.logSevere("Error while trying to disconnect from port forwarding host.",e);
- throw new IOException(e.getMessage());
- }
- }
-
- public synchronized boolean isConnected() {
- if (session == null) return false;
- if (!session.isConnected()) return false;
- int urls = yacyClient.queryUrlCount(yacyCore.seedDB.mySeed());
- return !(urls < 0);
- }
-
- class MyUserInfo
- implements UserInfo, UIKeyboardInteractive {
- String passwd;
-
- public MyUserInfo(String password) {
- this.passwd = password;
- }
-
- public String getPassword() {
- return this.passwd;
- }
-
- public boolean promptYesNo(String str){
- System.err.println("User was prompted from: " + str);
- return true;
- }
-
- public String getPassphrase() {
- return null;
- }
-
- public boolean promptPassphrase(String message) {
- System.out.println("promptPassphrase : " + message);
- return false;
- }
-
- public boolean promptPassword(String message) {
- System.out.println("promptPassword : " + message);
- return true;
- }
-
- /**
- * @see com.jcraft.jsch.UserInfo#showMessage(java.lang.String)
- */
- public void showMessage(String message) {
- System.out.println("Sch has tried to show the following message to the user: " + message);
- }
-
- public String[] promptKeyboardInteractive(String destination,
- String name,
- String instruction,
- String[] prompt,
- boolean[] echo) {
- System.out.println("User was prompted using interactive-keyboard: " +
- "\n\tDestination: " + destination +
- "\n\tName: " + name +
- "\n\tInstruction: " + instruction +
- "\n\tPrompt: " + arrayToString2(prompt,"|") +
- "\n\techo: " + arrayToString2(echo,"|"));
-
- if ((prompt.length >= 1) && (prompt[0].startsWith("Password")))
- return new String[]{this.passwd};
- return new String[]{};
- }
-
- String arrayToString2(String[] a, String separator) {
- StringBuffer result = new StringBuffer(); // start with first element
- if (a.length > 0) {
- result.append(a[0]);
- for (int i=1; i<a.length; i++) {
- result.append(separator);
- result.append(a[i]);
- }
- }
- return result.toString();
- }
-
- String arrayToString2(boolean[] a, String separator) {
- StringBuffer result = new StringBuffer(); // start with first element
- if (a.length > 0) {
- result.append(a[0]);
- for (int i=1; i<a.length; i++) {
- result.append(separator);
- result.append(a[i]);
- }
- }
- return result.toString();
- }
- }
-
-}
diff --git a/source/de/anomic/server/portForwarding/serverPortForwarding.java b/source/de/anomic/server/portForwarding/serverPortForwarding.java deleted file mode 100644 index 64170a448..000000000 --- a/source/de/anomic/server/portForwarding/serverPortForwarding.java +++ /dev/null @@ -1,63 +0,0 @@ -// serverPortForwarding.java
-// -------------------------------------
-// part of YACY
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2004
-//
-// This file ist contributed by Martin Thelian
-//
-// $LastChangedDate: 2006-02-20 23:57:42 +0100 (Mo, 20 Feb 2006) $
-// $LastChangedRevision: 1715 $
-// $LastChangedBy: borg-0300 $
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
-
-package de.anomic.server.portForwarding;
-
-import java.io.IOException;
-
-import de.anomic.server.serverSwitch;
-
-public interface serverPortForwarding {
- public void init(serverSwitch<?> switchboard, String localHost, int localPort) throws Exception;
-
- public String getHost();
- public int getPort();
-
- public void connect() throws IOException;
- public void disconnect() throws IOException;
- public boolean reconnect() throws IOException;
- public boolean isConnected();
-}
diff --git a/source/de/anomic/server/portForwarding/upnp/serverPortForwardingUpnp.java b/source/de/anomic/server/portForwarding/upnp/serverPortForwardingUpnp.java deleted file mode 100644 index c390cc4a0..000000000 --- a/source/de/anomic/server/portForwarding/upnp/serverPortForwardingUpnp.java +++ /dev/null @@ -1,220 +0,0 @@ -// serverPortForwardingUpnp.java
-// -------------------------------------
-// part of YACY
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2004
-//
-// This file ist contributed by Martin Thelian
-//
-// $LastChangedDate: 2006-02-20 23:57:42 +0100 (Mo, 20 Feb 2006) $
-// $LastChangedRevision: 1715 $
-// $LastChangedBy: theli $
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
-
-
-package de.anomic.server.portForwarding.upnp;
-
-import java.io.IOException;
-import java.net.InetAddress;
-
-import net.sbbi.upnp.impls.InternetGatewayDevice;
-import net.sbbi.upnp.messages.ActionResponse;
-import net.sbbi.upnp.messages.UPNPResponseException;
-import de.anomic.server.serverSwitch;
-import de.anomic.server.logging.serverLog;
-import de.anomic.server.portForwarding.serverPortForwarding;
-
-public class serverPortForwardingUpnp implements serverPortForwarding {
-
- private InternetGatewayDevice gateway;
- private String localHost;
- private int localHostPort;
- private String externalAddress = null;
- private serverLog log;
-
- public serverPortForwardingUpnp() {
- super();
- this.log = new serverLog("PORT_FORWARDING_UPNP");
- }
-
- public void connect() throws IOException {
- try {
- if (this.gateway != null)
- throw new IOException("Session already connected");
-
- int timeout = 8000;
- try {
- // trying to get all internet gateways on the local network
- this.log.logInfo("Trying to find all available internet gateways.");
- InternetGatewayDevice[] IGDs = InternetGatewayDevice.getDevices(timeout);
- boolean mapped = false;
- if ( IGDs != null ) {
- for ( int i = 0; i < IGDs.length; i++ ) {
-
- this.gateway = IGDs[i];
- this.log.logInfo("Found device " + this.gateway.getIGDRootDevice().getModelName() );
- this.log.logInfo("NAT table size is " + this.gateway.getNatTableSize() );
-
- // now let's open the port
- this.log.logInfo("Adding port mapping ...");
- mapped = this.gateway.addPortMapping(
- "YaCy port forwarding",
- null,
- this.localHostPort,
- this.localHostPort,
- this.localHost,
- 0,
- "TCP"
- );
-
- if ( mapped ) {
- this.log.logInfo("Gateway port " + this.localHostPort + " mapped to " + this.localHost );
- this.log.logInfo("Current mappings count is " + this.gateway.getNatMappingsCount() );
- mapped = isConnected();
- }
- if (mapped) break;
- }
- if (!mapped) {
- throw new IOException("Unable to configure the port mapping.");
- }
- } else {
- throw new IOException("No internet gateway device found.");
- }
-
- } catch ( IOException ex ) {
- throw new Exception("IOException occured during discovery or ports mapping. " + ex.getMessage());
- } catch( UPNPResponseException respEx ) {
- throw new Exception("UPNP device unhappy " + respEx.getDetailErrorCode() + " " + respEx.getDetailErrorDescription() );
- }
- } catch (Exception e) {
- this.gateway = null;
- this.log.logSevere("Unable to connect to remote port forwarding host. ",e);
- throw new IOException(e.getMessage());
- }
- }
-
- public boolean routerAvailable(int timeout) {
- try {
- InternetGatewayDevice[] IGDs = InternetGatewayDevice.getDevices(timeout);
- return (IGDs != null) && (IGDs.length > 0);
- } catch (Exception e) {
- this.log.logSevere("Unable to determine available routers: " + e.getMessage());
- return false;
- }
- }
-
- public void disconnect() throws IOException {
- if (this.gateway == null) throw new IOException("No connection established.");
-
- boolean unmapped;
- try {
- this.log.logInfo("Trying to disable port mapping ...");
- unmapped = this.gateway.deletePortMapping( null, this.localHostPort, "TCP" );
- if ( unmapped ) {
- this.log.logInfo("Port mapping disabled");
- }
- } catch (UPNPResponseException e) {
- new IOException("Unable to disable port forwarding. " + e.getMessage());
- }
- }
-
- public String getHost() {
- if (this.gateway == null) return "";
- if (this.externalAddress == null) {
- try {
- this.externalAddress = this.gateway.getExternalIPAddress();
- } catch (Exception e) {
- this.log.logWarning("Unable to get the external address of the gateway");
- }
- }
- return this.externalAddress;
- }
-
- public int getPort() {
- return this.localHostPort;
- }
-
- public void init(serverSwitch<?> switchboard, String localHost, int localPort)
- throws Exception {
- try {
- this.log.logFine("Initializing port forwarding via UPnP ...");
-
- if (localHost.equals("0.0.0.0")) {
- this.localHost = InetAddress.getLocalHost().getHostAddress();
- } else {
- this.localHost = localHost;
- }
- this.localHostPort = localPort;
-
- // checking if all needed libs are availalbe
- String javaClassPath = System.getProperty("java.class.path");
- if (javaClassPath.indexOf("sbbi-upnplib") == -1) {
- throw new IllegalStateException("Missing library.");
- }
-
- // setting the proper xml parser
-// if (System.getProperty("javax.xml.parsers.DocumentBuilderFactory", "").equals("")) {
-// System.setProperty("javax.xml.parsers.DocumentBuilderFactory", "");
-// }
-
- } catch (Exception e) {
- this.log.logSevere("Unable to initialize port forwarding.",e);
- throw e;
- }
- }
-
- public boolean isConnected() {
- if (this.gateway == null) return false;
- try {
- ActionResponse resp = this.gateway.getSpecificPortMappingEntry( null, this.localHostPort, "TCP" );
- return ( resp != null );
- } catch (Exception e) {
- this.log.logSevere("Unable to determine the connection status");
- return false;
- }
- }
-
- public boolean reconnect() throws IOException {
- if (!this.isConnected()) {
- this.log.logFine("Trying to reconnect to port forwarding host.");
- this.disconnect();
- this.connect();
- return this.isConnected();
- }
- return false;
- }
-
-}
diff --git a/source/de/anomic/server/serverCore.java b/source/de/anomic/server/serverCore.java index d5e2d02d1..1954e7c20 100644 --- a/source/de/anomic/server/serverCore.java +++ b/source/de/anomic/server/serverCore.java @@ -77,11 +77,8 @@ import javax.net.ssl.SSLSocketFactory; import de.anomic.icap.icapd;
import de.anomic.server.logging.serverLog;
-import de.anomic.server.portForwarding.serverPortForwarding;
import de.anomic.tools.PKCS12Tool;
import de.anomic.urlRedirector.urlRedirectord;
-import de.anomic.yacy.yacyCore;
-import de.anomic.yacy.yacySeed;
public final class serverCore extends serverAbstractBusyThread implements serverBusyThread {
@@ -142,9 +139,7 @@ public final class serverCore extends serverAbstractBusyThread implements server */
public boolean forceRestart = false;
- public static boolean portForwardingEnabled = false;
public static boolean useStaticIP = false;
- public static serverPortForwarding portForwarding = null;
private SSLSocketFactory sslSocketFactory = null;
private ServerSocket socket; // listener
@@ -261,25 +256,13 @@ public final class serverCore extends serverAbstractBusyThread implements server // updating the port information
//yacyCore.seedDB.mySeed.put(yacySeed.PORT,Integer.toString(bindAddress.getPort()));
- yacyCore.seedDB.mySeed().put(yacySeed.PORT, extendedPort);
+ //yacyCore.seedDB.mySeed().put(yacySeed.PORT, extendedPort);
} catch (Exception e) {
String errorMsg = "FATAL ERROR: " + e.getMessage() + " - probably root access rights needed. check port number";
this.log.logSevere(errorMsg);
System.out.println(errorMsg);
System.exit(0);
}
-
- // init port forwarding
- try {
- this.initPortForwarding();
- } catch (Exception e) {
- this.log.logSevere("Unable to initialize server port forwarding.",e);
- this.switchboard.setConfig("portForwardingEnabled","false");
- } catch (Error e) {
- this.log.logSevere("Unable to initialize server port forwarding.",e);
- this.switchboard.setConfig("portForwardingEnabled","false");
- }
-
}
public static int getPortNr(String extendedPortString) {
@@ -340,57 +323,6 @@ public final class serverCore extends serverAbstractBusyThread implements server : new InetSocketAddress(bindIP, bindPort);
}
- public void initPortForwarding() throws Exception {
- // doing the port forwarding stuff
- if (this.switchboard.getConfigBool("portForwarding.Enabled",false)) {
- this.log.logInfo("Initializing port forwarding ...");
- try {
- // getting the port forwarding type to use
- String forwardingType = this.switchboard.getConfig("portForwarding.Type","none");
-
- // loading port forwarding class
- this.log.logInfo("Trying to load port forwarding class for forwarding type '" + forwardingType + "'.");
- String forwardingClass = this.switchboard.getConfig("portForwarding." + forwardingType ,"");
-
- Class<?> forwarderClass = Class.forName(forwardingClass);
- serverCore.portForwarding = (serverPortForwarding) forwarderClass.newInstance();
-
- // initializing port forwarding
- String localHost = this.socket.getInetAddress().getHostName();
- Integer localPort = new Integer(this.socket.getLocalPort());
-
- serverCore.portForwarding.init(
- this.switchboard,
- localHost,
- localPort.intValue());
-
- // connection to port forwarding host
- serverCore.portForwarding.connect();
-
- serverCore.portForwardingEnabled = true;
- yacyCore.seedDB.mySeed().put(yacySeed.IP, serverDomains.myPublicIP());
- yacyCore.seedDB.mySeed().put(yacySeed.PORT,Integer.toString(serverCore.portForwarding.getPort()));
- } catch (Exception e) {
- serverCore.portForwardingEnabled = false;
- this.switchboard.setConfig("portForwarding.Enabled", "false");
- throw e;
- } catch (Error e) {
- serverCore.portForwardingEnabled = false;
- this.switchboard.setConfig("portForwarding.Enabled", "false");
- throw e;
- }
-
- } else {
- serverCore.portForwardingEnabled = false;
- serverCore.portForwarding = null;
- yacyCore.seedDB.mySeed().put(yacySeed.IP, serverDomains.myPublicIP());
- yacyCore.seedDB.mySeed().put(yacySeed.PORT,Integer.toString(serverCore.getPortNr(this.switchboard.getConfig("port", "8080"))));
- }
- if(! this.switchboard.getConfig("staticIP", "").equals(""))
- serverCore.useStaticIP=true;
-
- }
-
public void open() {
this.log.logConfig("* server started on " + serverDomains.myPublicLocalIP() + ":" + this.extendedPort);
}
@@ -494,18 +426,6 @@ public final class serverCore extends serverAbstractBusyThread implements server }
}
- // closing the port forwarding channel
- if ((portForwardingEnabled) && (portForwarding != null) ) {
- try {
- this.log.logInfo("Shutdown port forwarding ...");
- portForwarding.disconnect();
- portForwardingEnabled = false;
- portForwarding = null;
- } catch (Exception e) {
- this.log.logWarning("Unable to shutdown the port forwarding channel.");
- }
- }
-
// close the serverchannel and socket
try {
this.log.logInfo("Closing server socket ...");
diff --git a/source/de/anomic/server/serverDomains.java b/source/de/anomic/server/serverDomains.java index 009888ad3..0ab62b394 100644 --- a/source/de/anomic/server/serverDomains.java +++ b/source/de/anomic/server/serverDomains.java @@ -596,34 +596,19 @@ public class serverDomains { } public static String myPublicIP() { - try { - - // if a static IP was configured, we have to return it here ... - plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard(); - if (sb != null) { - String staticIP = sb.getConfig("staticIP", ""); - if ((!staticIP.equals(""))) { - return staticIP; - } + // if a static IP was configured, we have to return it here ... + plasmaSwitchboard sb = plasmaSwitchboard.getSwitchboard(); + if (sb != null) { + String staticIP = sb.getConfig("staticIP", ""); + if ((!staticIP.equals(""))) { + return staticIP; } - - // If port forwarding was enabled we need to return the remote IP - // Address - if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) { - // does not return serverCore.portForwarding.getHost(), because - // hostnames are not valid, except in DebugMode - return InetAddress.getByName( - serverCore.portForwarding.getHost()).getHostAddress(); - } - - // otherwise we return the real IP address of this host - InetAddress pLIP = myPublicLocalIP(); - if (pLIP != null) return pLIP.getHostAddress(); - return null; - } catch (java.net.UnknownHostException e) { - System.err.println("ERROR: (internal) " + e.getMessage()); - return null; } + + // otherwise we return the real IP address of this host + InetAddress pLIP = myPublicLocalIP(); + if (pLIP != null) return pLIP.getHostAddress(); + return null; } public static InetAddress myPublicLocalIP() { diff --git a/source/de/anomic/urlRedirector/urlRedirectord.java b/source/de/anomic/urlRedirector/urlRedirectord.java index 2765f654b..b589edb93 100644 --- a/source/de/anomic/urlRedirector/urlRedirectord.java +++ b/source/de/anomic/urlRedirector/urlRedirectord.java @@ -17,7 +17,6 @@ import de.anomic.server.serverCore; import de.anomic.server.serverHandler;
import de.anomic.server.logging.serverLog;
import de.anomic.server.serverCore.Session;
-import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyURL;
public class urlRedirectord implements serverHandler {
@@ -199,7 +198,7 @@ public class urlRedirectord implements serverHandler { reasonString = switchboard.crawlStacker.stackCrawl(
reqURL,
null,
- yacyCore.seedDB.mySeed().hash,
+ switchboard.wordIndex.seedDB.mySeed().hash,
"URL Redirector",
new Date(),
0,
diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 756938f63..ffba9a83b 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -91,7 +91,7 @@ import de.anomic.xml.RSSReader; public final class yacyClient {
- public static int publishMySeed(String address, String otherHash) {
+ public static int publishMySeed(yacySeed mySeed, String address, String otherHash) {
// this is called to enrich the seed information by
// - own address (if peer is behind a nat/router)
// - check peer type (virgin/junior/senior/principal)
@@ -116,7 +116,7 @@ public final class yacyClient { for (int retry = 0; retry < 4; retry++) try {
// generate request
post.add(new StringPart("count", "20"));
- post.add(new StringPart("seed", yacyCore.seedDB.mySeed().genSeedStr(salt)));
+ post.add(new StringPart("seed", mySeed.genSeedStr(salt)));
yacyCore.log.logFine("yacyClient.publishMySeed thread '" + Thread.currentThread().getName() + "' contacting peer at " + address);
// send request
byte[] content = wput("http://" + address + "/yacy/hello.html", yacySeed.b64Hash2hexHash(otherHash) + ".yacyh", post);
@@ -141,7 +141,6 @@ public final class yacyClient { // check consistency with expectation
yacySeed otherPeer = null;
- float otherPeerVersion = 0;
String seed;
if ((otherHash != null) &&
(otherHash.length() > 0) &&
@@ -154,72 +153,50 @@ public final class yacyClient { yacyCore.log.logFine("yacyClient.publishMySeed: consistency error: other peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' wrong");
return -1; // no success
}
- otherPeerVersion = otherPeer.getVersion();
}
}
// set my own seed according to new information
- // we overwrite our own IP number only, if we do not portForwarding
- if (serverCore.portForwardingEnabled || serverCore.useStaticIP) {
- yacyCore.seedDB.mySeed().put(yacySeed.IP, serverDomains.myPublicIP());
+ // we overwrite our own IP number only
+ if (serverCore.useStaticIP) {
+ mySeed.put(yacySeed.IP, serverDomains.myPublicIP());
} else {
- yacyCore.seedDB.mySeed().put(yacySeed.IP, (String) result.get("yourip"));
+ mySeed.put(yacySeed.IP, (String) result.get("yourip"));
}
- /* If we have port forwarding enabled but the other peer uses a too old yacy version
- * we can ignore the seed-type that was reported by the peer.
- *
- * Otherwise we have to change our seed-type
- *
- * @see serverCore#portForwardingEnabled
- */
- if (!serverCore.portForwardingEnabled || otherPeerVersion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING) {
- String mytype = (String) result.get(yacySeed.YOURTYPE);
- if (mytype == null) { mytype = ""; }
- yacyAccessible accessible = new yacyAccessible();
- if (mytype.equals(yacySeed.PEERTYPE_SENIOR)||mytype.equals(yacySeed.PEERTYPE_PRINCIPAL)) {
- accessible.IWasAccessed = true;
- if (yacyCore.seedDB.mySeed().isPrincipal()) {
- mytype = yacySeed.PEERTYPE_PRINCIPAL;
- }
- } else {
- accessible.IWasAccessed = false;
+ // change our seed-type
+ String mytype = (String) result.get(yacySeed.YOURTYPE);
+ if (mytype == null) { mytype = ""; }
+ yacyAccessible accessible = new yacyAccessible();
+ if (mytype.equals(yacySeed.PEERTYPE_SENIOR)||mytype.equals(yacySeed.PEERTYPE_PRINCIPAL)) {
+ accessible.IWasAccessed = true;
+ if (mySeed.isPrincipal()) {
+ mytype = yacySeed.PEERTYPE_PRINCIPAL;
}
- accessible.lastUpdated = System.currentTimeMillis();
- yacyCore.amIAccessibleDB.put(otherHash, accessible);
+ } else {
+ accessible.IWasAccessed = false;
+ }
+ accessible.lastUpdated = System.currentTimeMillis();
+ yacyCore.amIAccessibleDB.put(otherHash, accessible);
- /*
- * If we were reported as junior we have to check if your port forwarding channel is broken
- * If this is true we try to reconnect the sch channel to the remote server now.
- */
- if (mytype.equalsIgnoreCase(yacySeed.PEERTYPE_JUNIOR)) {
- yacyCore.log.logInfo("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as junior.");
- if (serverCore.portForwardingEnabled) {
- if (!Thread.currentThread().isInterrupted() &&
- serverCore.portForwarding != null &&
- !serverCore.portForwarding.isConnected()
- ) {
- yacyCore.log.logWarning("yacyClient.publishMySeed: Broken portForwarding channel detected. Trying to reconnect ...");
- try {
- serverCore.portForwarding.reconnect();
- } catch (IOException e) {
- yacyCore.log.logWarning("yacyClient.publishMySeed: Unable to reconnect to port forwarding host.");
- }
- }
- }
- } else if ((mytype.equalsIgnoreCase(yacySeed.PEERTYPE_SENIOR)) ||
- (mytype.equalsIgnoreCase(yacySeed.PEERTYPE_PRINCIPAL))) {
- yacyCore.log.logFine("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as " + mytype + ", accepted other peer.");
- } else {
- // wrong type report
- yacyCore.log.logFine("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as " + mytype + ", rejecting other peer.");
- return -1;
- }
- if (yacyCore.seedDB.mySeed().orVirgin().equals(yacySeed.PEERTYPE_VIRGIN))
- yacyCore.seedDB.mySeed().put(yacySeed.PEERTYPE, mytype);
+ /*
+ * If we were reported as junior we have to check if your port forwarding channel is broken
+ * If this is true we try to reconnect the sch channel to the remote server now.
+ */
+ if (mytype.equalsIgnoreCase(yacySeed.PEERTYPE_JUNIOR)) {
+ yacyCore.log.logInfo("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as junior.");
+ } else if ((mytype.equalsIgnoreCase(yacySeed.PEERTYPE_SENIOR)) ||
+ (mytype.equalsIgnoreCase(yacySeed.PEERTYPE_PRINCIPAL))) {
+ yacyCore.log.logFine("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as " + mytype + ", accepted other peer.");
+ } else {
+ // wrong type report
+ yacyCore.log.logFine("yacyClient.publishMySeed: Peer '" + ((otherPeer==null)?"unknown":otherPeer.getName()) + "' reported us as " + mytype + ", rejecting other peer.");
+ return -1;
}
+ if (mySeed.orVirgin().equals(yacySeed.PEERTYPE_VIRGIN))
+ mySeed.put(yacySeed.PEERTYPE, mytype);
- final String error = yacyCore.seedDB.mySeed().isProper();
+ final String error = mySeed.isProper();
if (error != null) {
yacyCore.log.logSevere("yacyClient.publishMySeed mySeed error - not proper: " + error);
return -1;
@@ -328,8 +305,8 @@ public final class yacyClient { private static byte[] postToFile(final yacySeed target, final String filename, final List<Part> post) throws IOException {
return wput("http://" + target.getClusterAddress() + "/yacy/" + filename, target.getHexHash() + ".yacyh", post);
}
- private static byte[] postToFile(final String targetHash, final String filename, final List<Part> post) throws IOException {
- return wput("http://" + targetAddress(targetHash) + "/yacy/" + filename, yacySeed.b64Hash2hexHash(targetHash)+ ".yacyh", post);
+ private static byte[] postToFile(yacySeedDB seedDB, final String targetHash, final String filename, final List<Part> post) throws IOException {
+ return wput("http://" + targetAddress(seedDB, targetHash) + "/yacy/" + filename, yacySeed.b64Hash2hexHash(targetHash)+ ".yacyh", post);
}
public static yacySeed querySeed(yacySeed target, String seedHash) {
@@ -376,7 +353,6 @@ public final class yacyClient { public static int queryUrlCount(yacySeed target) {
if (target == null) { return -1; }
- if (yacyCore.seedDB.mySeed() == null) return -1;
// prepare request
final String salt = crypt.randomSalt();
@@ -405,10 +381,9 @@ public final class yacyClient { }
}
- public static RSSFeed queryRemoteCrawlURLs(yacySeed target, int count) {
+ public static RSSFeed queryRemoteCrawlURLs(yacySeedDB seedDB, yacySeed target, int count) {
// returns a list of
if (target == null) { return null; }
- if (yacyCore.seedDB.mySeed() == null) return null;
// prepare request
final String salt = crypt.randomSalt();
@@ -427,7 +402,7 @@ public final class yacyClient { yacyCore.log.logWarning("yacyClient.queryRemoteCrawlURLs failed asking peer '" + target.getName() + "': probably bad response from remote peer");
System.out.println("***DEBUG*** rss input = " + new String(result));
target.put(yacySeed.RCOUNT, "0");
- yacyCore.seedDB.update(target.hash, target); // overwrite number of remote-available number to avoid that this peer is called again (until update is done by peer ping)
+ seedDB.update(target.hash, target); // overwrite number of remote-available number to avoid that this peer is called again (until update is done by peer ping)
//e.printStackTrace();
return null;
}
@@ -439,6 +414,7 @@ public final class yacyClient { }
public static String[] search(
+ yacySeed mySeed,
String wordhashes,
String excludehashes,
String urlhashes,
@@ -476,7 +452,7 @@ public final class yacyClient { // prepare request
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), target.hash, salt);
- post.add(new StringPart("myseed", yacyCore.seedDB.mySeed().genSeedStr(salt)));
+ post.add(new StringPart("myseed", mySeed.genSeedStr(salt)));
post.add(new StringPart("count", Integer.toString(Math.max(10, count))));
post.add(new StringPart("resource", ((global) ? "global" : "local")));
post.add(new StringPart("partitions", Integer.toString(partitions)));
@@ -590,7 +566,7 @@ public final class yacyClient { // passed all checks, store url
try {
wordIndex.putURL(urlEntry);
- crawlResults.stack(urlEntry, yacyCore.seedDB.mySeed().hash, target.hash, 2);
+ crawlResults.stack(urlEntry, mySeed.hash, target.hash, 2);
} catch (IOException e) {
yacyCore.log.logSevere("could not store search result", e);
continue; // db-error
@@ -681,11 +657,10 @@ public final class yacyClient { return urls;
}
- public static HashMap<String, String> permissionMessage(String targetHash) {
+ public static HashMap<String, String> permissionMessage(yacySeedDB seedDB, String targetHash) {
// ask for allowed message size and attachement size
// if this replies null, the peer does not answer
- if (yacyCore.seedDB == null || yacyCore.seedDB.mySeed() == null) { return null; }
-
+
// prepare request
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), targetHash, salt);
@@ -693,7 +668,7 @@ public final class yacyClient { // send request
try {
- final byte[] content = postToFile(targetHash, "message.html", post);
+ final byte[] content = postToFile(seedDB, targetHash, "message.html", post);
final HashMap<String, String> result = nxTools.table(content, "UTF-8");
return result;
} catch (Exception e) {
@@ -703,14 +678,14 @@ public final class yacyClient { }
}
- public static HashMap<String, String> postMessage(String targetHash, String subject, byte[] message) {
+ public static HashMap<String, String> postMessage(yacySeedDB seedDB, String targetHash, String subject, byte[] message) {
// this post a message to the remote message board
// prepare request
final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), targetHash, salt);
post.add(new StringPart("process", "post"));
- post.add(new StringPart("myseed", yacyCore.seedDB.mySeed().genSeedStr(salt)));
+ post.add(new StringPart("myseed", seedDB.mySeed().genSeedStr(salt)));
post.add(new StringPart("subject", subject));
try {
post.add(new StringPart("message", new String(message, "UTF-8")));
@@ -720,7 +695,7 @@ public final class yacyClient { // send request
try {
- final byte[] content = postToFile(targetHash, "message.html", post);
+ final byte[] content = postToFile(seedDB, targetHash, "message.html", post);
final HashMap<String, String> result = nxTools.table(content, "UTF-8");
return result;
} catch (Exception e) {
@@ -729,13 +704,13 @@ public final class yacyClient { }
}
- public static String targetAddress(String targetHash) {
+ public static String targetAddress(yacySeedDB seedDB, String targetHash) {
// find target address
String address;
- if (targetHash.equals(yacyCore.seedDB.mySeed().hash)) {
- address = yacyCore.seedDB.mySeed().getClusterAddress();
+ if (targetHash.equals(seedDB.mySeed().hash)) {
+ address = seedDB.mySeed().getClusterAddress();
} else {
- final yacySeed targetSeed = yacyCore.seedDB.getConnected(targetHash);
+ final yacySeed targetSeed = seedDB.getConnected(targetHash);
if (targetSeed == null) { return null; }
address = targetSeed.getClusterAddress();
}
@@ -818,10 +793,10 @@ public final class yacyClient { return "wrong protocol: " + protocol;
}
- public static HashMap<String, String> crawlReceipt(yacySeed target, String process, String result, String reason, indexURLReference entry, String wordhashes) {
+ public static HashMap<String, String> crawlReceipt(yacySeed mySeed, yacySeed target, String process, String result, String reason, indexURLReference entry, String wordhashes) {
assert (target != null);
- assert (yacyCore.seedDB.mySeed() != null);
- assert (yacyCore.seedDB.mySeed() != target);
+ assert (mySeed != null);
+ assert (mySeed != target);
/*
the result can have one of the following values:
@@ -868,7 +843,7 @@ public final class yacyClient { }
}
- public static HashMap<String, Object> transferIndex(yacySeed targetSeed, indexContainer[] indexes, HashMap<String, indexURLReference> urlCache, boolean gzipBody, int timeout) {
+ public static HashMap<String, Object> transferIndex(yacySeedDB seedDB, yacySeed targetSeed, indexContainer[] indexes, HashMap<String, indexURLReference> urlCache, boolean gzipBody, int timeout) {
HashMap<String, Object> resultObj = new HashMap<String, Object>();
int payloadSize = 0;
@@ -888,7 +863,7 @@ public final class yacyClient { }
// transfer the RWI without the URLs
- HashMap<String, String> in = transferRWI(targetSeed, indexes, gzipBody, timeout);
+ HashMap<String, String> in = transferRWI(seedDB, targetSeed, indexes, gzipBody, timeout);
resultObj.put("resultTransferRWI", in);
if (in == null) {
@@ -904,7 +879,7 @@ public final class yacyClient { }
if (!(result.equals("ok"))) {
targetSeed.setFlagAcceptRemoteIndex(false);
- yacyCore.seedDB.update(targetSeed.hash, targetSeed);
+ seedDB.update(targetSeed.hash, targetSeed);
resultObj.put("result", result);
return resultObj;
}
@@ -929,7 +904,7 @@ public final class yacyClient { }
}
- in = transferURL(targetSeed, urls, gzipBody, timeout);
+ in = transferURL(seedDB, targetSeed, urls, gzipBody, timeout);
resultObj.put("resultTransferURL", in);
if (in == null) {
@@ -945,7 +920,7 @@ public final class yacyClient { }
if (!(result.equals("ok"))) {
targetSeed.setFlagAcceptRemoteIndex(false);
- yacyCore.seedDB.update(targetSeed.hash, targetSeed);
+ seedDB.update(targetSeed.hash, targetSeed);
resultObj.put("result",result);
return resultObj;
}
@@ -958,7 +933,7 @@ public final class yacyClient { }
}
- private static HashMap<String, String> transferRWI(yacySeed targetSeed, indexContainer[] indexes, boolean gzipBody, int timeout) {
+ private static HashMap<String, String> transferRWI(yacySeedDB seedDB, yacySeed targetSeed, indexContainer[] indexes, boolean gzipBody, int timeout) {
final String address = targetSeed.getPublicAddress();
if (address == null) { return null; }
@@ -1002,7 +977,7 @@ public final class yacyClient { final ArrayList<String> v = nxTools.strings(content, "UTF-8");
// this should return a list of urlhashes that are unknown
if ((v != null) && (v.size() > 0)) {
- yacyCore.seedDB.mySeed().incSI(indexcount);
+ seedDB.mySeed().incSI(indexcount);
}
final HashMap<String, String> result = nxTools.table(v);
@@ -1015,7 +990,7 @@ public final class yacyClient { }
}
- private static HashMap<String, String> transferURL(yacySeed targetSeed, indexURLReference[] urls, boolean gzipBody, int timeout) {
+ private static HashMap<String, String> transferURL(yacySeedDB seedDB, yacySeed targetSeed, indexURLReference[] urls, boolean gzipBody, int timeout) {
// this post a message to the remote message board
final String address = targetSeed.getPublicAddress();
if (address == null) { return null; }
@@ -1048,7 +1023,7 @@ public final class yacyClient { final ArrayList<String> v = nxTools.strings(content, "UTF-8");
if ((v != null) && (v.size() > 0)) {
- yacyCore.seedDB.mySeed().incSU(urlc);
+ seedDB.mySeed().incSU(urlc);
}
HashMap<String, String> result = nxTools.table(v);
@@ -1094,15 +1069,15 @@ public final class yacyClient { final plasmaSwitchboard sb = new plasmaSwitchboard(new File(args[0]), "httpProxy.init", "DATA/SETTINGS/yacy.conf", false);
/*final yacyCore core =*/ new yacyCore(sb);
yacyCore.peerActions.loadSeedLists();
- final yacySeed target = yacyCore.seedDB.getConnected(args[1]);
+ final yacySeed target = sb.wordIndex.seedDB.getConnected(args[1]);
final String wordhashe = indexWord.word2hash("test");
//System.out.println("permission=" + permissionMessage(args[1]));
final byte[] content = HttpClient.wget(
"http://" + target.getPublicAddress() + "/yacy/search.html" +
- "?myseed=" + yacyCore.seedDB.mySeed().genSeedStr(null) +
+ "?myseed=" + sb.wordIndex.seedDB.mySeed().genSeedStr(null) +
"&youare=" + target.hash + "&key=" +
- "&myseed=" + yacyCore.seedDB.mySeed() .genSeedStr(null) +
+ "&myseed=" + sb.wordIndex.seedDB.mySeed() .genSeedStr(null) +
"&count=10" +
"&resource=global" +
"&query=" + wordhashe +
diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index 7d8ab1905..54d782c39 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -74,7 +74,6 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverSemaphore;
-import de.anomic.server.serverSwitch;
import de.anomic.server.logging.serverLog;
import de.anomic.xml.RSSFeed;
import de.anomic.xml.RSSMessage;
@@ -83,8 +82,6 @@ public class yacyCore { // statics
public static final ThreadGroup publishThreadGroup = new ThreadGroup("publishThreadGroup");
- public static yacySeedDB seedDB = null;
- public static yacyNewsPool newsPool = null;
public static final HashMap<String, String> seedUploadMethods = new HashMap<String, String>();
public static yacyPeerActions peerActions = null;
public static yacyDHTAction dhtAgent = null;
@@ -92,7 +89,6 @@ public class yacyCore { public static long lastOnlineTime = 0;
/** pseudo-random key derived from a time-interval while YaCy startup*/
public static long speedKey = 0;
- public static File yacyDBPath = null;
public static final Map<String, yacyAccessible> amIAccessibleDB = Collections.synchronizedMap(new HashMap<String, yacyAccessible>()); // Holds PeerHash / yacyAccessible Relations
// constants for PeerPing behaviour
private static final int PING_INITIAL = 10;
@@ -107,13 +103,8 @@ public class yacyCore { // public static boolean terminate = false;
// class variables
- private int lastSeedUpload_seedDBSize = 0;
- public long lastSeedUpload_timeStamp = System.currentTimeMillis();
- // private String lastSeedUpload_myPeerType = "";
- private String lastSeedUpload_myIP = "";
-
private static int onlineMode = 1;
- private plasmaSwitchboard switchboard;
+ private plasmaSwitchboard sb;
public static int yacyTime() {
// the time since startup of yacy in seconds
@@ -123,37 +114,21 @@ public class yacyCore { public yacyCore(plasmaSwitchboard sb) {
long time = System.currentTimeMillis();
- this.switchboard = sb;
- switchboard.setConfig("yacyStatus", "");
+ this.sb = sb;
+ sb.setConfig("yacyStatus", "");
// create a peer news channel
RSSFeed peernews = RSSFeed.channels(channelName);
peernews.setMaxsize(1000);
peernews.addMessage(new RSSMessage("YaCy started", ""));
- // create a yacy db
- yacyDBPath = sb.getConfigPath("yacyDB", "DATA/YACYDB");
- if (!yacyDBPath.exists()) { yacyDBPath.mkdir(); }
-
- // create or init seed cache
- seedDB = new yacySeedDB(
- sb,
- new File(yacyDBPath, "seed2.new.db"),
- new File(yacyDBPath, "seed2.old.db"),
- new File(yacyDBPath, "seed2.pot.db"));
-
- // create or init news database
- newsPool = new yacyNewsPool(yacyDBPath);
-
loadSeedUploadMethods();
// deploy peer actions
- peerActions = new yacyPeerActions(seedDB, switchboard);
- dhtAgent = new yacyDHTAction(seedDB);
+ peerActions = new yacyPeerActions(sb.wordIndex.seedDB, sb);
+ dhtAgent = new yacyDHTAction(sb.wordIndex.seedDB);
peerActions.deploy(dhtAgent);
- peerActions.deploy(new yacyNewsAction(newsPool));
-
- lastSeedUpload_seedDBSize = seedDB.sizeConnected();
+ peerActions.deploy(new yacyNewsAction(sb.wordIndex.newsPool));
log.logConfig("CORE INITIALIZED");
// ATTENTION, VERY IMPORTANT: before starting the thread, the httpd yacy server must be running!
@@ -161,7 +136,7 @@ public class yacyCore { speedKey = System.currentTimeMillis() - time;
// start with a seedList update to propagate out peer, if possible
- onlineMode = Integer.parseInt(switchboard.getConfig("onlineMode", "1"));
+ onlineMode = Integer.parseInt(sb.getConfig("onlineMode", "1"));
//lastSeedUpdate = universalTime();
lastOnlineTime = 0;
@@ -173,21 +148,16 @@ public class yacyCore { log.logConfig("YOU ARE OFFLINE! ---");
log.logConfig("--- TO START BOOTSTRAPING, YOU MUST USE THE PROXY,");
log.logConfig("--- OR HIT THE BUTTON 'go online'");
- log.logConfig("--- ON THE STATUS PAGE http://localhost:" + serverCore.getPortNr(switchboard.getConfig("port", "8080")) + "/Status.html");
+ log.logConfig("--- ON THE STATUS PAGE http://localhost:" + serverCore.getPortNr(sb.getConfig("port", "8080")) + "/Status.html");
}
}
- public synchronized void close() {
- seedDB.close();
- newsPool.close();
- }
-
synchronized static public void triggerOnlineAction() {
lastOnlineTime = System.currentTimeMillis();
}
public final boolean online() {
- onlineMode = Integer.parseInt(switchboard.getConfig("onlineMode", "1"));
+ onlineMode = Integer.parseInt(sb.getConfig("onlineMode", "1"));
return ((onlineMode == 2) || ((System.currentTimeMillis() - lastOnlineTime) < 10000));
}
@@ -212,37 +182,37 @@ public class yacyCore { yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself");
*/
- if ((this.lastSeedUpload_myIP.equals(seedDB.mySeed().get(yacySeed.IP, "127.0.0.1"))) &&
- (this.lastSeedUpload_seedDBSize == seedDB.sizeConnected()) &&
+ if ((sb.wordIndex.seedDB.lastSeedUpload_myIP.equals(sb.wordIndex.seedDB.mySeed().get(yacySeed.IP, "127.0.0.1"))) &&
+ (sb.wordIndex.seedDB.lastSeedUpload_seedDBSize == sb.wordIndex.seedDB.sizeConnected()) &&
(canReachMyself()) &&
- (System.currentTimeMillis() - this.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) &&
- (seedDB.mySeed().isPrincipal())
+ (System.currentTimeMillis() - sb.wordIndex.seedDB.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) &&
+ (sb.wordIndex.seedDB.mySeed().isPrincipal())
) {
log.logFine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP.");
return;
}
// getting the seed upload method that should be used ...
- final String seedUploadMethod = this.switchboard.getConfig("seedUploadMethod", "");
+ final String seedUploadMethod = this.sb.getConfig("seedUploadMethod", "");
if (
(!seedUploadMethod.equalsIgnoreCase("none")) ||
- ((seedUploadMethod.equals("")) && (this.switchboard.getConfig("seedFTPPassword", "").length() > 0)) ||
- ((seedUploadMethod.equals("")) && (this.switchboard.getConfig("seedFilePath", "").length() > 0))
+ ((seedUploadMethod.equals("")) && (this.sb.getConfig("seedFTPPassword", "").length() > 0)) ||
+ ((seedUploadMethod.equals("")) && (this.sb.getConfig("seedFilePath", "").length() > 0))
) {
if (seedUploadMethod.equals("")) {
- if (this.switchboard.getConfig("seedFTPPassword", "").length() > 0) {
- this.switchboard.setConfig("seedUploadMethod", "Ftp");
+ if (this.sb.getConfig("seedFTPPassword", "").length() > 0) {
+ this.sb.setConfig("seedUploadMethod", "Ftp");
}
- if (this.switchboard.getConfig("seedFilePath", "").length() > 0) {
- this.switchboard.setConfig("seedUploadMethod", "File");
+ if (this.sb.getConfig("seedFilePath", "").length() > 0) {
+ this.sb.setConfig("seedUploadMethod", "File");
}
}
// we want to be a principal...
- saveSeedList();
+ saveSeedList(sb);
} else {
if (seedUploadMethod.equals("")) {
- this.switchboard.setConfig("seedUploadMethod", "none");
+ this.sb.setConfig("seedUploadMethod", "none");
}
log.logFine("yacyCore.publishSeedList: No uploading method configured");
return;
@@ -251,7 +221,7 @@ public class yacyCore { public final void peerPing() {
if (!online()) { return; }
- if ((switchboard.isRobinsonMode()) && (switchboard.getConfig("cluster.mode", "").equals("privatepeer"))) {
+ if ((sb.isRobinsonMode()) && (sb.getConfig("cluster.mode", "").equals("privatepeer"))) {
// in case this peer is a privat peer we omit the peer ping
// all other robinson peer types do a peer ping:
// the privatecluster does the ping to the other cluster members
@@ -264,29 +234,29 @@ public class yacyCore { peerActions.updateMySeed();
// publish own seed to other peer, this can every peer, but makes only sense for senior peers
- if (seedDB.sizeConnected() == 0) {
+ if (sb.wordIndex.seedDB.sizeConnected() == 0) {
// reload the seed lists
peerActions.loadSeedLists();
- log.logInfo("re-initialized seed list. received " + seedDB.sizeConnected() + " new peer(s)");
+ log.logInfo("re-initialized seed list. received " + sb.wordIndex.seedDB.sizeConnected() + " new peer(s)");
}
final int newSeeds = publishMySeed(false);
if (newSeeds > 0) {
- log.logInfo("received " + newSeeds + " new peer(s), know a total of " + seedDB.sizeConnected() + " different peers");
+ log.logInfo("received " + newSeeds + " new peer(s), know a total of " + sb.wordIndex.seedDB.sizeConnected() + " different peers");
}
}
private boolean canReachMyself() { // TODO: check if this method is necessary - depending on the used router it will not work
// returns true if we can reach ourself under our known peer address
// if we cannot reach ourself, we call a forced publishMySeed and return false
- final int urlc = yacyClient.queryUrlCount(seedDB.mySeed());
+ final int urlc = yacyClient.queryUrlCount(sb.wordIndex.seedDB.mySeed());
if (urlc >= 0) {
- seedDB.mySeed().setLastSeenUTC();
+ sb.wordIndex.seedDB.mySeed().setLastSeenUTC();
return true;
}
log.logInfo("re-connect own seed");
- final String oldAddress = seedDB.mySeed().getPublicAddress();
+ final String oldAddress = sb.wordIndex.seedDB.mySeed().getPublicAddress();
/*final int newSeeds =*/ publishMySeed(true);
- return (oldAddress != null && oldAddress.equals(seedDB.mySeed().getPublicAddress()));
+ return (oldAddress != null && oldAddress.equals(sb.wordIndex.seedDB.mySeed().getPublicAddress()));
}
protected class publishThread extends Thread {
@@ -309,7 +279,7 @@ public class yacyCore { public final void run() {
try {
- this.added = yacyClient.publishMySeed(seed.getClusterAddress(), seed.hash);
+ this.added = yacyClient.publishMySeed(seed, seed.getClusterAddress(), seed.hash);
if (this.added < 0) {
// no or wrong response, delete that address
String cause = "peer ping to peer resulted in error response (added < 0)";
@@ -320,7 +290,7 @@ public class yacyCore { // update latest news from the other peer
log.logInfo("publish: handshaked " + this.seed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR) + " peer '" + this.seed.getName() + "' at " + this.seed.getPublicAddress());
// check if seed's lastSeen has been updated
- yacySeed newSeed = seedDB.getConnected(this.seed.hash);
+ yacySeed newSeed = sb.wordIndex.seedDB.getConnected(this.seed.hash);
if (newSeed != null) {
if (newSeed.getLastSeenUTC() < (System.currentTimeMillis() - 10000)) {
// update last seed date
@@ -373,13 +343,13 @@ public class yacyCore { // init yacyHello-process
Map<String, yacySeed> seeds; // hash/yacySeed relation
- int attempts = seedDB.sizeConnected();
+ int attempts = sb.wordIndex.seedDB.sizeConnected();
// getting a list of peers to contact
- if (seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
+ if (sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
if (attempts > PING_INITIAL) { attempts = PING_INITIAL; }
Map<String, String> ch = plasmaSwitchboard.getSwitchboard().clusterhashes;
- seeds = seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
+ seeds = sb.wordIndex.seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster
if (plasmaSwitchboard.getSwitchboard().clusterhashes != null) {
Iterator<Map.Entry<String, String>> i = ch.entrySet().iterator();
@@ -391,7 +361,7 @@ public class yacyCore { hash = entry.getKey();
seed = (yacySeed) seeds.get(hash);
if (seed == null) {
- seed = seedDB.get(hash);
+ seed = sb.wordIndex.seedDB.get(hash);
if (seed == null) continue;
}
seed.setAlternativeAddress((String) entry.getValue());
@@ -406,7 +376,7 @@ public class yacyCore { } else {
if (attempts > PING_MIN_RUNNING) { attempts = PING_MIN_RUNNING; }
}
- seeds = seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning
+ seeds = sb.wordIndex.seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning
}
if ((seeds == null) || seeds.size() == 0) { return 0; }
@@ -418,20 +388,20 @@ public class yacyCore { // include a YaCyNews record to my seed
try {
- final yacyNewsRecord record = newsPool.myPublication();
+ final yacyNewsRecord record = sb.wordIndex.newsPool.myPublication();
if (record == null) {
- seedDB.mySeed().put("news", "");
+ sb.wordIndex.seedDB.mySeed().put("news", "");
} else {
- seedDB.mySeed().put("news", de.anomic.tools.crypt.simpleEncode(record.toString()));
+ sb.wordIndex.seedDB.mySeed().put("news", de.anomic.tools.crypt.simpleEncode(record.toString()));
}
} catch (IOException e) {
log.logSevere("publishMySeed: problem with news encoding", e);
}
- seedDB.mySeed().setUnusedFlags();
+ sb.wordIndex.seedDB.mySeed().setUnusedFlags();
// include current citation-rank file count
- seedDB.mySeed().put(yacySeed.CRWCNT, Integer.toString(switchboard.rankingOwnDistribution.size()));
- seedDB.mySeed().put(yacySeed.CRTCNT, Integer.toString(switchboard.rankingOtherDistribution.size()));
+ sb.wordIndex.seedDB.mySeed().put(yacySeed.CRWCNT, Integer.toString(sb.rankingOwnDistribution.size()));
+ sb.wordIndex.seedDB.mySeed().put(yacySeed.CRTCNT, Integer.toString(sb.rankingOtherDistribution.size()));
int newSeeds = -1;
//if (seeds.length > 1) {
// holding a reference to all started threads
@@ -519,7 +489,7 @@ public class yacyCore { if ((accessible >= PING_MIN_PEERSEEN) ||
(accessible >= notaccessible)) {
// We can be reached from a majority of other Peers
- if (yacyCore.seedDB.mySeed().isPrincipal()) {
+ if (sb.wordIndex.seedDB.mySeed().isPrincipal()) {
newPeerType = yacySeed.PEERTYPE_PRINCIPAL;
} else {
newPeerType = yacySeed.PEERTYPE_SENIOR;
@@ -528,14 +498,14 @@ public class yacyCore { // We cannot be reached from the outside
newPeerType = yacySeed.PEERTYPE_JUNIOR;
}
- if (yacyCore.seedDB.mySeed().orVirgin().equals(newPeerType)) {
- log.logInfo("PeerPing: myType is " + yacyCore.seedDB.mySeed().orVirgin());
+ if (sb.wordIndex.seedDB.mySeed().orVirgin().equals(newPeerType)) {
+ log.logInfo("PeerPing: myType is " + sb.wordIndex.seedDB.mySeed().orVirgin());
} else {
- log.logInfo("PeerPing: changing myType from '" + yacyCore.seedDB.mySeed().orVirgin() + "' to '" + newPeerType + "'");
- yacyCore.seedDB.mySeed().put(yacySeed.PEERTYPE, newPeerType);
+ log.logInfo("PeerPing: changing myType from '" + sb.wordIndex.seedDB.mySeed().orVirgin() + "' to '" + newPeerType + "'");
+ sb.wordIndex.seedDB.mySeed().put(yacySeed.PEERTYPE, newPeerType);
}
} else {
- log.logInfo("PeerPing: No data, staying at myType: " + yacyCore.seedDB.mySeed().orVirgin());
+ log.logInfo("PeerPing: No data, staying at myType: " + sb.wordIndex.seedDB.mySeed().orVirgin());
}
if (newSeeds >= 0) {
@@ -547,20 +517,20 @@ public class yacyCore { }
// if we have an address, we do nothing
- if (seedDB.mySeed().isProper() == null && !force) { return 0; }
+ if (sb.wordIndex.seedDB.mySeed().isProper() == null && !force) { return 0; }
// still no success: ask own NAT or internet responder
//final boolean DI604use = switchboard.getConfig("DI604use", "false").equals("true");
//final String DI604pw = switchboard.getConfig("DI604pw", "");
- String ip = switchboard.getConfig("staticIP", "");
+ String ip = sb.getConfig("staticIP", "");
//if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw);
// yacyCore.log.logDebug("DEBUG: new IP=" + ip);
- seedDB.mySeed().put(yacySeed.IP, ip);
- if (seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_JUNIOR)) // ???????????????
- seedDB.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer
+ sb.wordIndex.seedDB.mySeed().put(yacySeed.IP, ip);
+ if (sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR).equals(yacySeed.PEERTYPE_JUNIOR)) // ???????????????
+ sb.wordIndex.seedDB.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer
log.logInfo("publish: no recipient found, our address is " +
- ((seedDB.mySeed().getPublicAddress() == null) ? "unknown" : seedDB.mySeed().getPublicAddress()));
+ ((sb.wordIndex.seedDB.mySeed().getPublicAddress() == null) ? "unknown" : sb.wordIndex.seedDB.mySeed().getPublicAddress()));
peerActions.saveMySeed();
return 0;
} catch (InterruptedException e) {
@@ -583,16 +553,6 @@ public class yacyCore { threadCount = yacyCore.publishThreadGroup.enumerate(threadList);
// we need to use a timeout here because of missing interruptable session threads ...
- log.logFine("publish: Trying to abort " + yacyCore.publishThreadGroup.activeCount() + " remaining publishing threads ...");
- for (int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++) {
- Thread currentThread = threadList[currentThreadIdx];
-
- /*if (currentThread.isAlive()) {
- // TODO: this object should care of all open clien connections within this class and close them here
- }*/
- }
-
- // we need to use a timeout here because of missing interruptable session threads ...
log.logFine("publish: Waiting for " + yacyCore.publishThreadGroup.activeCount() + " remaining publishing threads to finish shutdown ...");
for (int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++) {
final Thread currentThread = threadList[currentThreadIdx];
@@ -699,18 +659,13 @@ public class yacyCore { }
}
- public final String saveSeedList() {
- // return an error if this is not successful, and NULL if everything is fine
- return saveSeedList(this.switchboard);
- }
-
- public final String saveSeedList(serverSwitch<?> sb) {
+ public static final String saveSeedList(plasmaSwitchboard sb) {
try {
// return an error if this is not successful, and NULL if everything is fine
String logt;
// be shure that we have something to say
- if (seedDB.mySeed().getPublicAddress() == null) {
+ if (sb.wordIndex.seedDB.mySeed().getPublicAddress() == null) {
final String errorMsg = "We have no valid IP address until now";
log.logWarning("SaveSeedList: " + errorMsg);
return errorMsg;
@@ -764,20 +719,20 @@ public class yacyCore { }
// upload the seed-list using the configured uploader class
- String prevStatus = seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
+ String prevStatus = sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
if (prevStatus.equals(yacySeed.PEERTYPE_PRINCIPAL)) { prevStatus = yacySeed.PEERTYPE_SENIOR; }
try {
- seedDB.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL); // this information shall also be uploaded
+ sb.wordIndex.seedDB.mySeed().put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_PRINCIPAL); // this information shall also be uploaded
log.logFine("SaveSeedList: Using seed uploading method '" + seedUploadMethod + "' for seed-list uploading." +
- "\n\tPrevious peerType is '" + seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR) + "'.");
+ "\n\tPrevious peerType is '" + sb.wordIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR) + "'.");
// logt = seedDB.uploadCache(seedFTPServer, seedFTPAccount, seedFTPPassword, seedFTPPath, seedURL);
- logt = seedDB.uploadCache(uploader, sb, seedDB, seedURL);
+ logt = sb.wordIndex.seedDB.uploadCache(uploader, sb, sb.wordIndex.seedDB, seedURL);
if (logt != null) {
if (logt.indexOf("Error") >= 0) {
- seedDB.mySeed().put(yacySeed.PEERTYPE, prevStatus);
+ sb.wordIndex.seedDB.mySeed().put(yacySeed.PEERTYPE, prevStatus);
final String errorMsg = "SaveSeedList: seed upload failed using " + uploader.getClass().getName() + " (error): " + logt.substring(logt.indexOf("Error") + 6);
log.logSevere(errorMsg);
return errorMsg;
@@ -789,18 +744,16 @@ public class yacyCore { sb.setConfig("yacyStatus", yacySeed.PEERTYPE_PRINCIPAL);
return null;
} catch (Exception e) {
- seedDB.mySeed().put(yacySeed.PEERTYPE, prevStatus);
+ sb.wordIndex.seedDB.mySeed().put(yacySeed.PEERTYPE, prevStatus);
sb.setConfig("yacyStatus", prevStatus);
final String errorMsg = "SaveSeedList: Seed upload failed (IO error): " + e.getMessage();
log.logInfo(errorMsg, e);
return errorMsg;
}
} finally {
- this.lastSeedUpload_seedDBSize = seedDB.sizeConnected();
- this.lastSeedUpload_timeStamp = System.currentTimeMillis();
-
- this.lastSeedUpload_myIP = seedDB.mySeed().get(yacySeed.IP, "127.0.0.1");
- //this.lastSeedUpload_myPeerType = seedDB.mySeed.get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_JUNIOR);
+ sb.wordIndex.seedDB.lastSeedUpload_seedDBSize = sb.wordIndex.seedDB.sizeConnected();
+ sb.wordIndex.seedDB.lastSeedUpload_timeStamp = System.currentTimeMillis();
+ sb.wordIndex.seedDB.lastSeedUpload_myIP = sb.wordIndex.seedDB.mySeed().get(yacySeed.IP, "127.0.0.1");
}
}
diff --git a/source/de/anomic/yacy/yacyDHTAction.java b/source/de/anomic/yacy/yacyDHTAction.java index 8b400ca6e..20e200caf 100644 --- a/source/de/anomic/yacy/yacyDHTAction.java +++ b/source/de/anomic/yacy/yacyDHTAction.java @@ -315,11 +315,11 @@ public class yacyDHTAction implements yacyPeerAction { public void processPeerPing(yacySeed peer) {
}
- public static boolean shallBeOwnWord(String wordhash) {
- if (yacyCore.seedDB == null) return false;
- if (yacyCore.seedDB.mySeed().isPotential()) return false;
- final double distance = dhtDistance(yacyCore.seedDB.mySeed().hash, wordhash);
- final double max = 1.2 / yacyCore.seedDB.sizeConnected();
+ public static boolean shallBeOwnWord(yacySeedDB seedDB, String wordhash) {
+ if (seedDB == null) return false;
+ if (seedDB.mySeed().isPotential()) return false;
+ final double distance = dhtDistance(seedDB.mySeed().hash, wordhash);
+ final double max = 1.2 / seedDB.sizeConnected();
//System.out.println("Distance for " + wordhash + ": " + distance + "; max is " + max);
return (distance > 0) && (distance <= max);
}
@@ -347,13 +347,13 @@ public class yacyDHTAction implements yacyPeerAction { return ((double) (kelondroBase64Order.enhancedCoder.cardinal(from.getBytes()) - kelondroBase64Order.enhancedCoder.cardinal(to.getBytes()))) / ((double) Long.MAX_VALUE);
}
- public synchronized ArrayList<yacySeed> getDHTTargets(serverLog log, int primaryPeerCount, int reservePeerCount, String firstKey, String lastKey, double maxDist) {
+ public synchronized ArrayList<yacySeed> getDHTTargets(yacySeedDB seedDB, serverLog log, int primaryPeerCount, int reservePeerCount, String firstKey, String lastKey, double maxDist) {
// find a list of DHT-peers
assert firstKey != null;
assert lastKey != null;
- assert yacyCore.seedDB != null;
- assert yacyCore.seedDB.mySeed() != null;
- assert yacyCore.seedDB.mySeed().hash != null;
+ assert seedDB != null;
+ assert seedDB.mySeed() != null;
+ assert seedDB.mySeed().hash != null;
/*
assert
!(kelondroBase64Order.enhancedCoder.cardinal(firstKey.getBytes()) < kelondroBase64Order.enhancedCoder.cardinal(yacyCore.seedDB.mySeed.hash.getBytes()) &&
@@ -367,7 +367,7 @@ public class yacyDHTAction implements yacyPeerAction { double firstdist, lastdist;
Iterator<yacySeed> e = this.getAcceptRemoteIndexSeeds(lastKey);
TreeSet<String> doublecheck = new TreeSet<String>(kelondroBase64Order.enhancedComparator);
- int maxloop = Math.min(100, yacyCore.seedDB.sizeConnected()); // to ensure termination
+ int maxloop = Math.min(100, seedDB.sizeConnected()); // to ensure termination
if (log != null) log.logInfo("Collecting DHT target peers for first_hash = " + firstKey + ", last_hash = " + lastKey);
while ((e.hasNext()) && (seeds.size() < (primaryPeerCount + reservePeerCount)) && (maxloop-- > 0)) {
seed = (yacySeed) e.next();
diff --git a/source/de/anomic/yacy/yacyNetwork.java b/source/de/anomic/yacy/yacyNetwork.java index d22b30b8b..590d0400f 100644 --- a/source/de/anomic/yacy/yacyNetwork.java +++ b/source/de/anomic/yacy/yacyNetwork.java @@ -71,14 +71,14 @@ public class yacyNetwork { return false; } - public static final List<Part> basicRequestPost(serverSwitch<?> env, String targetHash, String salt) { + public static final List<Part> basicRequestPost(plasmaSwitchboard sb, String targetHash, String salt) { // put in all the essentials for routing and network authentification // generate a session key ArrayList<Part> post = new ArrayList<Part>(); post.add(new StringPart("key", salt)); // just standard identification essentials - post.add(new StringPart("iam", yacyCore.seedDB.mySeed().hash)); + post.add(new StringPart("iam", sb.wordIndex.seedDB.mySeed().hash)); if (targetHash != null) post.add(new StringPart("youare", targetHash)); // time information for synchronization @@ -89,13 +89,13 @@ public class yacyNetwork { post.add(new StringPart("network.unit.name", plasmaSwitchboard.getSwitchboard().getConfig("network.unit.name", yacySeed.DFLT_NETWORK_UNIT))); // authentification essentials - String authentificationControl = env.getConfig("network.unit.protocol.control", "uncontrolled"); - String authentificationMethod = env.getConfig("network.unit.protocol.request.authentification.method", ""); + String authentificationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled"); + String authentificationMethod = sb.getConfig("network.unit.protocol.request.authentification.method", ""); if ((authentificationControl.equals("controlled")) && (authentificationMethod.length() > 0)) { if (authentificationMethod.equals("salted-magic-sim")) { // generate an authentification essential using the salt, the iam-hash and the network magic - String magic = env.getConfig("network.unit.protocol.request.authentification.essentials", ""); - String md5 = serverCodings.encodeMD5Hex(salt + yacyCore.seedDB.mySeed().hash + magic); + String magic = sb.getConfig("network.unit.protocol.request.authentification.essentials", ""); + String md5 = serverCodings.encodeMD5Hex(salt + sb.wordIndex.seedDB.mySeed().hash + magic); post.add(new StringPart("magicmd5", md5)); } } diff --git a/source/de/anomic/yacy/yacyNewsPool.java b/source/de/anomic/yacy/yacyNewsPool.java index d06514983..d8acaac48 100644 --- a/source/de/anomic/yacy/yacyNewsPool.java +++ b/source/de/anomic/yacy/yacyNewsPool.java @@ -265,11 +265,11 @@ public class yacyNewsPool { public yacyNewsPool(File yacyDBPath) {
- newsDB = new yacyNewsDB(new File(yacyDBPath, "news2.db"));
- outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut1.stack"), newsDB);
- publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished1.stack"), newsDB);
- incomingNews = new yacyNewsQueue(new File(yacyDBPath, "newsIn1.stack"), newsDB);
- processedNews = new yacyNewsQueue(new File(yacyDBPath, "newsProcessed1.stack"), newsDB);
+ newsDB = new yacyNewsDB(new File(yacyDBPath, "news.db"));
+ outgoingNews = new yacyNewsQueue(new File(yacyDBPath, "newsOut.stack"), newsDB);
+ publishedNews = new yacyNewsQueue(new File(yacyDBPath, "newsPublished.stack"), newsDB);
+ incomingNews = new yacyNewsQueue(new File(yacyDBPath, "newsIn.stack"), newsDB);
+ processedNews = new yacyNewsQueue(new File(yacyDBPath, "newsProcessed.stack"), newsDB);
maxDistribution = 30;
}
@@ -348,7 +348,7 @@ public class yacyNewsPool { return switchQueue(dbKey).size();
}
- public int automaticProcess() throws IOException, InterruptedException {
+ public int automaticProcess(yacySeedDB seedDB) throws IOException, InterruptedException {
// processes news in the incoming-db
// returns number of processes
yacyNewsRecord record;
@@ -361,7 +361,7 @@ public class yacyNewsPool { // get next news record
record = (yacyNewsRecord) i.next();
- if (automaticProcessP(record)) {
+ if (automaticProcessP(seedDB, record)) {
this.processedNews.push(record);
i.remove();
pc++;
@@ -372,7 +372,7 @@ public class yacyNewsPool { }
long day = 1000 * 60 * 60 * 24;
- private boolean automaticProcessP(yacyNewsRecord record) {
+ private boolean automaticProcessP(yacySeedDB seedDB, yacyNewsRecord record) {
if (record == null) return false;
if (record.category() == null) return true;
if ((System.currentTimeMillis() - record.created().getTime()) > (14 * day)) {
@@ -393,7 +393,7 @@ public class yacyNewsPool { }
if ((record.category().equals(CATEGORY_CRAWL_START)) &&
((System.currentTimeMillis() - record.created().getTime()) > (2 * day))) {
- yacySeed seed = yacyCore.seedDB.get(record.originator());
+ yacySeed seed = seedDB.get(record.originator());
if (seed == null) return false;
try {
return (Integer.parseInt(seed.get(yacySeed.ISPEED, "-")) < 10);
diff --git a/source/de/anomic/yacy/yacyNewsRecord.java b/source/de/anomic/yacy/yacyNewsRecord.java index c1755009b..bd54881c1 100644 --- a/source/de/anomic/yacy/yacyNewsRecord.java +++ b/source/de/anomic/yacy/yacyNewsRecord.java @@ -93,7 +93,7 @@ public class yacyNewsRecord { }
}
- public static yacyNewsRecord newRecord(String category, Properties attributes) {
+ public static yacyNewsRecord newRecord(yacySeed mySeed, String category, Properties attributes) {
try {
HashMap<String, String> m = new HashMap<String, String>();
Iterator<Entry<Object, Object>> e = attributes.entrySet().iterator();
@@ -102,16 +102,16 @@ public class yacyNewsRecord { entry = e.next();
m.put((String) entry.getKey(), (String) entry.getValue());
}
- return new yacyNewsRecord(category, m);
+ return new yacyNewsRecord(mySeed, category, m);
} catch (IllegalArgumentException e) {
yacyCore.log.logWarning("rejected bad yacy news record: " + e.getMessage());
return null;
}
}
- public static yacyNewsRecord newRecord(String category, Map<String, String> attributes) {
+ public static yacyNewsRecord newRecord(yacySeed mySeed, String category, Map<String, String> attributes) {
try {
- return new yacyNewsRecord(category, attributes);
+ return new yacyNewsRecord(mySeed, category, attributes);
} catch (IllegalArgumentException e) {
yacyCore.log.logWarning("rejected bad yacy news record: " + e.getMessage());
return null;
@@ -139,7 +139,7 @@ public class yacyNewsRecord { removeStandards();
}
- public yacyNewsRecord(String category, Map<String, String> attributes) {
+ public yacyNewsRecord(yacySeed mySeed, String category, Map<String, String> attributes) {
if (category.length() > categoryStringLength) throw new IllegalArgumentException("category length (" + category.length() + ") exceeds maximum (" + categoryStringLength + ")");
if (attributes.toString().length() > attributesMaxLength) throw new IllegalArgumentException("attributes length (" + attributes.toString().length() + ") exceeds maximum (" + attributesMaxLength + ")");
this.attributes = attributes;
@@ -147,7 +147,7 @@ public class yacyNewsRecord { this.created = new Date();
this.category = category;
this.distributed = 0;
- this.originator = yacyCore.seedDB.mySeed().hash;
+ this.originator = mySeed.hash;
removeStandards();
}
diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index c54a6f59b..705d84144 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -91,11 +91,7 @@ public class yacyPeerActions { sb.setConfig("peerName", yacySeed.makeDefaultPeerName());
}
seedDB.mySeed().put(yacySeed.NAME, sb.getConfig("peerName", "nameless"));
- if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) {
- seedDB.mySeed().put(yacySeed.PORT, Integer.toString(serverCore.portForwarding.getPort()));
- } else {
- seedDB.mySeed().put(yacySeed.PORT, Integer.toString(serverCore.getPortNr(sb.getConfig("port", "8080"))));
- }
+ seedDB.mySeed().put(yacySeed.PORT, Integer.toString(serverCore.getPortNr(sb.getConfig("port", "8080"))));
long uptime = (System.currentTimeMillis() - serverCore.startupTime) / 1000;
long uptimediff = uptime - sb.lastseedcheckuptime;
diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index 27d5a5a15..bef3a321e 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -109,6 +109,7 @@ public class yacySearch extends Thread { public void run() {
this.urls = yacyClient.search(
+ wordIndex.seedDB.mySeed(),
wordhashes, excludehashes, urlhashes, prefer, filter, count, maxDistance, global, partitions,
targetPeer, wordIndex, crawlResults, containerCache, abstractCache,
blacklist, rankingProfile, constraint);
@@ -117,8 +118,8 @@ public class yacySearch extends Thread { StringBuffer urllist = new StringBuffer(this.urls.length * 13);
for (int i = 0; i < this.urls.length; i++) urllist.append(this.urls[i]).append(' ');
yacyCore.log.logInfo("REMOTE SEARCH - remote peer " + targetPeer.hash + ":" + targetPeer.getName() + " contributed " + urls.length + " links for word hash " + wordhashes + ": " + new String(urllist));
- yacyCore.seedDB.mySeed().incRI(urls.length);
- yacyCore.seedDB.mySeed().incRU(urls.length);
+ wordIndex.seedDB.mySeed().incRI(urls.length);
+ wordIndex.seedDB.mySeed().incRU(urls.length);
} else {
yacyCore.log.logInfo("REMOTE SEARCH - no answer from remote peer " + targetPeer.hash + ":" + targetPeer.getName());
}
@@ -143,14 +144,14 @@ public class yacySearch extends Thread { return targetPeer;
}
- private static yacySeed[] selectClusterPeers(TreeMap<String, String> peerhashes) {
+ private static yacySeed[] selectClusterPeers(yacySeedDB seedDB, TreeMap<String, String> peerhashes) {
Iterator<Map.Entry<String, String>> i = peerhashes.entrySet().iterator();
ArrayList<yacySeed> l = new ArrayList<yacySeed>();
Map.Entry<String, String> entry;
yacySeed s;
while (i.hasNext()) {
entry = i.next();
- s = yacyCore.seedDB.get(entry.getKey()); // should be getConnected; get only during testing time
+ s = seedDB.get(entry.getKey()); // should be getConnected; get only during testing time
if (s != null) {
s.setAlternativeAddress(entry.getValue());
l.add(s);
@@ -164,13 +165,13 @@ public class yacySearch extends Thread { //return (yacySeed[]) l.toArray();
}
- private static yacySeed[] selectSearchTargets(Set<String> wordhashes, int seedcount) {
+ private static yacySeed[] selectSearchTargets(yacySeedDB seedDB, Set<String> wordhashes, int seedcount) {
// find out a specific number of seeds, that would be relevant for the given word hash(es)
// the result is ordered by relevance: [0] is most relevant
// the seedcount is the maximum number of wanted results
- if (yacyCore.seedDB == null) { return null; }
- if ((seedcount >= yacyCore.seedDB.sizeConnected()) || (yacyCore.seedDB.noDHTActivity())) {
- seedcount = yacyCore.seedDB.sizeConnected();
+ if (seedDB == null) { return null; }
+ if ((seedcount >= seedDB.sizeConnected()) || (seedDB.noDHTActivity())) {
+ seedcount = seedDB.sizeConnected();
}
// put in seeds according to dht
@@ -199,10 +200,10 @@ public class yacySearch extends Thread { }
// put in seeds according to size of peer
- dhtEnum = yacyCore.seedDB.seedsSortedConnected(false, yacySeed.ICOUNT);
+ dhtEnum = seedDB.seedsSortedConnected(false, yacySeed.ICOUNT);
c = seedcount;
int score;
- if (c > yacyCore.seedDB.sizeConnected()) { c = yacyCore.seedDB.sizeConnected(); }
+ if (c > seedDB.sizeConnected()) { c = seedDB.sizeConnected(); }
while (dhtEnum.hasNext() && c > 0) {
seed = dhtEnum.next();
if (seed == null) continue;
@@ -216,7 +217,7 @@ public class yacySearch extends Thread { // put in seeds that are public robinson peers and where the peer tags match with query
// or seeds that are newbies to ensure that public demonstrations always work
- dhtEnum = yacyCore.seedDB.seedsConnected(true, false, null, (float) 0.50);
+ dhtEnum = seedDB.seedsConnected(true, false, null, (float) 0.50);
while (dhtEnum.hasNext()) {
seed = dhtEnum.next();
if (seed == null) continue;
@@ -261,10 +262,10 @@ public class yacySearch extends Thread { kelondroBitfield constraint,
TreeMap<String, String> clusterselection) {
// check own peer status
- if (yacyCore.seedDB.mySeed() == null || yacyCore.seedDB.mySeed().getPublicAddress() == null) { return null; }
+ if (wordIndex.seedDB.mySeed() == null || wordIndex.seedDB.mySeed().getPublicAddress() == null) { return null; }
// prepare seed targets and threads
- final yacySeed[] targetPeers = (clusterselection == null) ? selectSearchTargets(plasmaSearchQuery.hashes2Set(wordhashes), targets) : selectClusterPeers(clusterselection);
+ final yacySeed[] targetPeers = (clusterselection == null) ? selectSearchTargets(wordIndex.seedDB, plasmaSearchQuery.hashes2Set(wordhashes), targets) : selectClusterPeers(wordIndex.seedDB, clusterselection);
if (targetPeers == null) return new yacySearch[0];
targets = targetPeers.length;
if (targets == 0) return new yacySearch[0];
@@ -286,10 +287,10 @@ public class yacySearch extends Thread { plasmaSearchRankingProfile rankingProfile,
kelondroBitfield constraint, TreeMap<String, String> clusterselection) {
// check own peer status
- if (yacyCore.seedDB.mySeed() == null || yacyCore.seedDB.mySeed().getPublicAddress() == null) { return null; }
+ if (wordIndex.seedDB.mySeed() == null || wordIndex.seedDB.mySeed().getPublicAddress() == null) { return null; }
// prepare seed targets and threads
- final yacySeed targetPeer = yacyCore.seedDB.getConnected(targethash);
+ final yacySeed targetPeer = wordIndex.seedDB.getConnected(targethash);
if (targetPeer == null) return null;
if (clusterselection != null) targetPeer.setAlternativeAddress((String) clusterselection.get(targetPeer.hash));
yacySearch searchThread = new yacySearch(wordhashes, excludehashes, urlhashes, "", "", 0, 9999, true, 0, targetPeer,
diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index 492d46f7b..bbc91acb6 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -74,7 +74,6 @@ import java.util.TreeMap; import de.anomic.index.indexWord;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.net.natLib;
-import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCodings;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
@@ -783,22 +782,22 @@ public class yacySeed { return bestHash;
}
- public static yacySeed genLocalSeed(plasmaSwitchboard sb) {
- // genera a seed for the local peer
+ public static yacySeed genLocalSeed(yacySeedDB db) {
+ return genLocalSeed(db, 0, null); // an anonymous peer
+ }
+
+ public static yacySeed genLocalSeed(yacySeedDB db, int port, String name) {
+ // generate a seed for the local peer
// this is the birthplace of a seed, that then will start to travel to other peers
- final String hash = bestGap(yacyCore.seedDB);
+ final String hash = bestGap(db);
yacyCore.log.logInfo("init: OWN SEED = " + hash);
final yacySeed newSeed = new yacySeed(hash);
// now calculate other information about the host
- newSeed.dna.put(yacySeed.NAME, sb.getConfig("peerName", "unnamed"));
- if ((serverCore.portForwardingEnabled) && (serverCore.portForwarding != null)) {
- newSeed.dna.put(yacySeed.PORT, Integer.toString(serverCore.portForwarding.getPort()));
- } else {
- newSeed.dna.put(yacySeed.PORT, Integer.toString(serverCore.getPortNr(sb.getConfig("port", "8080"))));
- }
+ newSeed.dna.put(yacySeed.NAME, (name) == null ? "anonymous" : name);
+ newSeed.dna.put(yacySeed.PORT, Integer.toString((port <= 0) ? 8080 : port));
newSeed.dna.put(yacySeed.BDATE, serverDate.formatShortSecond(new Date(System.currentTimeMillis() - serverDate.UTCDiff())) );
newSeed.dna.put(yacySeed.LASTSEEN, newSeed.dna.get(yacySeed.BDATE)); // just as initial setting
newSeed.dna.put(yacySeed.UTC, serverDate.UTCDiffString());
diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index bc2b4768e..9583189b5 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -64,13 +64,13 @@ import de.anomic.http.JakartaCommonsHttpClient; import de.anomic.http.JakartaCommonsHttpResponse;
import de.anomic.http.httpHeader;
import de.anomic.http.httpd;
+import de.anomic.http.httpdAlternativeDomainNames;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroDyn;
import de.anomic.kelondro.kelondroException;
import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapObjects;
import de.anomic.plasma.plasmaHTCache;
-import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.server.serverCore;
import de.anomic.server.serverDate;
import de.anomic.server.serverDomains;
@@ -79,7 +79,7 @@ import de.anomic.server.serverSwitch; import de.anomic.server.logging.serverLog;
import de.anomic.tools.nxTools;
-public final class yacySeedDB {
+public final class yacySeedDB implements httpdAlternativeDomainNames {
// global statics
@@ -99,25 +99,28 @@ public final class yacySeedDB { // class objects
protected File seedActiveDBFile, seedPassiveDBFile, seedPotentialDBFile;
-
+ protected File myOwnSeedFile;
protected kelondroMapObjects seedActiveDB, seedPassiveDB, seedPotentialDB;
- private final plasmaSwitchboard sb;
+ public int lastSeedUpload_seedDBSize = 0;
+ public long lastSeedUpload_timeStamp = System.currentTimeMillis();
+ public String lastSeedUpload_myIP = "";
+
private yacySeed mySeed; // my own seed
private final Hashtable<String, yacySeed> nameLookupCache;
private final Hashtable<InetAddress, SoftReference<yacySeed>> ipLookupCache;
- public yacySeedDB(plasmaSwitchboard sb,
+ public yacySeedDB(
File seedActiveDBFile,
File seedPassiveDBFile,
- File seedPotentialDBFile) {
-
+ File seedPotentialDBFile,
+ File myOwnSeedFile) {
this.seedActiveDBFile = seedActiveDBFile;
this.seedPassiveDBFile = seedPassiveDBFile;
this.seedPotentialDBFile = seedPotentialDBFile;
this.mySeed = null; // my own seed
- this.sb = sb;
+ this.myOwnSeedFile = myOwnSeedFile;
// set up seed database
seedActiveDB = openSeedTable(seedActiveDBFile);
@@ -132,19 +135,23 @@ public final class yacySeedDB { // check if we are in the seedCaches: this can happen if someone else published our seed
removeMySeed();
+
+ lastSeedUpload_seedDBSize = sizeConnected();
+
+ // tell the httpdProxy how to find this table as address resolver
+ httpd.alternativeResolver = this;
}
private synchronized void initMySeed() {
if (this.mySeed != null) return;
// create or init own seed
- File myOwnSeedFile = sb.getOwnSeedFile();
if (myOwnSeedFile.length() > 0) try {
// load existing identity
mySeed = yacySeed.load(myOwnSeedFile);
} catch (IOException e) {
// create new identity
- mySeed = yacySeed.genLocalSeed(sb);
+ mySeed = yacySeed.genLocalSeed(this);
try {
mySeed.save(myOwnSeedFile);
} catch (IOException ee) {
@@ -153,7 +160,7 @@ public final class yacySeedDB { }
} else {
// create new identity
- mySeed = yacySeed.genLocalSeed(sb);
+ mySeed = yacySeed.genLocalSeed(this);
try {
mySeed.save(myOwnSeedFile);
} catch (IOException ee) {
@@ -162,15 +169,8 @@ public final class yacySeedDB { }
}
- if (sb.getConfig("portForwardingEnabled","false").equalsIgnoreCase("true")) {
- mySeed.put(yacySeed.PORT, sb.getConfig("portForwardingPort","8080"));
- mySeed.put(yacySeed.IP, sb.getConfig("portForwardingHost","localhost"));
- } else {
- mySeed.put(yacySeed.IP, ""); // we delete the old information to see what we have now
- mySeed.put(yacySeed.PORT, Integer.toString(serverCore.getPortNr(sb.getConfig("port", "8080")))); // set my seed's correct port number
- }
+ mySeed.put(yacySeed.IP, ""); // we delete the old information to see what we have now
mySeed.put(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN); // markup startup condition
-
}
public boolean mySeedIsDefined() {
@@ -185,6 +185,26 @@ public final class yacySeedDB { return this.mySeed;
}
+ public String myAlternativeAddress() {
+ return mySeed().getName() + ".yacy";
+ }
+
+ public String myIP() {
+ return mySeed().getIP();
+ }
+
+ public int myPort() {
+ return mySeed().getPort();
+ }
+
+ public String myName() {
+ return mySeed.getName();
+ }
+
+ public String myID() {
+ return mySeed.hash;
+ }
+
public synchronized void removeMySeed() {
if ((seedActiveDB.size() == 0) && (seedPassiveDB.size() == 0) && (seedPotentialDB.size() == 0)) return; // avoid that the own seed is initialized too early
if (this.mySeed == null) initMySeed();
@@ -866,7 +886,10 @@ public final class yacySeedDB { return null;
}
- public String resolveYacyAddress(String host) {
+ /**
+ * resolve a yacy address
+ */
+ public String resolve(String host) {
yacySeed seed;
int p;
String subdom = null;
@@ -908,7 +931,7 @@ public final class yacySeedDB { if (this.mySeed == null) initMySeed();
if ((seed == mySeed) && (!(seed.isOnline()))) {
// take local ip instead of external
- return serverDomains.myPublicIP() + ":" + serverCore.getPortNr(sb.getConfig("port", "8080")) + ((subdom == null) ? "" : ("/" + subdom));
+ return serverDomains.myPublicIP() + ":8080" + ((subdom == null) ? "" : ("/" + subdom));
}
return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
} else {
diff --git a/source/de/anomic/yacy/yacyURL.java b/source/de/anomic/yacy/yacyURL.java index c55433828..69e97deb0 100644 --- a/source/de/anomic/yacy/yacyURL.java +++ b/source/de/anomic/yacy/yacyURL.java @@ -46,14 +46,7 @@ public class yacyURL implements Serializable { * generated with svn4751 on 2008-05-01 */ private static final long serialVersionUID = -1173233022912141884L; - - public static final int TLD_any_zone_filter = 255; // from TLD zones can be filtered during search; this is the catch-all filter - - public static String dummyHash = ""; - static { - // create a dummy hash - for (int i = 0; i < yacySeedDB.commonHashLength; i++) dummyHash += "-"; - } + public static final int TLD_any_zone_filter = 255; // from TLD zones can be filtered during search; this is the catch-all filter // class variables private String protocol, host, userInfo, path, quest, ref, hash; diff --git a/source/migration.java b/source/migration.java index f424fd6d2..a1685695d 100644 --- a/source/migration.java +++ b/source/migration.java @@ -230,21 +230,6 @@ public class migration { sb.setConfig("locale.translated_html", "DATA/LOCALE/htroot"); } - // migration for port forwarding settings - if ((value = sb.getConfig("portForwardingHost","")).length() > 0) { - sb.setConfig("portForwarding.Enabled", sb.getConfig("portForwardingEnabled","")); - if (sb.getConfigBool("portForwardingEnabled", false)) { - sb.setConfig("portForwarding.Type", "sch"); - } - - sb.setConfig("portForwarding.sch.UseProxy", sb.getConfig("portForwardingUseProxy","")); - sb.setConfig("portForwarding.sch.Port", sb.getConfig("portForwardingPort","")); - sb.setConfig("portForwarding.sch.Host", sb.getConfig("portForwardingHost","")); - sb.setConfig("portForwarding.sch.HostPort", sb.getConfig("portForwardingHostPort","")); - sb.setConfig("portForwarding.sch.HostUser", sb.getConfig("portForwardingHostUser","")); - sb.setConfig("portForwarding.sch.HostPwd", sb.getConfig("portForwardingHostPwd","")); - } - // migration for blacklists if ((value = sb.getConfig("proxyBlackListsActive","")).length() > 0) { sb.setConfig("proxy.BlackLists", value); diff --git a/source/yacy.java b/source/yacy.java index e61529276..b3216ddf6 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -643,7 +643,7 @@ public final class yacy { int cacheMem = (int)(serverMemory.max() - serverMemory.total());
if (cacheMem < 2048000) throw new OutOfMemoryError("Not enough memory available to start clean up.");
- plasmaWordIndex wordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, networkName, log);
+ plasmaWordIndex wordIndex = new plasmaWordIndex(networkName, log, indexPrimaryRoot, indexSecondaryRoot);
Iterator<indexContainer> indexContainerIterator = wordIndex.wordContainers("AAAAAAAAAAAA", false, false);
long urlCounter = 0, wordCounter = 0;
@@ -834,7 +834,7 @@ public final class yacy { try {
Iterator<indexContainer> indexContainerIterator = null;
if (resource.equals("all")) {
- WordIndex = new plasmaWordIndex(indexPrimaryRoot, indexSecondaryRoot, "freeworld", log);
+ WordIndex = new plasmaWordIndex("freeworld", log, indexPrimaryRoot, indexSecondaryRoot);
indexContainerIterator = WordIndex.wordContainers(wordChunkStartHash, false, false);
}
int counter = 0;
@@ -889,7 +889,7 @@ public final class yacy { public static void testPeerDB(File homePath) {
try {
- File yacyDBPath = new File(homePath, "DATA/YACYDB");
+ File yacyDBPath = new File(homePath, "DATA/INDEX/freeworld/NETWORK");
String[] dbFileNames = {"seed.new.db","seed.old.db","seed.pot.db"};
for (int i=0; i < dbFileNames.length; i++) {
|