diff options
| author | Michael Peter Christen <mc@yacy.net> | 2014-09-30 14:53:52 +0200 |
|---|---|---|
| committer | Michael Peter Christen <mc@yacy.net> | 2014-09-30 14:53:52 +0200 |
| commit | 6491270b3a17a26834956d9aaf396b211e0d6b2b (patch) | |
| tree | 9e74950911d0fae3145ffca84fc323a7508a6d9c | |
| parent | 67cd4c37bd5f46c84a89d7a76283e2621e46904d (diff) | |
large IPv6 redesign of peer ping methods!
removed preferred IPv4 in start options and added a new field IP6 in
peer seeds which will contain one or more IPv6 addresses. Now every peer
has one or more IP addresses assigned, even several IPv6 addresses are
possible. The peer-ping process must check all given and possible IP
addresses for a backping and return the one IP which was successful when
pinging the peer. The ping-ing peer must be able to recognize which of
the given IPs are available for outside access of the peer and store
this accordingly. If only one IPv6 address is available and no IPv4,
then the IPv6 is stored in the old IP field of the seed DNA.
Many methods in Seed.java are now marked as @deprecated because they had
been used for a single IP only. There is still a large construction site
left in YaCy now where all these deprecated methods must be replaced
with new method calls. The 'extra'-IPs, used by cluster assignment had
been removed since that can be replaced with IPv6 usage in p2p clusters.
All clusters must now use IPv6 if they want an intranet-routing.
45 files changed, 663 insertions, 657 deletions
diff --git a/addon/YaCy.app/Contents/Info.plist b/addon/YaCy.app/Contents/Info.plist index 54931fed0..a7fee64bc 100644 --- a/addon/YaCy.app/Contents/Info.plist +++ b/addon/YaCy.app/Contents/Info.plist @@ -25,7 +25,7 @@ <key>Java</key> <dict> <key>VMOptions</key> - <string>-Xmx600m -Xms600m -Djava.net.preferIPv4Stack=true -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory</string> + <string>-Xmx600m -Xms600m -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory</string> <key>WorkingDirectory</key> <string>$APP_PACKAGE/Contents/Resources/Java</string> <key>MainClass</key> diff --git a/addon/yacyInit.m4 b/addon/yacyInit.m4 index 333028cd2..b165f36ea 100644 --- a/addon/yacyInit.m4 +++ b/addon/yacyInit.m4 @@ -57,7 +57,7 @@ SHUTDOWN_TIMEOUT=50 # Default niceness if not set in config file NICE_VAL=0 -JAVA_ARGS="-server -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory" +JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8 -Dsolr.directoryFactory=solr.MMapDirectoryFactory" ifdef(`openSUSE', `dnl . /etc/rc.status diff --git a/htroot/Blog.java b/htroot/Blog.java index 1a2ab6eea..7a5a59a08 100644 --- a/htroot/Blog.java +++ b/htroot/Blog.java @@ -192,7 +192,7 @@ public class Blog { prop.putHTML("mode_author", UTF8.String(author)); prop.putHTML("mode_subject", post.get("subject","")); prop.put("mode_date", dateString(new Date())); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", post.get("content", "")); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", "")); prop.putHTML("mode_page-code", post.get("content", "")); } else { @@ -327,7 +327,7 @@ public class Blog { prop.put("mode_entries_" + number + "_page", entry.getPage()); prop.put("mode_entries_" + number + "_timestamp", entry.getTimestamp()); } else { - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_entries_" + number + "_page", entry.getPage()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(sb.peers.mySeed().getIP()), "mode_entries_" + number + "_page", entry.getPage()); } if (hasRights) { diff --git a/htroot/BlogComments.java b/htroot/BlogComments.java index 51849b73f..06a2d3508 100644 --- a/htroot/BlogComments.java +++ b/htroot/BlogComments.java @@ -175,7 +175,7 @@ public class BlogComments { prop.putHTML("mode_allow_author", UTF8.String(author)); prop.putHTML("mode_subject", post.get("subject","")); prop.put("mode_date", dateString(new Date())); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", post.get("content", "")); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", "")); prop.put("mode_page-code", post.get("content", "")); } else { // show blog-entry/entries @@ -191,7 +191,7 @@ public class BlogComments { prop.putHTML("mode_allow_author", UTF8.String(author)); prop.put("mode_comments", page.getCommentsSize()); prop.put("mode_date", dateString(page.getDate())); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", page.getPage()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", page.getPage()); if (hasRights) { prop.put("mode_admin", "1"); prop.put("mode_admin_pageid", page.getKey()); @@ -234,7 +234,7 @@ public class BlogComments { if (!xml) { prop.putHTML("mode_entries_"+count+"_subject", UTF8.String(entry.getSubject())); prop.putHTML("mode_entries_"+count+"_author", UTF8.String(entry.getAuthor())); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_entries_"+count+"_page", entry.getPage()); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_entries_"+count+"_page", entry.getPage()); } else { prop.putHTML("mode_entries_"+count+"_subject", UTF8.String(entry.getSubject())); prop.putHTML("mode_entries_"+count+"_author", UTF8.String(entry.getAuthor())); diff --git a/htroot/ConfigNetwork_p.java b/htroot/ConfigNetwork_p.java index 7cebf0e79..2ca5a194b 100644 --- a/htroot/ConfigNetwork_p.java +++ b/htroot/ConfigNetwork_p.java @@ -174,10 +174,10 @@ public class ConfigNetwork_p prop.putHTML("cluster.peers.ipport", sb.getConfig("cluster.peers.ipport", "")); prop.putHTML("cluster.peers.yacydomain", sb.getConfig("cluster.peers.yacydomain", "")); StringBuilder hashes = new StringBuilder(); - for ( final byte[] h : sb.clusterhashes.keySet() ) { + for (final byte[] h: sb.clusterhashes) { hashes.append(", ").append(ASCII.String(h)); } - if ( hashes.length() > 2 ) { + if (hashes.length() > 2) { hashes = hashes.delete(0, 2); } diff --git a/htroot/CrawlStartScanner_p.java b/htroot/CrawlStartScanner_p.java index 09072404a..a8651b240 100644 --- a/htroot/CrawlStartScanner_p.java +++ b/htroot/CrawlStartScanner_p.java @@ -97,7 +97,7 @@ public class CrawlStartScanner_p } else { ip = Domains.myPublicLocalIP(); if ( Domains.isThisHostIP(ip) ) { - ip = sb.peers.mySeed().getInetAddress(); + ip = Domains.dnsResolve(sb.peers.mySeed().getIP()); } } if ( ip != null ) { diff --git a/htroot/MessageSend_p.java b/htroot/MessageSend_p.java index f192dd59e..443b8b88c 100644 --- a/htroot/MessageSend_p.java +++ b/htroot/MessageSend_p.java @@ -106,7 +106,7 @@ public class MessageSend_p { prop.putXML("mode_permission_message", message);
prop.putHTML("mode_permission_hash", hash);
if (post.containsKey("preview")) {
- prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_permission_previewmessage", message);
+ prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_permission_previewmessage", message);
}
diff --git a/htroot/Messages_p.java b/htroot/Messages_p.java index 45c59a749..ac05dc53f 100644 --- a/htroot/Messages_p.java +++ b/htroot/Messages_p.java @@ -162,7 +162,7 @@ public class Messages_p { prop.putXML("mode_subject", message.subject());
String theMessage = null;
theMessage = UTF8.String(message.message());
- prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_message", theMessage);
+ prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_message", theMessage);
prop.put("mode_hash", message.authorHash());
prop.putXML("mode_key", key);
}
diff --git a/htroot/Network.html b/htroot/Network.html index f75b1a760..a96c2888c 100644 --- a/htroot/Network.html +++ b/htroot/Network.html @@ -162,7 +162,7 @@ document.getElementById("apilink").setAttribute("href", "Network.xml?" + window. <td align="right">#[rU]#</td>
#(complete)#
::
- <td><a href="http://#[ip]#:#[port]#/Network.html?page=1&ip=">http://#[ip]#:#[port]#</a></td>
+ <td><a href="http://#[ip]#:#[port]#/Network.html?page=1&ip=">http://#[ip]#:#[port]#</a> #[ips]#</td>
<td >#[hash]#</td>
<td >#[age]#</td>
<td align="right">#[seeds]#</td>
diff --git a/htroot/Network.java b/htroot/Network.java index fed15aab5..8b070dd22 100644 --- a/htroot/Network.java +++ b/htroot/Network.java @@ -191,7 +191,7 @@ public class Network { if (sb.peers.mySeed() != null) {
prop.put("table_my-hash", sb.peers.mySeed().hash );
- prop.put("table_my-ip", sb.peers.mySeed().getIP() );
+ prop.put("table_my-ip", sb.peers.mySeed().getIPs().toString());
prop.put("table_my-port", sb.peers.mySeed().getPort() );
}
@@ -210,11 +210,11 @@ public class Network { Seed peer = new Seed(post.get("peerHash"), map);
sb.updateMySeed();
- final int added = Protocol.hello(sb.peers.mySeed(), sb.peers.peerActions, peer.getPublicAddress(), peer.hash, peer.getName());
+ final int added = Protocol.hello(sb.peers.mySeed(), sb.peers.peerActions, peer);
if (added <= 0) {
prop.put("table_comment",1);
- prop.putHTML("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getPublicAddress());
+ prop.putHTML("table_comment_status","publish: disconnected peer '" + peer.getName() + "/" + post.get("peerHash") + "' from " + peer.getIPs());
} else {
peer = sb.peers.getConnected(peer.hash);
if (peer == null) {
@@ -222,7 +222,7 @@ public class Network { prop.putHTML("table_comment_status","publish: disconnected peer 'UNKNOWN/" + post.get("peerHash") + "' from UNKNOWN");
} else {
prop.put("table_comment",2);
- prop.putHTML("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress());
+ prop.putHTML("table_comment_status","publish: handshaked " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) + " peer '" + peer.getName() + "' at " + peer.getIPs());
prop.putHTML("table_comment_details",peer.toString());
}
}
@@ -397,7 +397,8 @@ public class Network { prop.putHTML(STR_TABLE_LIST + conCount + "_location", location);
if (complete) {
prop.put(STR_TABLE_LIST + conCount + "_complete", 1);
- prop.put(STR_TABLE_LIST + conCount + "_complete_ip", seed.getIP() );
+ prop.putHTML(STR_TABLE_LIST + conCount + "_complete_ip", seed.getIP() );
+ prop.putHTML(STR_TABLE_LIST + conCount + "_complete_ips", seed.getIPs().toString() );
prop.put(STR_TABLE_LIST + conCount + "_complete_port", seed.get(Seed.PORT, "-") );
prop.put(STR_TABLE_LIST + conCount + "_complete_hash", seed.hash);
prop.put(STR_TABLE_LIST + conCount + "_complete_age", seed.getAge());
@@ -447,7 +448,7 @@ public class Network { }
}
prop.put(STR_TABLE_LIST + conCount + "_nodestate", seed.getFlagRootNode() ? 1 : 0);
- prop.put(STR_TABLE_LIST + conCount + "_nodestate_ip", seed.getIP() );
+ prop.put(STR_TABLE_LIST + conCount + "_nodestate_ip", seed.getIP());
prop.put(STR_TABLE_LIST + conCount + "_nodestate_port", seed.get(Seed.PORT, "-") );
if (seed.getFlagAcceptRemoteIndex()) {
prop.put(STR_TABLE_LIST + conCount + "_dhtreceive_peertags", "");
diff --git a/htroot/SettingsAck_p.java b/htroot/SettingsAck_p.java index fc72eea58..3251b384d 100644 --- a/htroot/SettingsAck_p.java +++ b/htroot/SettingsAck_p.java @@ -177,7 +177,7 @@ public class SettingsAck_p { } else if (staticIP.startsWith("https://")) { if (staticIP.length() > 8) { staticIP = staticIP.substring(8); } else { staticIP = ""; } } - String error = Seed.isProperIP(staticIP); + String error = Seed.isProperIP(staticIP) ? null : "ip not proper: " + staticIP; if (error == null) { serverCore.useStaticIP = true; sb.peers.mySeed().setIP(staticIP); diff --git a/htroot/Status.java b/htroot/Status.java index 3d43644df..42a5d7a47 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -27,8 +27,8 @@ // javac -classpath .:../Classes Status.java
// if the shell's current path is HTROOT
-import java.net.InetAddress;
import java.util.Date;
+import java.util.Set;
import net.yacy.cora.protocol.ConnectionInfo;
import net.yacy.cora.protocol.Domains;
@@ -189,8 +189,9 @@ public class Status } else {
prop.put("extPortFormat", "0");
}
- final InetAddress hostIP = Domains.myPublicLocalIP();
- prop.put("host", hostIP != null ? hostIP.getHostAddress() : "Unkown IP");
+
+ Set<String> ips = Domains.myPublicIPs();
+ prop.put("host", ips.toString());
// ssl support
prop.put("sslSupport", sb.getConfig("keyStore", "").isEmpty() || !sb.getConfigBool("server.https", false) ? 0 : 1);
@@ -227,7 +228,7 @@ public class Status prop.putNum("peerStatistics_disconnects", sb.peers.peerActions.disconnects);
prop.put("peerStatistics_connects", Formatter.number(sb.peers.mySeed().get(Seed.CCOUNT, "0")));
thisHash = sb.peers.mySeed().hash;
- if ( sb.peers.mySeed().getPublicAddress() == null ) {
+ if ( sb.peers.mySeed().getIPs().size() == 0 ) {
prop.put("peerAddress", "0"); // not assigned + instructions
prop.put("warningGoOnline", "1");
} else {
diff --git a/htroot/Surftips.java b/htroot/Surftips.java index 0926a8728..dad98d129 100644 --- a/htroot/Surftips.java +++ b/htroot/Surftips.java @@ -277,7 +277,7 @@ public class Surftips { Seed seed = sb.peers.getConnected(record.originator()); if (seed == null) seed = sb.peers.getDisconnected(record.originator()); if (seed != null) { - url = "http://" + seed.getPublicAddress() + "/Wiki.html?page=" + record.attribute("page", ""); + url = "http://" + seed.getPublicAddress(seed.getIP()) + "/Wiki.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ UTF8.getBytes(url), UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")), @@ -292,7 +292,7 @@ public class Surftips { Seed seed = sb.peers.getConnected(record.originator()); if (seed == null) seed = sb.peers.getDisconnected(record.originator()); if (seed != null) { - url = "http://" + seed.getPublicAddress() + "/Blog.html?page=" + record.attribute("page", ""); + url = "http://" + seed.getPublicAddress(seed.getIP()) + "/Blog.html?page=" + record.attribute("page", ""); entry = rowdef.newEntry(new byte[][]{ UTF8.getBytes(url), UTF8.getBytes(record.attribute("author", "Anonymous") + ": " + record.attribute("page", "")), diff --git a/htroot/ViewProfile.java b/htroot/ViewProfile.java index ad96150f4..7a56c5ca8 100644 --- a/htroot/ViewProfile.java +++ b/htroot/ViewProfile.java @@ -113,7 +113,6 @@ public class ViewProfile { }
// try to get the profile from remote peer
- if (sb.clusterhashes != null) seed.setAlternativeAddress(sb.clusterhashes.get(seed.hash.getBytes()));
profile = Protocol.getProfile(seed);
// if profile did not arrive, say that peer is disconnected
@@ -170,7 +169,7 @@ public class ViewProfile { prop.put("success_" + key, "1");
// only comments get "wikified"
if(key.equals("comment")){
- prop.putWiki(sb.peers.mySeed().getClusterAddress(),
+ prop.putWiki(sb.peers.mySeed().getPublicAddress(),
"success_" + key + "_value",
entry.getValue().replaceAll("\r", "").replaceAll("\\\\n", "\n"));
prop.put("success_" + key + "_b64value", Base64Order.standardCoder.encodeString(entry.getValue()));
diff --git a/htroot/Wiki.java b/htroot/Wiki.java index b74c76140..7ca1185fe 100644 --- a/htroot/Wiki.java +++ b/htroot/Wiki.java @@ -147,7 +147,7 @@ public class Wiki { prop.putHTML("mode_pagename", pagename);
prop.putHTML("mode_author", author);
prop.put("mode_date", dateString(new Date()));
- prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", post.get("content", ""));
+ prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", post.get("content", ""));
prop.putHTML("mode_page-code", post.get("content", ""));
}
//end contrib of [MN]
@@ -237,7 +237,7 @@ public class Wiki { prop.putHTML("mode_versioning_pagename", pagename);
prop.putHTML("mode_versioning_author", oentry.author());
prop.put("mode_versioning_date", dateString(oentry.date()));
- prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_versioning_page", oentry.page());
+ prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_versioning_page", oentry.page());
prop.putHTML("mode_versioning_page-code", UTF8.String(oentry.page()));
}
} catch (final IOException e) {
@@ -252,7 +252,7 @@ public class Wiki { prop.putHTML("mode_pagename", pagename);
prop.putHTML("mode_author", page.author());
prop.put("mode_date", dateString(page.date()));
- prop.putWiki(sb.peers.mySeed().getClusterAddress(), "mode_page", page.page());
+ prop.putWiki(sb.peers.mySeed().getPublicAddress(), "mode_page", page.page());
prop.put("controls", "0");
prop.putHTML("controls_pagename", pagename);
diff --git a/htroot/goto_p.java b/htroot/goto_p.java index 387665ac4..b958742d9 100644 --- a/htroot/goto_p.java +++ b/htroot/goto_p.java @@ -43,7 +43,7 @@ public class goto_p { * hash= of remote peer
* path= path part to forward to
*/
- public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
+ public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) {
final Switchboard sb = (Switchboard) env;
final serverObjects prop = new serverObjects();
diff --git a/htroot/mediawiki_p.java b/htroot/mediawiki_p.java index 3a6bf3864..140c9e115 100644 --- a/htroot/mediawiki_p.java +++ b/htroot/mediawiki_p.java @@ -70,7 +70,7 @@ public class mediawiki_p { page = page.substring(p, q); prop.putHTML("title", title); - prop.putWiki(sb.peers.mySeed().getClusterAddress(), "page", page); + prop.putWiki(sb.peers.mySeed().getPublicAddress(), "page", page); return prop; } diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 7dadac23f..2659c14a1 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -30,17 +30,18 @@ import java.io.IOException;
import java.net.InetAddress;
import java.util.Iterator;
+import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import net.yacy.cora.protocol.Domains;
import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.util.ConcurrentLog;
import net.yacy.peers.Network;
import net.yacy.peers.DHTSelection;
import net.yacy.peers.Protocol;
import net.yacy.peers.Seed;
import net.yacy.peers.graphics.ProfilingGraph;
-import net.yacy.peers.operation.yacyVersion;
import net.yacy.search.EventTracker;
import net.yacy.search.Switchboard;
import net.yacy.server.serverCore;
@@ -58,6 +59,7 @@ public final class hello { final long start = System.currentTimeMillis();
prop.put("message", "none");
final String clientip = header.get(HeaderFramework.CONNECTION_PROP_CLIENTIP, "<unknown>"); // read an artificial header addendum
+ ConcurrentLog.info("**hello-DEBUG**", "client request from = " + clientip);
final InetAddress ias = Domains.dnsResolve(clientip);
long time = System.currentTimeMillis();
final long time_dnsResolve = System.currentTimeMillis() - time;
@@ -78,13 +80,13 @@ public final class hello { return prop;
}
-// final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer
+// final String iam = (String) post.get("iam", ""); // complete seed of the requesting peer
// final String mytime = (String) post.get(MYTIME, ""); //
final String key = post.get("key", ""); // transmission key for response
final String seed = post.get("seed", "");
int count = post.getInt("count", 0);
- final long magic = post.getLong("magic", 0);
-// final Date remoteTime = yacyCore.parseUniversalDate(post.get(MYTIME)); // read remote time
+ // final long magic = post.getLong("magic", 0);
+ // final Date remoteTime = yacyCore.parseUniversalDate(post.get(MYTIME)); // read remote time
if (seed.length() > Seed.maxsize) {
Network.log.info("hello/server: rejected contacting seed; too large (" + seed.length() + " > " + Seed.maxsize + ", time_dnsResolve=" + time_dnsResolve + ")");
prop.put("message", "your seed is too long (" + seed.length() + ")");
@@ -105,21 +107,21 @@ public final class hello { return prop;
}
-// final String properTest = remoteSeed.isProper();
- // The remote peer might not know its IP yet, so don't abort if the IP check fails
-// if ((properTest != null) && (! properTest.substring(0,1).equals("IP"))) { return null; }
-
// we easily know the caller's IP:
final String userAgent = header.get(HeaderFramework.USER_AGENT, "<unknown>");
sb.peers.peerActions.setUserAgent(clientip, userAgent);
- final String reportedip = remoteSeed.getIP();
+ final Set<String> reportedips = remoteSeed.getIPs();
final String reportedPeerType = remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR);
- final double clientversion = remoteSeed.getVersion();
+ //final double clientversion = remoteSeed.getVersion();
- if ((reportedip + ':' + remoteSeed.getPort()).equals(sb.peers.mySeed().getPublicAddress())) {
- // reject a self-ping
- prop.put("message", "I am I");
- return prop;
+ if (remoteSeed.getPort() == sb.peers.mySeed().getPort()) {
+ for (String reportedip: reportedips) {
+ if (sb.peers.mySeed().getIPs().contains(reportedip)) {
+ // reject a self-ping
+ prop.put("message", "I am I");
+ return prop;
+ }
+ }
}
if (remoteSeed.hash.equals(sb.peers.mySeed().hash)) {
// reject a ping with my own hash
@@ -140,58 +142,42 @@ public final class hello { }
long[] callback = new long[]{-1, -1};
- if (sb.clusterhashes != null) remoteSeed.setAlternativeAddress(sb.clusterhashes.get(remoteSeed.hash.getBytes()));
// if the remote client has reported its own IP address and the client supports
// the port forwarding feature (if client version >= 0.383) then we try to
// connect to the reported IP address first
long time_backping = 0;
String backping_method = "none";
- if (reportedip.length() > 0 &&
- !clientip.equals(reportedip) &&
- clientversion >= yacyVersion.YACY_SUPPORTS_PORT_FORWARDING &&
- magic != 0) {
-
- // try first the reportedip, since this may be a connect from a port-forwarding host
- prop.put("yourip", reportedip);
- remoteSeed.setIP(reportedip);
- time = System.currentTimeMillis();
- callback = Protocol.queryRWICount(remoteSeed, "Tq418bNZd6AO");
- time_backping = System.currentTimeMillis() - time;
- backping_method = "reportedip=" + reportedip;
- } else {
- prop.put("yourip", ias.getHostAddress());
- remoteSeed.setIP(ias.getHostAddress());
+ boolean success = false;
+ // TODO: make this a concurrent process
+ if (!serverCore.useStaticIP || !ias.isSiteLocalAddress()) {
+ reportedips.add(ias.getHostAddress());
}
-
- // if the previous attempt (using the reported ip address) was not successful,
- // then try the ip where the request came from
- if (callback[0] < 0 || (magic != 0 && magic != callback[1])) {
- boolean isNotLocal = true;
-
- // we are only allowed to connect to the client IP address if it's not our own address
- if (serverCore.useStaticIP) {
- isNotLocal = !ias.isSiteLocalAddress();
- }
-
- if (isNotLocal) {
-
- prop.put("yourip", clientip);
- remoteSeed.setIP(clientip);
- time = System.currentTimeMillis();
- callback = Protocol.queryRWICount(remoteSeed, "Tq418bNZd6AO"); // hash for "www"; the actual count is irrelevant, we just want to know if this works
- time_backping = System.currentTimeMillis() - time;
- backping_method = "clientip=" + clientip;
+ final int connectedBefore = sb.peers.sizeConnected();
+ ConcurrentLog.info("**hello-DEBUG**", "peer " + remoteSeed.getName() + " challenged us with IPs " + reportedips);
+ int callbackRemain = Math.min(5, reportedips.size());
+ long callbackStart = System.currentTimeMillis();
+ if (reportedips.size() > 0) {
+ for (String reportedip: reportedips) {
+ int partialtimeout = ((int) (callbackStart + 6500 - System.currentTimeMillis())) / callbackRemain; // bad hack until a concurrent version is implemented
+ if (partialtimeout <= 0) break;
+ ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " is handled");
+ if (Seed.isProperIP(reportedip)) {
+ ConcurrentLog.info("**hello-DEBUG**", "starting callback to reportedip = " + reportedip + ", timeout = " + partialtimeout);
+ prop.put("yourip", reportedip);
+ remoteSeed.setIP(reportedip);
+ time = System.currentTimeMillis();
+ callback = Protocol.queryRWICount(remoteSeed.getPublicAddress(reportedip), remoteSeed.hash, partialtimeout);
+ ConcurrentLog.info("**hello-DEBUG**", "reportedip = " + reportedip + " returns callback " + (callback == null ? "NULL" : callback[0]));
+ time_backping = System.currentTimeMillis() - time;
+ backping_method = "reportedip=" + reportedip;
+ if (callback[0] >= 0) { success = true; break; }
+ if (callbackRemain-- <= 0) break; // no more tries left / restrict to a limited number of ips
+ }
}
}
-
-// System.out.println("YACYHELLO: YOUR IP=" + clientip);
- // set lastseen value (we have seen that peer, it contacted us!)
- remoteSeed.setLastSeenUTC();
-
- // assign status
- final int connectedBefore = sb.peers.sizeConnected();
- if (callback[0] >= 0) {
+ if (success) {
+ ConcurrentLog.info("**hello-DEBUG**", "success for IP(s) " + remoteSeed.getIPs() + ", port " + remoteSeed.getPort());
if (remoteSeed.get(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR) == null) {
prop.put(Seed.YOURTYPE, Seed.PEERTYPE_SENIOR);
remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR);
@@ -202,26 +188,27 @@ public final class hello { remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR);
}
// connect the seed
- Network.log.info("hello/server: responded remote senior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
+ Network.log.info("hello/server: responded remote " + reportedPeerType + " peer '" + remoteSeed.getName() + "' from " + reportedips + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
sb.peers.peerActions.peerArrival(remoteSeed, true);
} else {
+ ConcurrentLog.info("**hello-DEBUG**", "fail for IP(s) " + remoteSeed.getIPs() + ", port " + remoteSeed.getPort());
+ prop.put("yourip", ias.getHostAddress());
+ remoteSeed.setIP(ias.getHostAddress());
prop.put(Seed.YOURTYPE, Seed.PEERTYPE_JUNIOR);
remoteSeed.put(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR);
- Network.log.info("hello/server: responded remote junior peer '" + remoteSeed.getName() + "' from " + reportedip + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
+ Network.log.info("hello/server: responded remote " + reportedPeerType + " peer '" + remoteSeed.getName() + "' from " + reportedips + ", time_dnsResolve=" + time_dnsResolve + ", time_backping=" + time_backping + ", method=" + backping_method + ", urls=" + callback[0]);
// no connection here, instead store junior in connection cache
if ((remoteSeed.hash != null) && (remoteSeed.isProper(false) == null)) {
sb.peers.peerActions.peerPing(remoteSeed);
}
}
+ remoteSeed.setLastSeenUTC();
final int connectedAfter = sb.peers.sizeConnected();
// update event tracker
EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(remoteSeed.getName(), sb.peers.myName(), false, connectedAfter - connectedBefore), false);
if (!(prop.get(Seed.YOURTYPE)).equals(reportedPeerType)) {
- Network.log.info("hello/server: changing remote peer '" + remoteSeed.getName() +
- "' [" + reportedip +
- "] peerType from '" + reportedPeerType +
- "' to '" + prop.get(Seed.YOURTYPE) + "'.");
+ Network.log.info("hello/server: changing remote peer '" + remoteSeed.getName() + "' " + reportedips + " peerType from '" + reportedPeerType + "' to '" + prop.get(Seed.YOURTYPE) + "'.");
}
final StringBuilder seeds = new StringBuilder(768);
@@ -262,7 +249,7 @@ public final class hello { prop.put("seedlist", seeds.toString());
// return rewrite properties
prop.put("message", "ok " + seed.length());
- Network.log.info("hello/server: responded remote peer '" + remoteSeed.getName() + "' [" + reportedip + "] in " + (System.currentTimeMillis() - start) + " milliseconds");
+ Network.log.info("hello/server: responded remote peer '" + remoteSeed.getName() + "' " + reportedips + " in " + (System.currentTimeMillis() - start) + " milliseconds");
return prop;
}
diff --git a/htroot/yacy/query.java b/htroot/yacy/query.java index a2f7d27c9..5e881c8ef 100644 --- a/htroot/yacy/query.java +++ b/htroot/yacy/query.java @@ -109,7 +109,7 @@ public final class query { if (obj.equals("lurlcount")) {
// return the number of all available l-url's
- prop.put("response", 1 /*sb.index.fulltext().collectionSize()*/); // patched to not call collectionSize() any more because the acutal size is not needed. Instead, rwicount should be called
+ prop.put("response", "1" /*sb.index.fulltext().collectionSize()*/); // patched to not call collectionSize() any more because the acutal size is not needed. Instead, rwicount should be called
return prop;
}
diff --git a/installYaCyWindowsService.bat b/installYaCyWindowsService.bat index 38e701383..d91c9599d 100644 --- a/installYaCyWindowsService.bat +++ b/installYaCyWindowsService.bat @@ -3,7 +3,7 @@ title YaCy Windows Service Install :STARTJAVA
REM set the Java options
-set javaopts=-Xss256k;-XX:MaxPermSize=256m;-Djava.net.preferIPv4Stack=true;-Djava.awt.headless=true;-Dfile.encoding=UTF-8
+set javaopts=-Xss256k;-XX:MaxPermSize=256m;-Djava.awt.headless=true;-Dfile.encoding=UTF-8
REM set max Java heap memory (in MB)
set jmx=800
diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java index bf759fd2a..8a0d1ef21 100644 --- a/source/net/yacy/cora/protocol/Domains.java +++ b/source/net/yacy/cora/protocol/Domains.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.lang.reflect.Array; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.net.Inet4Address; import java.net.InetAddress; import java.net.NetworkInterface; import java.net.SocketException; @@ -68,15 +69,19 @@ public class Domains { private final static ConcurrentLog log = new ConcurrentLog(Domains.class.getName()); - public static final String LOCALHOST = "127.0.0.1"; // replace with IPv6 0:0:0:0:0:0:0:1 ? + public static final String LOCALHOST = "localhost"; // replace with IPv6 0:0:0:0:0:0:0:1 ? private static String LOCALHOST_NAME = LOCALHOST; // this will be replaced with the actual name of the local host private static Class<?> InetAddressLocatorClass; private static Method InetAddressLocatorGetLocaleInetAddressMethod; private static final Set<String> ccSLD_TLD = new HashSet<String>(); private static final String PRESENT = ""; - private static final Pattern LOCALHOST_PATTERNS = Pattern.compile("(127\\..*)|(localhost)|(\\[?(fe80|0)\\:0\\:0\\:0\\:0\\:0\\:0\\:1.*)"); - private static final Pattern INTRANET_PATTERNS = Pattern.compile("(10\\..*)|(127\\..*)|(172\\.(1[6-9]|2[0-9]|3[0-1])\\..*)|(169\\.254\\..*)|(192\\.168\\..*)|(localhost)|(\\[?\\:\\:1/.*)|(\\[?fc.*\\:.*)|(\\[?fd.*\\:.*)|(\\[?(fe80|0)\\:0\\:0\\:0\\:0\\:0\\:0\\:1.*)", Pattern.CASE_INSENSITIVE); + private static final String LOCALHOST_IPv4_PATTERN = "(127\\..*)"; + private static final String LOCALHOST_IPv6_PATTERN = "(\\[?fe80\\:\\:(/.*|\\z))|(\\[?0\\:0\\:0\\:0\\:0\\:0\\:0\\:1.*)|(\\[?\\:\\:1(/.*|\\z))"; + private static final String INTRANET_IPv4_PATTERN = "(10\\..*)|(172\\.(1[6-9]|2[0-9]|3[0-1])\\..*)|(169\\.254\\..*)|(192\\.168\\..*)"; + private static final String INTRANET_IPv6_PATTERN = "(\\[?(fc|fd).*\\:.*)"; + private static final Pattern LOCALHOST_PATTERNS = Pattern.compile("(localhost)|" + LOCALHOST_IPv4_PATTERN + "|" + LOCALHOST_IPv6_PATTERN, Pattern.CASE_INSENSITIVE); + private static final Pattern INTRANET_PATTERNS = Pattern.compile(LOCALHOST_PATTERNS.pattern() + "|" + INTRANET_IPv4_PATTERN + "|" + INTRANET_IPv6_PATTERN, Pattern.CASE_INSENSITIVE); private static final int MAX_NAME_CACHE_HIT_SIZE = 10000; private static final int MAX_NAME_CACHE_MISS_SIZE = 1000; @@ -90,6 +95,92 @@ public class Domains { public static long cacheHit_Hit = 0, cacheHit_Miss = 0, cacheHit_Insert = 0; // for statistics only; do not write public static long cacheMiss_Hit = 0, cacheMiss_Miss = 0, cacheMiss_Insert = 0; // for statistics only; do not write + private static Set<InetAddress> myHostAddresses = new HashSet<InetAddress>(); + private static Set<InetAddress> localHostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses + private static Set<InetAddress> publicIPv4HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses + private static Set<InetAddress> publicIPv6HostAddresses = new HashSet<InetAddress>(); // subset of myHostAddresses + private static Set<String> myHostNames = new HashSet<String>(); + private static Set<String> localHostNames = new HashSet<String>(); // subset of myHostNames + private static Set<String> publicIPv4HostNames = new HashSet<String>(); // subset of myHostNames + private static Set<String> publicIPv6HostNames = new HashSet<String>(); // subset of myHostNames + static { + myHostNames.add(LOCALHOST); + localHostNames.add(LOCALHOST); + try { + InetAddress localHostAddress = InetAddress.getLocalHost(); + if (localHostAddress != null) myHostAddresses.add(localHostAddress); + } catch (final UnknownHostException e) {} + try { + final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME); + if (moreAddresses != null) myHostAddresses.addAll(Arrays.asList(moreAddresses)); + } catch (final UnknownHostException e) {} + + // to get the local host name, a dns lookup is necessary. + // if such a lookup blocks, it can cause that the static initiatializer does not finish fast + // therefore we start the host name lookup as concurrent thread + // meanwhile the host name is "127.0.0.1" which is not completely wrong + new Thread() { + @Override + public void run() { + Thread.currentThread().setName("Domains: init"); + // try to get local addresses from interfaces + try { + final Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces(); + while (nis.hasMoreElements()) { + final NetworkInterface ni = nis.nextElement(); + final Enumeration<InetAddress> addrs = ni.getInetAddresses(); + while (addrs.hasMoreElements()) { + final InetAddress addr = addrs.nextElement(); + if (addr != null) myHostAddresses.add(addr); + } + } + } catch (final SocketException e) { + } + + // now look up the host name + try { + LOCALHOST_NAME = getHostName(InetAddress.getLocalHost()); + } catch (final UnknownHostException e) {} + + // after the host name was resolved, we try to look up more local addresses + // using the host name: + try { + final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME); + if (moreAddresses != null) myHostAddresses.addAll(Arrays.asList(moreAddresses)); + } catch (final UnknownHostException e) { + } + + // fill a cache of local host names + for (final InetAddress a: myHostAddresses) { + final String hostname = getHostName(a); + if (hostname != null) { + myHostNames.add(hostname); + myHostNames.add(a.getHostAddress()); + } + // we write the local tests into variables to be able to debug these values + boolean isAnyLocalAddress = a.isAnyLocalAddress(); + boolean isLinkLocalAddress = a.isLinkLocalAddress(); // true i.e. for localhost/fe80:0:0:0:0:0:0:1%1, myhost.local/fe80:0:0:0:223:dfff:fedf:30ce%7 + boolean isLoopbackAddress = a.isLoopbackAddress(); // true i.e. for localhost/0:0:0:0:0:0:0:1, localhost/127.0.0.1 + boolean isSiteLocalAddress = a.isSiteLocalAddress(); // true i.e. for myhost.local/192.168.1.33 + if (isAnyLocalAddress || isLinkLocalAddress || isLoopbackAddress || isSiteLocalAddress) { + ConcurrentLog.info("Domain Init", "local host address: " + a + " (local)"); + localHostAddresses.add(a); + if (hostname != null) {localHostNames.add(hostname); localHostNames.add(a.getHostAddress());} + } else { + ConcurrentLog.info("Domain Init", "local host address: " + a + " (public)"); + if (a instanceof Inet4Address) { + publicIPv4HostAddresses.add(a); + if (hostname != null) {publicIPv4HostNames.add(hostname); publicIPv4HostNames.add(a.getHostAddress());} + } else { + publicIPv6HostAddresses.add(a); + if (hostname != null) {publicIPv6HostNames.add(hostname); publicIPv6HostNames.add(a.getHostAddress());} + } + } + } + } + }.start(); + } + /** * ! ! ! A T T E N T I O N A T T E N T I O N A T T E N T I O N ! ! ! * @@ -810,7 +901,7 @@ public class Domains { // add also the isLocal host name caches final boolean localp = ip.isAnyLocalAddress() || ip.isLinkLocalAddress() || ip.isSiteLocalAddress(); if (localp) { - localHostNames.add(host); + myHostNames.add(host); } else { if (globalHosts != null) try { globalHosts.add(host); @@ -847,131 +938,54 @@ public class Domains { return nameCacheNoCachingPatterns.size(); } - private static Set<InetAddress> localHostAddresses = new HashSet<InetAddress>(); - private static Set<String> localHostNames = new HashSet<String>(); - static { - try { - final InetAddress localHostAddress = InetAddress.getLocalHost(); - if (localHostAddress != null) localHostAddresses.add(localHostAddress); - } catch (final UnknownHostException e) {} - try { - final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME); - if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses)); - } catch (final UnknownHostException e) {} - - // to get the local host name, a dns lookup is necessary. - // if such a lookup blocks, it can cause that the static initiatializer does not finish fast - // therefore we start the host name lookup as concurrent thread - // meanwhile the host name is "127.0.0.1" which is not completely wrong - new Thread() { - @Override - public void run() { - Thread.currentThread().setName("Domains: init"); - // try to get local addresses from interfaces - try { - final Enumeration<NetworkInterface> nis = NetworkInterface.getNetworkInterfaces(); - while (nis.hasMoreElements()) { - final NetworkInterface ni = nis.nextElement(); - final Enumeration<InetAddress> addrs = ni.getInetAddresses(); - while (addrs.hasMoreElements()) { - final InetAddress addr = addrs.nextElement(); - if (addr != null) localHostAddresses.add(addr); - } - } - } catch (final SocketException e) { - } - - // now look up the host name - try { - LOCALHOST_NAME = getHostName(InetAddress.getLocalHost()); - } catch (final UnknownHostException e) {} - - // after the host name was resolved, we try to look up more local addresses - // using the host name: - try { - final InetAddress[] moreAddresses = InetAddress.getAllByName(LOCALHOST_NAME); - if (moreAddresses != null) localHostAddresses.addAll(Arrays.asList(moreAddresses)); - } catch (final UnknownHostException e) { - } - - // fill a cache of local host names - for (final InetAddress a: localHostAddresses) { - final String hostname = getHostName(a); - if (hostname != null) { - localHostNames.add(hostname); - localHostNames.add(a.getHostAddress()); - } - } - } - }.start(); + /** + * myPublicLocalIP() returns the IP of this host which is reachable in the public network under this address + * This is deprecated since it should be possible that the host is reachable with more than one IP + * That is particularly the case if the host supports IPv4 and IPv6. + * Please use myPublicIPv4() or (preferred) myPublicIPv6() instead. + * @return + */ + @Deprecated + public static InetAddress myPublicLocalIP() { + // for backward compatibility, we try to select a IPv4 address here. + // future methods should use myPublicIPs() and prefer IPv6 + if (publicIPv4HostAddresses.size() > 0) return publicIPv4HostAddresses.iterator().next(); + if (publicIPv6HostAddresses.size() > 0) return publicIPv6HostAddresses.iterator().next(); + return null; } - public static InetAddress myPublicLocalIP() { - // list all addresses - // for (int i = 0; i < localHostAddresses.length; i++) System.out.println("IP: " + localHostAddresses[i].getHostAddress()); // DEBUG - if (localHostAddresses.isEmpty()) { - return null; - } - if (localHostAddresses.size() == 1) { - // only one network connection available - return localHostAddresses.iterator().next(); - } - // we have more addresses, find an address that is not local - int b0, b1; - for (final InetAddress a: localHostAddresses) { - b0 = 0Xff & a.getAddress()[0]; - b1 = 0Xff & a.getAddress()[1]; - if (b0 != 10 && // class A reserved - b0 != 127 && // loopback - (b0 != 172 || b1 < 16 || b1 > 31) && // class B reserved - (b0 != 192 || b1 != 168) && // class C reserved - (a.getHostAddress().indexOf(':',0) < 0)) - return a; - } - // there is only a local address - // return that one that is returned with InetAddress.getLocalHost() - // if appropriate - try { - final InetAddress localHostAddress = InetAddress.getLocalHost(); - if (localHostAddress != null && - (0Xff & localHostAddress.getAddress()[0]) != 127 && - localHostAddress.getHostAddress().indexOf(':',0) < 0) return localHostAddress; - } catch (final UnknownHostException e) { - } - // we filter out the loopback address 127.0.0.1 and all addresses without a name - for (final InetAddress a: localHostAddresses) { - if ((0Xff & a.getAddress()[0]) != 127 && - a.getHostAddress().indexOf(':',0) < 0 && - a.getHostName() != null && - !a.getHostName().isEmpty()) return a; - } - // if no address has a name, then take any other than the loopback - for (final InetAddress a: localHostAddresses) { - if ((0Xff & a.getAddress()[0]) != 127 && - a.getHostAddress().indexOf(':',0) < 0) return a; - } - // if all fails, give back whatever we have - for (final InetAddress a: localHostAddresses) { - if (a.getHostAddress().indexOf(':',0) < 0) return a; - } - // finally, just get any - return localHostAddresses.iterator().next(); + public static Set<String> myPublicIPs() { + Set<String> h = new HashSet<>(publicIPv4HostAddresses.size() + publicIPv6HostAddresses.size()); + for (InetAddress i: publicIPv4HostAddresses) h.add(i.getHostAddress()); + for (InetAddress i: publicIPv6HostAddresses) h.add(i.getHostAddress()); + return h; + } + + /** + * Get all IPv4 addresses which are assigned to the local host but are public IP addresses. + * These should be the possible addresses which can be used to access this peer. + * @return the public IPv4 Addresses of this peer + */ + public static Set<InetAddress> myPublicIPv4() { + return publicIPv4HostAddresses; + } + + /** + * Get all IPv6 addresses which are assigned to the local host but are public IP addresses. + * These should be the possible addresses which can be used to access this peer. + * @return the public IPv6 addresses of this peer + */ + public static Set<InetAddress> myPublicIPv6() { + return publicIPv6HostAddresses; } /** - * generate a list of intranet InetAddresses without the loopback address 127.0.0.1 + * generate a list of intranet InetAddresses * @return list of all intranet addresses */ public static Set<InetAddress> myIntranetIPs() { - // list all local addresses - if (localHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (final InterruptedException e) {} - final Set<InetAddress> list = new HashSet<InetAddress>(); - if (localHostAddresses.isEmpty()) return list; // give up - for (final InetAddress a: localHostAddresses) { - if ((0Xff & a.getAddress()[0]) == 127) continue; - list.add(a); - } - return list; + if (myHostAddresses.size() < 1) try {Thread.sleep(1000);} catch (final InterruptedException e) {} + return localHostAddresses; } public static boolean isThisHostIP(final String hostName) { @@ -982,7 +996,7 @@ public class Domains { final InetAddress clientAddress = Domains.dnsResolve(hostName); if (clientAddress == null) return false; if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true; - for (final InetAddress a: localHostAddresses) { + for (final InetAddress a: myHostAddresses) { if (a.equals(clientAddress)) { isThisHostIP = true; break; @@ -999,7 +1013,7 @@ public class Domains { try { if (clientAddress.isAnyLocalAddress() || clientAddress.isLoopbackAddress()) return true; - for (final InetAddress a: localHostAddresses) { + for (final InetAddress a: myHostAddresses) { if (a.equals(clientAddress)) { isThisHostIP = true; break; @@ -1024,7 +1038,9 @@ public class Domains { * @return true if the host from the string is the localhost */ public static boolean isLocalhost(final String host) { - return (host != null && LOCALHOST_PATTERNS.matcher(host).matches()); + return host == null || // filesystems do not have host names + LOCALHOST_PATTERNS.matcher(host).matches() || + localHostNames.contains(host); } /** @@ -1038,7 +1054,9 @@ public class Domains { */ public static boolean isIntranet(final String host) { return (noLocalCheck || // DO NOT REMOVE THIS! it is correct to return true if the check is off - (host != null && INTRANET_PATTERNS.matcher(host).matches())); + host == null || // filesystems do not have host names + INTRANET_PATTERNS.matcher(host).matches()) || + localHostNames.contains(host); } /** @@ -1060,10 +1078,7 @@ public class Domains { // check local ip addresses if (isIntranet(host)) return true; - if (hostaddress != null && ( - isIntranet(hostaddress.getHostAddress()) || - isLocal(hostaddress) - )) return true; + if (hostaddress != null && (isIntranet(hostaddress.getHostAddress()) || isLocal(hostaddress))) return true; // check if there are other local IP addresses that are not in // the standard IP range diff --git a/source/net/yacy/cora/protocol/RequestHeader.java b/source/net/yacy/cora/protocol/RequestHeader.java index 8ec7be751..56404cf79 100644 --- a/source/net/yacy/cora/protocol/RequestHeader.java +++ b/source/net/yacy/cora/protocol/RequestHeader.java @@ -134,6 +134,7 @@ public class RequestHeader extends HeaderFramework { return false; } final String refererHost = this.refererHost(); - return refererHost == null || refererHost.isEmpty() || Domains.isLocalhost(refererHost); + if (refererHost == null || refererHost.isEmpty() || Domains.isLocalhost(refererHost)) return true; + return false; } } diff --git a/source/net/yacy/cora/protocol/http/HTTPClient.java b/source/net/yacy/cora/protocol/http/HTTPClient.java index a45c05844..4082cb486 100644 --- a/source/net/yacy/cora/protocol/http/HTTPClient.java +++ b/source/net/yacy/cora/protocol/http/HTTPClient.java @@ -37,6 +37,11 @@ import java.security.cert.X509Certificate; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLContext; @@ -58,6 +63,7 @@ import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.ClientProtocolException; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.CookieSpecs; import org.apache.http.client.config.RequestConfig; @@ -77,7 +83,6 @@ import org.apache.http.conn.routing.HttpRoute; import org.apache.http.conn.socket.ConnectionSocketFactory; import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.entity.InputStreamEntity; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.ContentBody; import org.apache.http.impl.client.BasicCredentialsProvider; @@ -112,7 +117,7 @@ public class HTTPClient { private long upbytes = 0L; private String host = null; private final long timeout; - + private static ExecutorService executor = Executors.newFixedThreadPool(200); public HTTPClient(final ClientIdentification.Agent agent) { super(); @@ -236,23 +241,6 @@ public class HTTPClient { } } -// public static void setAuth(final String host, final int port, final String user, final String pw) { -// final UsernamePasswordCredentials creds = new UsernamePasswordCredentials(user, pw); -// final AuthScope scope = new AuthScope(host, port); -// credsProvider.setCredentials(scope, creds); -// httpClient.getParams().setParameter(ClientContext.CREDS_PROVIDER, credsProvider); -// } - -// /** -// * this method sets a host on which more than the default of 2 router per host are allowed -// * -// * @param the host to be raised in 'route per host' -// */ -// public static void setMaxRouteHost(final String host) { -// final HttpHost mHost = new HttpHost(host); -// ((PoolingClientConnectionManager) httpClient.getConnectionManager()).setMaxPerRoute(new HttpRoute(mHost), 50); -// } - /** * This method sets the Header used for the request * @@ -454,10 +442,12 @@ public class HTTPClient { * @param length the contentlength * @throws IOException */ + /* public void POST(final String uri, final InputStream instream, final long length, final boolean concurrent) throws IOException { POST(new MultiProtocolURL(uri), instream, length, concurrent); } - + */ + /** * This method POSTs a page from the server. * to be used for streaming out @@ -490,11 +480,13 @@ public class HTTPClient { * @return content bytes * @throws IOException */ + /* public byte[] POSTbytes(final String uri, final Map<String, ContentBody> parts, final boolean usegzip, final boolean concurrent) throws IOException { final MultiProtocolURL url = new MultiProtocolURL(uri); return POSTbytes(url, url.getHost(), parts, usegzip, concurrent); } - + */ + /** * send data to the server named by vhost * @@ -512,8 +504,7 @@ public class HTTPClient { if (vhost == null) setHost(Domains.LOCALHOST); final MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); - for (final Entry<String,ContentBody> part : post.entrySet()) - entityBuilder.addPart(part.getKey(), part.getValue()); + for (final Entry<String,ContentBody> part : post.entrySet()) entityBuilder.addPart(part.getKey(), part.getValue()); final HttpEntity multipartEntity = entityBuilder.build(); // statistics this.upbytes = multipartEntity.getContentLength(); @@ -536,6 +527,7 @@ public class HTTPClient { * @return content bytes * @throws IOException */ + /* public byte[] POSTbytes(final String uri, final InputStream instream, final long length, final boolean concurrent) throws IOException { final MultiProtocolURL url = new MultiProtocolURL(uri); final HttpPost httpPost = new HttpPost(url.toNormalform(true)); @@ -549,8 +541,9 @@ public class HTTPClient { httpPost.setEntity(inputStreamEntity); return getContentBytes(httpPost, Integer.MAX_VALUE, concurrent); } + */ - /** + /** * * @return HttpResponse from call */ @@ -682,30 +675,26 @@ public class HTTPClient { assert !hrequest.expectContinue(); } - Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI().getHost()); + Thread.currentThread().setName("HTTPClient-" + httpUriRequest.getURI()); final long time = System.currentTimeMillis(); try { final CloseableHttpClient client = clientBuilder.build(); if (concurrent) { - final CloseableHttpResponse[] thr = new CloseableHttpResponse[]{null}; - final Throwable[] te = new Throwable[]{null}; - Thread t = new Thread() { + FutureTask<CloseableHttpResponse> t = new FutureTask<CloseableHttpResponse>(new Callable<CloseableHttpResponse>() { @Override - public void run() { - this.setName("HTTPClient.execute(" + httpUriRequest.getURI() + ")"); - try { - thr[0] = client.execute(httpUriRequest, context); - } catch (Throwable e) { - te[0] = e; - } + public CloseableHttpResponse call() throws ClientProtocolException, IOException { + CloseableHttpResponse response = client.execute(httpUriRequest, context); + return response; } - }; - t.start(); - try {t.join(this.timeout);} catch (InterruptedException e) {} - if (t.isAlive()) try {t.interrupt();} catch (Throwable e) {} - if (te[0] != null) throw te[0]; - if (thr[0] == null) throw new IOException("timout to client after " + this.timeout + "ms"); - this.httpResponse = thr[0]; + }); + executor.execute(t); + try { + this.httpResponse = t.get(this.timeout, TimeUnit.MILLISECONDS); + } catch (ExecutionException e) { + throw e.getCause(); + } catch (Throwable e) {} + try {t.cancel(true);} catch (Throwable e) {} + if (this.httpResponse == null) throw new IOException("timout to client after " + this.timeout + "ms"); } else { this.httpResponse = client.execute(httpUriRequest, context); } diff --git a/source/net/yacy/cora/util/CommonPattern.java b/source/net/yacy/cora/util/CommonPattern.java index 1b1894e87..93f7a1f99 100644 --- a/source/net/yacy/cora/util/CommonPattern.java +++ b/source/net/yacy/cora/util/CommonPattern.java @@ -43,6 +43,7 @@ public class CommonPattern { public final static Pattern SEMICOLON = Pattern.compile(";"); public final static Pattern DOUBLEPOINT = Pattern.compile(":"); public final static Pattern SLASH = Pattern.compile("/"); + public final static Pattern PIPE = Pattern.compile("\\|"); public final static Pattern BACKSLASH = Pattern.compile("\\\\"); public final static Pattern QUESTION = Pattern.compile("\\?"); public final static Pattern AMP = Pattern.compile("&"); diff --git a/source/net/yacy/crawler/CrawlStacker.java b/source/net/yacy/crawler/CrawlStacker.java index a2bb4ab80..4c11e08d4 100644 --- a/source/net/yacy/crawler/CrawlStacker.java +++ b/source/net/yacy/crawler/CrawlStacker.java @@ -36,7 +36,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.atomic.AtomicInteger;
import net.yacy.contentcontrol.ContentControlFilterUpdateThread;
-import net.yacy.cora.document.analysis.Classification.ContentDomain;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.AnchorURL;
diff --git a/source/net/yacy/http/YacyDomainHandler.java b/source/net/yacy/http/YacyDomainHandler.java index 7041b5c35..5df7f3b13 100644 --- a/source/net/yacy/http/YacyDomainHandler.java +++ b/source/net/yacy/http/YacyDomainHandler.java @@ -82,7 +82,7 @@ public class YacyDomainHandler extends AbstractHandler implements Handler { newHost = hostPort; newPort = 80; } - if (newHost.equals(alternativeResolvers.myIP())) return; + if (alternativeResolvers.myIPs().contains(newHost)) return; if (Domains.isLocal(newHost, null)) return; RequestDispatcher dispatcher = request.getRequestDispatcher(path + target); dispatcher.forward(new DomainRequestWrapper(request, newHost, newPort), response); diff --git a/source/net/yacy/migration.java b/source/net/yacy/migration.java index 630c22692..8d83e428b 100644 --- a/source/net/yacy/migration.java +++ b/source/net/yacy/migration.java @@ -37,7 +37,6 @@ import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import com.google.common.io.Files; -import java.util.concurrent.ExecutionException; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.protocol.TimeoutRequest; import net.yacy.cora.storage.Configuration.Entry; diff --git a/source/net/yacy/peers/DHTSelection.java b/source/net/yacy/peers/DHTSelection.java index 689d2e755..e40be93c4 100644 --- a/source/net/yacy/peers/DHTSelection.java +++ b/source/net/yacy/peers/DHTSelection.java @@ -31,10 +31,9 @@ import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Random; import java.util.Set; -import java.util.SortedMap; +import java.util.SortedSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -55,18 +54,12 @@ import net.yacy.peers.operation.yacyVersion; */ public class DHTSelection { - public static Set<Seed> selectClusterPeers(final SeedDB seedDB, final SortedMap<byte[], String> peerhashes) { - final Iterator<Map.Entry<byte[], String>> i = peerhashes.entrySet().iterator(); + public static Set<Seed> selectClusterPeers(final SeedDB seedDB, final SortedSet<byte[]> peerhashes) { final Set<Seed> l = new HashSet<Seed>(); - Map.Entry<byte[], String> entry; Seed s; - while (i.hasNext()) { - entry = i.next(); - s = seedDB.get(ASCII.String(entry.getKey())); // should be getConnected; get only during testing time - if (s != null) { - s.setAlternativeAddress(entry.getValue()); - l.add(s); - } + for (byte[] hashb: peerhashes) { + s = seedDB.get(ASCII.String(hashb)); // should be getConnected; get only during testing time + if (s != null) l.add(s); } return l; } diff --git a/source/net/yacy/peers/Network.java b/source/net/yacy/peers/Network.java index 97dfa6c43..e6197b686 100644 --- a/source/net/yacy/peers/Network.java +++ b/source/net/yacy/peers/Network.java @@ -46,6 +46,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Semaphore; @@ -101,7 +102,7 @@ public class Network // ensure that correct IP is used final String staticIP = sb.getConfig("staticIP", ""); - if ( staticIP.length() != 0 && Seed.isProperIP(staticIP) == null ) { + if (staticIP.length() != 0 && Seed.isProperIP(staticIP)) { serverCore.useStaticIP = true; sb.peers.mySeed().setIP(staticIP); log.info("staticIP set to " + staticIP); @@ -118,9 +119,7 @@ public class Network } public final void publishSeedList() { - if ( log.isFine() ) { - log.fine("yacyCore.publishSeedList: Triggered Seed Publish"); - } + if (log.isFine()) log.fine("yacyCore.publishSeedList: Triggered Seed Publish"); /* if (oldIPStamp.equals((String) seedDB.mySeed.get(yacySeed.IP, "127.0.0.1"))) @@ -131,14 +130,11 @@ public class Network yacyCore.log.logDebug("***DEBUG publishSeedList: I can reach myself"); */ - if ( (this.sb.peers.lastSeedUpload_myIP.equals(this.sb.peers.mySeed().getIP())) + if ((this.sb.peers.mySeed().getIPs().contains(this.sb.peers.lastSeedUpload_myIP)) && (this.sb.peers.lastSeedUpload_seedDBSize == this.sb.peers.sizeConnected()) && (System.currentTimeMillis() - this.sb.peers.lastSeedUpload_timeStamp < 1000 * 60 * 60 * 24) && (this.sb.peers.mySeed().isPrincipal()) ) { - if ( log.isFine() ) { - log - .fine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP."); - } + if (log.isFine()) log.fine("yacyCore.publishSeedList: not necessary to publish: oldIP is equal, sizeConnected is equal and I can reach myself under the old IP."); return; } @@ -162,9 +158,7 @@ public class Network if ( seedUploadMethod.equals("") ) { this.sb.setConfig("seedUploadMethod", "none"); } - if ( log.isFine() ) { - log.fine("yacyCore.publishSeedList: No uploading method configured"); - } + if (log.isFine()) log.fine("yacyCore.publishSeedList: No uploading method configured"); return; } } @@ -231,13 +225,7 @@ public class Network @Override public final void run() { try { - this.added = - Protocol.hello( - Network.this.sb.peers.mySeed(), - Network.this.sb.peers.peerActions, - this.seed.getClusterAddress(), - this.seed.hash, - this.seed.getName()); + this.added = Protocol.hello(Network.this.sb.peers.mySeed(), Network.this.sb.peers.peerActions, this.seed); if ( this.added < 0 ) { // no or wrong response, delete that address final String cause = "peer ping to peer resulted in error response (added < 0)"; @@ -246,7 +234,7 @@ public class Network + " peer '" + this.seed.getName() + "' from " - + this.seed.getPublicAddress() + + this.seed.getIPs() + ": " + cause); Network.this.sb.peers.peerActions.peerDeparture(this.seed, cause); @@ -258,7 +246,7 @@ public class Network + " peer '" + this.seed.getName() + "' at " - + this.seed.getPublicAddress()); + + this.seed.getIPs()); // check if seed's lastSeen has been updated final Seed newSeed = Network.this.sb.peers.getConnected(this.seed.hash); if ( newSeed != null ) { @@ -269,7 +257,7 @@ public class Network + " peer '" + this.seed.getName() + "' at " - + this.seed.getPublicAddress() + + this.seed.getIPs() + " is not online." + " Removing Peer from connected"); } @@ -284,7 +272,7 @@ public class Network + " peer '" + this.seed.getName() + "' at " - + this.seed.getPublicAddress() + + this.seed.getIPs() + " with old LastSeen: '" + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed .getLastSeenUTC())) + "'"); @@ -299,7 +287,7 @@ public class Network + " peer '" + this.seed.getName() + "' at " - + this.seed.getPublicAddress() + + this.seed.getIPs() + " with old LastSeen: '" + my_SHORT_SECOND_FORMATTER.format(new Date(newSeed .getLastSeenUTC())) @@ -319,7 +307,7 @@ public class Network + " peer '" + this.seed.getName() + "' at " - + this.seed.getPublicAddress() + + this.seed.getIPs() + " not in connectedDB"); } } @@ -364,25 +352,21 @@ public class Network if ( attempts > PING_INITIAL ) { attempts = PING_INITIAL; } - final Map<byte[], String> ch = Switchboard.getSwitchboard().clusterhashes; + final Set<byte[]> ch = Switchboard.getSwitchboard().clusterhashes; seeds = DHTSelection.seedsByAge(this.sb.peers, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection // add also all peers from cluster if this is a public robinson cluster if ( ch != null ) { - final Iterator<Map.Entry<byte[], String>> i = ch.entrySet().iterator(); String hash; - Map.Entry<byte[], String> entry; Seed seed; - while ( i.hasNext() ) { - entry = i.next(); - hash = ASCII.String(entry.getKey()); + for (byte[] hashb: ch) { + hash = ASCII.String(hashb); seed = seeds.get(hash); - if ( seed == null ) { + if (seed == null) { seed = this.sb.peers.get(hash); if ( seed == null ) { continue; } } - seed.setAlternativeAddress(entry.getValue()); seeds.put(hash, seed); } } @@ -441,17 +425,14 @@ public class Network } i++; - final String address = seed.getClusterAddress(); + final String address = seed.getPublicAddress(seed.getIP()); if ( log.isFine() ) { log.fine("HELLO #" + i + " to peer '" + seed.get(Seed.NAME, "") + "' at " + address); // debug } final String seederror = seed.isProper(false); if ( (address == null) || (seederror != null) ) { // we don't like that address, delete it - this.sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " - + address - + "; seederror = " - + seederror); + this.sb.peers.peerActions.peerDeparture(seed, "peer ping to peer resulted in address = " + address + "; seederror = " + seederror); sync.acquire(); } else { // starting a new publisher thread @@ -567,16 +548,13 @@ public class Network //if (ip.equals("")) ip = natLib.retrieveIP(DI604use, DI604pw); // yacyCore.log.logDebug("DEBUG: new IP=" + ip); - if ( Seed.isProperIP(ip) == null ) { + if (Seed.isProperIP(ip)) { this.sb.peers.mySeed().setIP(ip); } if ( this.sb.peers.mySeed().get(Seed.PEERTYPE, Seed.PEERTYPE_JUNIOR).equals(Seed.PEERTYPE_JUNIOR) ) { this.sb.peers.mySeed().put(Seed.PEERTYPE, Seed.PEERTYPE_SENIOR); // to start bootstraping, we need to be recognised as PEERTYPE_SENIOR peer } - log.info("publish: no recipient found, our address is " - + ((this.sb.peers.mySeed().getPublicAddress() == null) ? "unknown" : this.sb.peers - .mySeed() - .getPublicAddress())); + log.info("publish: no recipient found, our address is " + this.sb.peers.mySeed().getIPs()); this.sb.peers.saveMySeed(); return 0; } catch (final InterruptedException e ) { diff --git a/source/net/yacy/peers/PeerActions.java b/source/net/yacy/peers/PeerActions.java index 845b10946..c79f6ab82 100644 --- a/source/net/yacy/peers/PeerActions.java +++ b/source/net/yacy/peers/PeerActions.java @@ -28,6 +28,7 @@ import java.util.Map; import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.feed.RSSMessage;
+import net.yacy.cora.protocol.Domains;
import net.yacy.cora.storage.ConcurrentARC;
import net.yacy.kelondro.util.MapTools;
import net.yacy.peers.operation.yacyVersion;
@@ -66,7 +67,7 @@ public class PeerActions { return false;
}
if ((this.seedDB.mySeedIsDefined()) && (seed.hash.equals(this.seedDB.mySeed().hash))) {
- Network.log.info("connect: SELF reference " + seed.getPublicAddress());
+ Network.log.info("connect: SELF reference " + seed.getIPs());
return false;
}
final String peerType = seed.get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN);
@@ -82,7 +83,7 @@ public class PeerActions { return false;
}
- final Seed doubleSeed = this.seedDB.lookupByIP(seed.getInetAddress(), seed.getPort(), true, false, false);
+ final Seed doubleSeed = this.seedDB.lookupByIP(Domains.dnsResolve(seed.getIP()), seed.getPort(), true, false, false);
if ((doubleSeed != null) && (doubleSeed.getPort() == seed.getPort()) && (!(doubleSeed.hash.equals(seed.hash)))) {
// a user frauds with his peer different peer hashes
if (Network.log.isFine()) Network.log.fine("connect: rejecting FRAUD (double hashes " + doubleSeed.hash + "/" + seed.hash + " on same port " + seed.getPort() + ") peer " + seed.getName());
@@ -107,7 +108,7 @@ public class PeerActions { }
if (Math.abs(nowUTC0Time - ctimeUTC0) / 1000 / 60 > 60 * 6 ) {
// the new connection is out-of-age, we reject the connection
- if (Network.log.isFine()) Network.log.fine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getPublicAddress() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + formatInterval(Math.abs(nowUTC0Time - ctimeUTC0)));
+ if (Network.log.isFine()) Network.log.fine("connect: rejecting out-dated peer '" + seed.getName() + "' from " + seed.getIPs() + "; nowUTC0=" + nowUTC0Time + ", seedUTC0=" + ctimeUTC0 + ", TimeDiff=" + formatInterval(Math.abs(nowUTC0Time - ctimeUTC0)));
return false;
}
@@ -144,13 +145,13 @@ public class PeerActions { if (!direct) {
if (ctimeUTC0 < dtimeUTC0) {
// the disconnection was later, we reject the connection
- if (Network.log.isFine()) Network.log.fine("connect: rejecting disconnected peer '" + seed.getName() + "' from " + seed.getPublicAddress());
+ if (Network.log.isFine()) Network.log.fine("connect: rejecting disconnected peer '" + seed.getName() + "' from " + seed.getIPs());
return false;
}
}
// this is a return of a lost peer
- if (Network.log.isFine()) Network.log.fine("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress());
+ if (Network.log.isFine()) Network.log.fine("connect: returned KNOWN " + peerType + " peer '" + seed.getName() + "' from " + seed.getIPs());
this.seedDB.addConnected(seed);
return true;
}
@@ -169,10 +170,10 @@ public class PeerActions { // TODO: update seed name lookup cache
}*/
} catch (final NumberFormatException e) {
- if (Network.log.isFine()) Network.log.fine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getPublicAddress() + ". Cause: " + e.getMessage());
+ if (Network.log.isFine()) Network.log.fine("connect: rejecting wrong peer '" + seed.getName() + "' from " + seed.getIPs() + ". Cause: " + e.getMessage());
return false;
}
- if (Network.log.isFine()) Network.log.fine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress());
+ if (Network.log.isFine()) Network.log.fine("connect: updated KNOWN " + ((direct) ? "direct " : "") + peerType + " peer '" + seed.getName() + "' from " + seed.getIPs());
this.seedDB.addConnected(seed);
return true;
}
@@ -181,10 +182,10 @@ public class PeerActions { if ((this.seedDB.mySeedIsDefined()) && (seed.getIP().equals(this.seedDB.mySeed().getIP()))) {
// seed from the same IP as the calling client: can be
// the case if there runs another one over a NAT
- if (Network.log.isFine()) Network.log.fine("connect: saved NEW seed (myself IP) " + seed.getPublicAddress());
+ if (Network.log.isFine()) Network.log.fine("connect: saved NEW seed (myself IP) " + seed.getIPs());
} else {
// completely new seed
- if (Network.log.isFine()) Network.log.fine("connect: saved NEW " + peerType + " peer '" + seed.getName() + "' from " + seed.getPublicAddress());
+ if (Network.log.isFine()) Network.log.fine("connect: saved NEW " + peerType + " peer '" + seed.getName() + "' from " + seed.getIPs());
}
this.seedDB.addConnected(seed);
return true;
@@ -204,7 +205,7 @@ public class PeerActions { public void peerDeparture(final Seed peer, final String cause) {
if (peer == null) return;
// we do this if we did not get contact with the other peer
- if (Network.log.isFine()) Network.log.fine("connect: no contact to a " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN) + " peer '" + peer.getName() + "' at " + peer.getPublicAddress() + ". Cause: " + cause);
+ if (Network.log.isFine()) Network.log.fine("connect: no contact to a " + peer.get(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN) + " peer '" + peer.getName() + "' at " + peer.getIPs() + ". Cause: " + cause);
synchronized (this.seedDB) {
if (!this.seedDB.hasDisconnected(ASCII.getBytes(peer.hash))) { this.disconnects++; }
peer.put(Seed.DCT, Long.toString(System.currentTimeMillis()));
diff --git a/source/net/yacy/peers/Protocol.java b/source/net/yacy/peers/Protocol.java index 0692af33c..e2a79fa6a 100644 --- a/source/net/yacy/peers/Protocol.java +++ b/source/net/yacy/peers/Protocol.java @@ -51,6 +51,7 @@ import java.net.InetAddress; import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
@@ -128,14 +129,7 @@ import org.apache.solr.common.SolrInputDocument; public final class Protocol {
- private static byte[] postToFile(
- final Seed target,
- final String filename,
- final Map<String, ContentBody> parts,
- final int timeout) throws IOException {
- return postToFile(target.getClusterAddress(), target.hash, filename, parts, timeout);
- }
-
+ @Deprecated
private static byte[] postToFile(
final SeedDB seedDB,
final String targetHash,
@@ -176,9 +170,8 @@ public final class Protocol { public static int hello(
final Seed mySeed,
final PeerActions peerActions,
- final String address,
- final String otherHash,
- final String otherName) {
+ final Seed otherSeed) {
+ final String address = otherSeed.getPublicAddress(otherSeed.getIP());
Map<String, String> result = null;
final String salt = crypt.randomSalt();
@@ -198,7 +191,7 @@ public final class Protocol { content =
httpClient.POSTbytes(
new MultiProtocolURL("http://" + address + "/yacy/hello.html"),
- Seed.b64Hash2hexHash(otherHash) + ".yacyh",
+ Seed.b64Hash2hexHash(otherSeed.hash) + ".yacyh",
parts,
false, true);
responseTime = System.currentTimeMillis() - start;
@@ -210,12 +203,7 @@ public final class Protocol { + "' interrupted.");
return -1;
}
- Network.log.info("yacyClient.hello thread '"
- + Thread.currentThread().getName()
- + "', peer "
- + address
- + "; exception: "
- + e.getMessage());
+ Network.log.info("yacyClient.hello thread '" + Thread.currentThread().getName() + "', peer " + address + "; exception: " + e.getMessage());
// try again (go into loop)
result = null;
}
@@ -238,35 +226,25 @@ public final class Protocol { // check consistency with expectation
Seed otherPeer = null;
String seed;
- if ( (otherHash != null) && (otherHash.length() > 0) && ((seed = result.get("seed0")) != null) ) {
+ if ( (otherSeed.hash != null) && (otherSeed.hash.length() > 0) && ((seed = result.get("seed0")) != null) ) {
if ( seed.length() > Seed.maxsize ) {
- Network.log.info("hello/client 0: rejected contacting seed; too large ("
- + seed.length()
- + " > "
- + Seed.maxsize
- + ")");
+ Network.log.info("hello/client 0: rejected contacting seed; too large (" + seed.length() + " > " + Seed.maxsize + ")");
} else {
try {
- final int p = address.indexOf(':');
- if ( p < 0 ) {
- return -1;
- }
+ // patch the remote peer address to avoid that remote peers spoof the network with wrong addresses
+ final int p = address.lastIndexOf(':');
+ if ( p < 0 ) return -1;
String h = address.substring(0, p);
+ if (h.charAt(0) == '[') h = h.substring(1);
+ if (h.charAt(h.length() - 1) == ']') h = h.substring(0, h.length() - 1);
InetAddress ie = Domains.dnsResolve(h);
- final String host = ie == null ? h : ie.getHostAddress(); // hack to prevent NPEs
- otherPeer = Seed.genRemoteSeed(seed, false, host);
- if ( !otherPeer.hash.equals(otherHash) ) {
- Network.log.info("yacyClient.hello: consistency error: otherPeer.hash = "
- + otherPeer.hash
- + ", otherHash = "
- + otherHash);
+ otherPeer = Seed.genRemoteSeed(seed, false, ie.getHostAddress());
+ if ( !otherPeer.hash.equals(otherSeed.hash) ) {
+ Network.log.info("yacyClient.hello: consistency error: otherPeer.hash = " + otherPeer.hash + ", otherHash = " + otherSeed.hash);
return -1; // no success
}
} catch (final IOException e ) {
- Network.log.info("yacyClient.hello: consistency error: other seed bad:"
- + e.getMessage()
- + ", seed="
- + seed);
+ Network.log.info("yacyClient.hello: consistency error: other seed bad:" + e.getMessage() + ", seed=" + seed);
return -1; // no success
}
}
@@ -281,11 +259,15 @@ public final class Protocol { // set my own seed according to new information
// we overwrite our own IP number only
if ( serverCore.useStaticIP ) {
- mySeed.setIP(Switchboard.getSwitchboard().myPublicIP());
+ mySeed.setIPs(Switchboard.getSwitchboard().myPublicIPs());
} else {
final String myIP = result.get("yourip");
- final String properIP = Seed.isProperIP(myIP);
- if ( properIP == null ) mySeed.setIP(myIP);
+ // with the IPv6 extension, this may contain several ips, separated by comma ','
+ HashSet<String> h = new HashSet<>();
+ for (String s: myIP.split(",")) {
+ if (s.length() > 0 && Seed.isProperIP(s)) h.add(s);
+ }
+ if (h.size() > 0) mySeed.setIPs(h);
}
mySeed.setFlagRootNode(
(mytype.equals(Seed.PEERTYPE_SENIOR) || mytype.equals(Seed.PEERTYPE_PRINCIPAL)) &&
@@ -303,7 +285,7 @@ public final class Protocol { accessible.IWasAccessed = false;
}
accessible.lastUpdated = System.currentTimeMillis();
- Network.amIAccessibleDB.put(otherHash, accessible);
+ Network.amIAccessibleDB.put(otherSeed.hash, accessible);
/*
* If we were reported as junior we have to check if your port forwarding channel is broken
@@ -387,52 +369,11 @@ public final class Protocol { final int connectedAfter = peerActions.sizeConnected();
// update event tracker
- EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(
- mySeed.getName(),
- otherName,
- true,
- connectedAfter - connectedBefore), false);
+ EventTracker.update(EventTracker.EClass.PEERPING, new ProfilingGraph.EventPing(mySeed.getName(), otherSeed.getName(), true, connectedAfter - connectedBefore), false);
return count;
}
- /*
- private int readSeeds(String prefix) {
- String seedStr;
- while ( (seedStr = result.get("seed" + i++)) != null ) {
- // integrate new seed into own database
- // the first seed, "seed0" is the seed of the responding peer
- if ( seedStr.length() > Seed.maxsize ) {
- Network.log.logInfo("hello/client: rejected contacting seed; too large ("
- + seedStr.length()
- + " > "
- + Seed.maxsize
- + ")");
- } else {
- try {
- if ( i == 1 ) {
- final int p = address.indexOf(':');
- if ( p < 0 ) {
- return -1;
- }
- InetAddress ia = Domains.dnsResolve(address.substring(0, p));
- if (ia == null) continue;
- final String host = ia.getHostAddress();
- s = Seed.genRemoteSeed(seedStr, false, host);
- } else {
- s = Seed.genRemoteSeed(seedStr, false, null);
- }
- if ( peerActions.peerArrival(s, (i == 1)) ) {
- count++;
- }
- } catch (final IOException e ) {
- Network.log.logInfo("hello/client: rejected contacting seed; bad ("
- + e.getMessage()
- + ")");
- }
- }
- }
- }
-*/
+
public static Seed querySeed(final Seed target, final String seedHash) {
// prepare request
final String salt = crypt.randomSalt();
@@ -443,7 +384,7 @@ public final class Protocol { basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
parts.put("object", UTF8.StringBody("seed"));
parts.put("env", UTF8.StringBody(seedHash));
- final byte[] content = postToFile(target, "query.html", parts, 10000);
+ final byte[] content = postToFile(target.getPublicAddress(target.getIP()), target.hash, "query.html", parts, 10000);
final Map<String, String> result = FileUtils.table(content);
if ( result == null || result.isEmpty() ) {
@@ -457,22 +398,24 @@ public final class Protocol { }
}
- public static long[] queryRWICount(final Seed target, final String wordHash) {
- if (target == null) return new long[] {-1, -1};
-
+ public static long[] queryRWICount(final String targetAddress, final String targetHash, int timeout) {
// prepare request
final String salt = crypt.randomSalt();
// send request
try {
- final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
+ final Map<String, ContentBody> parts = basicRequestParts(Switchboard.getSwitchboard(), targetHash, salt);
parts.put("object", UTF8.StringBody("rwicount"));
parts.put("ttl", UTF8.StringBody("0"));
- parts.put("env", UTF8.StringBody(wordHash));
- final byte[] content = postToFile(target, "query.html", parts, 6000);
+ parts.put("env", UTF8.StringBody(""));
+ ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "posting request to " + targetAddress);
+ final byte[] content = postToFile(targetAddress, targetHash, "query.html", parts, timeout);
+ ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received CONTENT from requesting " + targetAddress + (content == null ? "NULL" : (": length = " + content.length)));
final Map<String, String> result = FileUtils.table(content);
if (result == null || result.isEmpty()) return new long[] {-1, -1};
+ ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESULT from requesting " + targetAddress + " : result = " + result.toString());
final String resp = result.get("response");
+ ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received RESPONSE from requesting " + targetAddress + " : response = " + resp);
if (resp == null) return new long[] {-1, -1};
String magic = result.get("magic");
if (magic == null) magic = "0";
@@ -482,6 +425,7 @@ public final class Protocol { return new long[] {-1, -1};
}
} catch (final Exception e ) {
+ ConcurrentLog.info("**hello-DEBUG**queryRWICount**", "received EXCEPTION from requesting " + targetAddress + ": " + e.getMessage());
if (Network.log.isFine()) Network.log.fine("yacyClient.queryRWICount error:" + e.getMessage());
return new long[] {-1, -1};
}
@@ -516,10 +460,7 @@ public final class Protocol { parts.put("time", UTF8.StringBody(Long.toString(maxTime)));
// final byte[] result = HTTPConnector.getConnector(MultiProtocolURI.yacybotUserAgent).post(new MultiProtocolURI("http://" + target.getClusterAddress() + "/yacy/urls.xml"), (int) maxTime, target.getHexHash() + ".yacyh", parts);
final HTTPClient httpClient = new HTTPClient(ClientIdentification.yacyInternetCrawlerAgent, (int) maxTime);
- final byte[] result =
- httpClient.POSTbytes(new MultiProtocolURL("http://"
- + target.getClusterAddress()
- + "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);
+ final byte[] result = httpClient.POSTbytes(new MultiProtocolURL("http://" + target.getPublicAddress(target.getIP()) + "/yacy/urls.xml"), target.getHexHash() + ".yacyh", parts, false, true);
final RSSReader reader = RSSReader.parse(RSSFeed.DEFAULT_MAXSIZE, result);
if ( reader == null ) {
Network.log.warn("yacyClient.queryRemoteCrawlURLs failed asking peer '"
@@ -586,8 +527,8 @@ public final class Protocol { final long timestamp = System.currentTimeMillis();
event.addExpectedRemoteReferences(count);
SearchResult result;
- String clusteraddress = target.getClusterAddress();
- if (clusteraddress.equals(event.peers.mySeed().getClusterAddress())) clusteraddress = "localhost:" + event.peers.mySeed().getPort();
+ String clusteraddress = target.getPublicAddress(target.getIP());
+ if (target.clash(event.peers.mySeed().getIPs())) clusteraddress = "localhost:" + event.peers.mySeed().getPort();
try {
result =
new SearchResult(
@@ -680,7 +621,7 @@ public final class Protocol { maxDistance,
partitions,
target.getHexHash() + ".yacyh",
- target.getClusterAddress(),
+ target.getPublicAddress(),
null
);
} catch (final IOException e ) {
@@ -1033,7 +974,7 @@ public final class Protocol { Network.log.info("SEARCH skip (solr), remote Solr interface not accessible, peer=" + target.getName());
return -1;
}
- final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress();
+ final String address = myseed ? "localhost:" + target.getPort() : target.getPublicAddress(target.getIP());
final int solrtimeout = Switchboard.getSwitchboard().getConfigInt(SwitchboardConstants.FEDERATED_SERVICE_SOLR_INDEXING_TIMEOUT, 6000);
Thread remoteRequest = new Thread() {
@Override
@@ -1285,7 +1226,7 @@ public final class Protocol { final String salt = crypt.randomSalt();
// determining target address
- final String address = target.getClusterAddress();
+ final String address = target.getPublicAddress(target.getIP());
if ( address == null ) {
return null;
}
@@ -1435,7 +1376,7 @@ public final class Protocol { final ReferenceContainerCache<WordReference> indexes,
boolean gzipBody,
final int timeout) {
- final String address = targetSeed.getPublicAddress();
+ final String address = targetSeed.getPublicAddress(targetSeed.getIP());
if ( address == null ) {
Network.log.warn("no address for transferRWI");
return null;
@@ -1507,7 +1448,7 @@ public final class Protocol { boolean gzipBody,
final int timeout) {
// this post a message to the remote message board
- final String address = targetSeed.getPublicAddress();
+ final String address = targetSeed.getPublicAddress(targetSeed.getIP());
if ( address == null ) {
return null;
}
@@ -1577,7 +1518,7 @@ public final class Protocol { // this post a message to the remote message board
final String salt = crypt.randomSalt();
- String address = targetSeed.getClusterAddress();
+ String address = targetSeed.getPublicAddress(targetSeed.getIP());
if ( address == null ) {
address = "localhost:8090";
}
@@ -1619,7 +1560,7 @@ public final class Protocol { final Map<String, ContentBody> parts =
basicRequestParts(Switchboard.getSwitchboard(), target.hash, salt);
parts.put("object", UTF8.StringBody("host"));
- final byte[] content = postToFile(target, "idx.json", parts, 30000);
+ final byte[] content = postToFile(target.getPublicAddress(target.getIP()), target.hash, "idx.json", parts, 30000);
if ( content == null || content.length == 0 ) {
Network.log.warn("yacyClient.loadIDXHosts error: empty result");
return null;
@@ -1702,64 +1643,45 @@ public final class Protocol { return false;
}
- private static final LinkedHashMap<String, ContentBody> basicRequestParts(
- final Switchboard sb,
- final String targetHash,
- final String salt) {
- // put in all the essentials for routing and network authentication
- // generate a session key
- final LinkedHashMap<String, ContentBody> parts =
- basicRequestParts(
- sb.peers.mySeed().hash,
- targetHash,
- Switchboard.getSwitchboard().getConfig(
- SwitchboardConstants.NETWORK_NAME,
- Seed.DFLT_NETWORK_UNIT));
- parts.put("key", UTF8.StringBody(salt));
-
- // authentication essentials
- final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled");
- final String authenticationMethod =
- sb.getConfig("network.unit.protocol.request.authentication.method", "");
- if ( (authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0) ) {
- if ( authenticationMethod.equals("salted-magic-sim") ) {
- // generate an authentication essential using the salt, the iam-hash and the network magic
- final String magic =
- sb.getConfig("network.unit.protocol.request.authentication.essentials", "");
- final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic);
- parts.put("magicmd5", UTF8.StringBody(md5));
- }
- }
-
- return parts;
- }
-
- private static final LinkedHashMap<String, ContentBody> basicRequestParts(
- final String myHash,
- final String targetHash,
- final String networkName) {
- // put in all the essentials for routing and network authentication
- // generate a session key
+ /**
+ * put in all the essentials for routing and network authentication
+ * @param sb
+ * @param targetHash
+ * @param salt
+ * @return
+ */
+ private static final LinkedHashMap<String, ContentBody> basicRequestParts(final Switchboard sb, final String targetHash, final String salt) {
final LinkedHashMap<String, ContentBody> parts = new LinkedHashMap<String, ContentBody>();
-
+
// just standard identification essentials
- if ( myHash != null ) {
- parts.put("iam", UTF8.StringBody(myHash));
- if ( targetHash != null ) {
- parts.put("youare", UTF8.StringBody(targetHash));
- }
-
+ if ( sb.peers.mySeed().hash != null ) {
+ parts.put("iam", UTF8.StringBody(sb.peers.mySeed().hash));
+ if ( targetHash != null ) parts.put("youare", UTF8.StringBody(targetHash));
+
// time information for synchronization
// use our own formatter to prevent concurrency locks with other processes
- final GenericFormatter my_SHORT_SECOND_FORMATTER =
- new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
+ final GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second);
parts.put("mytime", UTF8.StringBody(my_SHORT_SECOND_FORMATTER.format()));
parts.put("myUTC", UTF8.StringBody(Long.toString(System.currentTimeMillis())));
-
+
// network identification
- parts.put(SwitchboardConstants.NETWORK_NAME, UTF8.StringBody(networkName));
+ parts.put(SwitchboardConstants.NETWORK_NAME, UTF8.StringBody(Switchboard.getSwitchboard().getConfig(
+ SwitchboardConstants.NETWORK_NAME,
+ Seed.DFLT_NETWORK_UNIT)));
}
+ parts.put("key", UTF8.StringBody(salt));
+ // authentication essentials
+ final String authenticationControl = sb.getConfig("network.unit.protocol.control", "uncontrolled");
+ final String authenticationMethod = sb.getConfig("network.unit.protocol.request.authentication.method", "");
+ if ((authenticationControl.equals("controlled")) && (authenticationMethod.length() > 0) ) {
+ if (authenticationMethod.equals("salted-magic-sim") ) {
+ // generate an authentication essential using the salt, the iam-hash and the network magic
+ final String magic = sb.getConfig("network.unit.protocol.request.authentication.essentials", "");
+ final String md5 = Digest.encodeMD5Hex(salt + sb.peers.mySeed().hash + magic);
+ parts.put("magicmd5", UTF8.StringBody(md5));
+ }
+ }
return parts;
}
diff --git a/source/net/yacy/peers/RemoteSearch.java b/source/net/yacy/peers/RemoteSearch.java index facd3a25b..f74de4609 100644 --- a/source/net/yacy/peers/RemoteSearch.java +++ b/source/net/yacy/peers/RemoteSearch.java @@ -30,7 +30,7 @@ import java.util.HashSet; import java.util.Iterator;
import java.util.Random;
import java.util.Set;
-import java.util.SortedMap;
+import java.util.SortedSet;
import org.apache.solr.client.solrj.SolrQuery;
@@ -144,7 +144,7 @@ public class RemoteSearch extends Thread { final int start, final int count,
final long time,
final Blacklist blacklist,
- final SortedMap<byte[], String> clusterselection) {
+ final SortedSet<byte[]> clusterselection) {
// check own peer status
//if (wordIndex.seedDB.mySeed() == null || wordIndex.seedDB.mySeed().getPublicAddress() == null) { return null; }
@@ -277,14 +277,13 @@ public class RemoteSearch extends Thread { final Blacklist blacklist) {
// check own peer status
- if (event.peers.mySeed() == null || event.peers.mySeed().getPublicAddress() == null) { return null; }
+ if (event.peers.mySeed() == null || event.peers.mySeed().getIPs().size() == 0) { return null; }
assert urlhashes != null;
assert urlhashes.length() > 0;
// prepare seed targets and threads
final Seed targetPeer = event.peers.getConnected(targethash);
if (targetPeer == null || targetPeer.hash == null) return null;
- if (event.preselectedPeerHashes != null) targetPeer.setAlternativeAddress(event.preselectedPeerHashes.get(ASCII.getBytes(targetPeer.hash)));
Thread secondary = new Thread() {
@Override
public void run() {
@@ -332,12 +331,8 @@ public class RemoteSearch extends Thread { assert solrQuery != null;
// check own peer status
- if (event.peers.mySeed() == null || event.peers.mySeed().getPublicAddress() == null) { return null; }
- // prepare seed targets and threads
- if (targetPeer != null && targetPeer.hash != null && event.preselectedPeerHashes != null) {
- if (!targetPeer.getFlagSolrAvailable()) return null; // solr interface not avail.
- targetPeer.setAlternativeAddress(event.preselectedPeerHashes.get(ASCII.getBytes(targetPeer.hash)));
- }
+ if (event.peers.mySeed() == null || event.peers.mySeed().getIPs().size() == 0) { return null; }
+ // prepare threads
Thread solr = new Thread() {
@Override
public void run() {
diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java index b7a31df9f..07ba8d5c9 100644 --- a/source/net/yacy/peers/Seed.java +++ b/source/net/yacy/peers/Seed.java @@ -44,13 +44,16 @@ import java.io.File; import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.net.Inet6Address;
import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.ParseException;
import java.util.Comparator;
import java.util.Date;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
@@ -132,6 +135,7 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> private static final String IPTYPE = "IPType";
private static final String FLAGS = "Flags";
public static final String FLAGSZERO = " ";
+
/** the applications version */
public static final String VERSION = "Version";
@@ -142,31 +146,52 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> /** the name of the peer (user-set) */
public static final String NAME = "Name";
public static final String HASH = "Hash";
+
/** Birthday - first startup */
private static final String BDATE = "BDate";
+
/** UTC-Offset */
public static final String UTC = "UTC";
private static final String PEERTAGS = "Tags";
/** the speed of indexing (pages/minute) of the peer */
public static final String ISPEED = "ISpeed";
+
/** the speed of retrieval (queries/minute) of the peer */
public static final String RSPEED = "RSpeed";
+
/** the number of minutes that the peer is up in minutes/day (moving average MA30) */
public static final String UPTIME = "Uptime";
+
/** the number of links that the peer has stored (LURL's) */
public static final String LCOUNT = "LCount";
+
/** the number of links that the peer has noticed, but not loaded (NURL's) */
public static final String NCOUNT = "NCount";
+
/** the number of links that the peer provides for remote crawls (ZURL's) */
public static final String RCOUNT = "RCount";
+
/** the number of different words the peer has indexed */
public static final String ICOUNT = "ICount";
+
/** the number of seeds that the peer has stored */
public static final String SCOUNT = "SCount";
+
/** the number of clients that the peer connects (connects/hour as double) */
public static final String CCOUNT = "CCount";
+
+ /** the public IP of this peer (old field, will be used to carry the IPv4) */
public static final String IP = "IP";
+
+ /** more public IPs of this peer, containing only IPv6 addresses. This list of of IPv6 addresses is separated with a vertical bar/pipe '|'.
+ * This list may have zero entries if the host does not have a IPv6 address. It may have more than one IPv6 address if the
+ * Host detects more than one public IPv6 locally but did not get a feedback from other peers if any of these addresses are
+ * reachable. The list may contain only one address, if a 'hello' with backping to another peer was successful and the other peer
+ * peer confirmed one of the IPv6 IPs which is then the only entry in this field.
+ */
+ public static final String IP6 = "IP6";
+
public static final String PORT = "Port";
public static final String SEEDLISTURL = "seedURL";
public static final String NEWS = "news"; // news attachment
@@ -192,9 +217,8 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> public final String hash;
/** a set of identity founding values, eg. IP, name of the peer, YaCy-version, ... */
private final ConcurrentMap<String, String> dna;
- private String alternativeIP = null;
private long birthdate; // keep this value in ram since it is often used and may cause lockings in concurrent situations.
-
+
public Seed(final String theHash, final ConcurrentMap<String, String> theDna) {
// create a seed with a pre-defined hash map
assert theHash != null;
@@ -297,36 +321,87 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> }
/**
- * used when doing routing within a cluster; this can assign a ip and a port that is used instead the
- * address stored in the seed DNA
+ * try to get the public IP<br>
+ *
+ * @return the IP or localhost IP (127.0.0.1)
*/
- public void setAlternativeAddress(final String ipport) {
- if ( ipport == null ) {
- return;
+ @Deprecated
+ public final String getIP() {
+ final String ipx = this.dna.get(Seed.IP); // may contain both, IPv4 or IPv6
+ final String ip6 = this.dna.get(Seed.IP6);
+ Set<String> ip6s = MapTools.string2set(ip6, "|");
+
+ if (ip6s == null || ip6s.size() == 0) {
+ if (ipx != null && !ipx.isEmpty()) return chopZoneID(ipx);
}
- final int p = ipport.lastIndexOf(':');
- if ( p < 0 ) {
- this.alternativeIP = ipport;
- } else {
- this.alternativeIP = ipport.substring(0, p);
+ if (ip6s != null && ip6s.size() == 1) {
+ // We prefer IPv6
+ for (String s: ip6s) if (s.length() > 0) return chopZoneID(s);
+ if (ipx != null && !ipx.isEmpty()) return chopZoneID(ipx);
}
+
+ // if we have more than one IPv6, then chances are high that one of them do not work.
+ // in that case we prefer the IPv4
+ if (ipx != null && !ipx.isEmpty()) return chopZoneID(ipx);
+ if (ip6s != null) for (String s: ip6s) if (s.length() > 0) return chopZoneID(s);
+
+ // in case that we don't have any address using the dna (i.e. a fresh peer), then use all locally known addresses
+ for (InetAddress i: Domains.myPublicIPv4()) return chopZoneID(i.getHostAddress());
+ for (InetAddress i: Domains.myPublicIPv6()) return chopZoneID(i.getHostAddress());
- if (this.alternativeIP.charAt(0) == '[' && this.alternativeIP.charAt(this.alternativeIP.length() - 1) == ']') {
- // IPv6 patch
- this.alternativeIP = this.alternativeIP.substring(1, this.alternativeIP.length() - 1);
- }
+ // final chance
+ return Domains.LOCALHOST;
}
/**
- * try to get the public IP<br>
- *
- * @return the IP or localhost IP (127.0.0.1)
+ * Get all my public IPs. If there was a static IP assignment, only one, that IP is returned.
+ * If no feedback from other peers exist, then all locally determined IPs are returned.
+ * If a feedback from other peers exist, then return at most two IPs:
+ * the latest IPv4 and the latest IPv6 which was returned during a hello process from a remote peer
+ * @return a set of IPs which are supposed to be my own public IPs
*/
- public final String getIP() {
- final String ip = this.dna.get(Seed.IP);
- return (ip == null || ip.isEmpty()) ? Domains.LOCALHOST : ip; // not public (but leave as is for now 2014-08-24)
+ public final Set<String> getIPs() {
+ Set<String> h = new LinkedHashSet<>();
+ final String ipx = this.dna.get(Seed.IP); // may contain both, IPv4 or IPv6
+ final String ip6 = this.dna.get(Seed.IP6);
+ Set<String> ip6s = MapTools.string2set(ip6, "|");
+
+ if (ip6s == null || ip6s.size() == 0) {
+ if (ipx != null && !ipx.isEmpty()) h.add(chopZoneID(ipx));
+ } else if (ip6s != null && ip6s.size() == 1) {
+ // We add IPv6 first because then those addresses appear first
+ // in the LinkedHashSet and are preferred by methods using only the first one.
+ for (String s: ip6s) if (s.length() > 0) h.add(chopZoneID(s));
+ if (ipx != null && !ipx.isEmpty()) h.add(chopZoneID(ipx));
+ } else {
+ // if we have more than one IPv6, then chances are high that one of them do not work.
+ // in that case we prefer the IPv4
+ if (ipx != null && !ipx.isEmpty()) h.add(chopZoneID(ipx));
+ if (ip6s != null) for (String s: ip6s) if (s.length() > 0) h.add(chopZoneID(s));
+ }
+
+ // in case that we don't have any address using the dna (i.e. a fresh peer), then use all locally known addresses
+ if (h.size() == 0) {
+ for (InetAddress i: Domains.myPublicIPv4()) h.add(chopZoneID(i.getHostAddress()));
+ for (InetAddress i: Domains.myPublicIPv6()) h.add(chopZoneID(i.getHostAddress()));
+ h.add(Domains.LOCALHOST);
+ }
+ return h;
+ }
+
+ private String chopZoneID(String ip) {
+ int i = ip.indexOf('%');
+ return i < 0 ? ip : ip.substring(0, i);
}
+ public boolean clash(Set<String> ips) {
+ Set<String> myIPs = getIPs();
+ for (String s: ips) {
+ if (myIPs.contains(s) && isProperIP(s)) return true;
+ }
+ return false;
+ }
+
/**
* try to get the peertype<br>
*
@@ -423,9 +498,55 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> return dflt;
}
}
-
+
+ /**
+ * set the Peer ip.
+ * This sets the IP and IP6 field according to the current fill state of that fields:
+ * - if no field has a content, then IP is filled with the given ip, even if that ip is of type IPv6
+ * - if IP is already set then check if this is equivalent with the given ip. If both are equal, nothing is done.
+ * If they are not equal, the IP and IPv6 field is set according to the type if the given ip: if the given ip
+ * is of type IPv4, then IP is set with ip, otherwise IP6 is set with the ip.
+ * ATTENTION: if the given IP is IPv6, then after the call that IP is the only one assigned to the peer!
+ * @param ip
+ */
public final void setIP(final String ip) {
- this.dna.put(Seed.IP, ip);
+ if (!isProperIP(ip)) return;
+ String oldIP = this.dna.get(Seed.IP);
+ String oldIP6 = this.dna.get(Seed.IP6);
+ if ((oldIP == null || oldIP.length() == 0) && (oldIP6 == null || oldIP6.length() == 0)) {
+ this.dna.put(Seed.IP, ip);
+ } else {
+ if (oldIP == null || !oldIP.equals(ip)) {
+ if (oldIP == null || oldIP.length() == 0 || ip.indexOf(':') == 0) this.dna.put(Seed.IP, ip); else this.dna.put(Seed.IP6, ip);
+ }
+ }
+ }
+
+ /**
+ * Set several local IPs which are good to access this peer.
+ * This OVERWRITES ALL ips stored before!
+ * @param ips list of IPs
+ */
+ public final void setIPs(final Set<String> ips) {
+ // we must sort IPv4 and IPv6 here
+ Set<String> ipv6 = new HashSet<>();
+ Set<String> ipv4 = new HashSet<>();
+ for (String ip: ips) if (isProperIP(ip)) {
+ if (ip.indexOf(':') >= 0) ipv6.add(ip); else ipv4.add(ip);
+ }
+ if (ipv4.size() == 0) {
+ if (ipv6.size() == 1) {
+ // if the only IP we have is IPv6, then put this into IP field
+ this.dna.put(Seed.IP, ipv6.iterator().next());
+ this.dna.put(Seed.IP6, "");
+ } else if (ipv6.size() > 1) {
+ this.dna.put(Seed.IP, "");
+ this.dna.put(Seed.IP6, MapTools.set2string(ipv6, "|", false));
+ }
+ } else {
+ this.dna.put(Seed.IP, ipv4.iterator().next());
+ this.dna.put(Seed.IP6, MapTools.set2string(ipv6, "|", false));
+ }
}
public final void setPort(final String port) {
@@ -575,78 +696,65 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> }
/**
+ * deprecated, use getIPs() instead
* @return the public address of the peer as IP:port string or <code>null</code> if no valid values for
* either the IP or the port could be retrieved from this yacySeed object
*/
+ @Deprecated
public final String getPublicAddress() {
- String ip = getIP();
- if (ip == null) ip = Domains.LOCALHOST; // that should not happen
-
- int p = ip.lastIndexOf(':');
- if (p > 0 && (ip.indexOf(':') == p || "]:".equals(ip.substring(p - 1, p + 1)))) return ip; // includes already the port
-
- final String port = this.dna.get(Seed.PORT);
+ return getPublicAddress(getIP());
+ }
+
+ /**
+ * generate a public address using a given ip. This combines the ip with the port and encloses the ip
+ * with square brackets if the ip is of typeIPv6
+ * @param ip
+ * @return an address string which can be used as host:port part of an url
+ */
+ public final String getPublicAddress(final InetAddress ip) {
+ // we do not use getPublicAddress(String ip) here to be able to check IPv6 with instanceof Inet6Address which is faster than indexOf(':')
+ if (ip == null) throw new RuntimeException("ip == NULL"); // that should not happen
+ final String port = this.dna.get(Seed.PORT); // we do not use getPort() here to avoid String->Integer->toString() conversion
if ( port == null || port.length() < 2 || port.length() > 5 ) {
- return null;
+ throw new RuntimeException("port not wellformed: " + port); // that should not happen
}
-
- final StringBuilder sb = new StringBuilder(ip.length() + port.length() + 3);
- if (ip.indexOf(':') >= 0) {
- // IPv6 Address!, see: http://en.wikipedia.org/wiki/IPv6_address#Literal_IPv6_addresses_in_network_resource_identifiers
- if (!ip.startsWith("[")) sb.append('[');
- sb.append(ip);
- if (!ip.endsWith("]")) sb.append(']');
- sb.append(':');
- sb.append(port);
+ final StringBuilder sb = new StringBuilder();
+ if (ip instanceof Inet6Address) {
+ sb.append('[').append(ip.getHostAddress()).append(']');
} else {
- sb.append(ip);
- sb.append(':');
- sb.append(port);
+ sb.append(ip.getHostAddress());
}
+ sb.append(':');
+ sb.append(port);
+
return sb.toString();
}
-
+
/**
- * If this seed is part of a cluster, the peer has probably the {@linkplain #alternativeIP} object set to
- * a local IP. If this is present and the public IP of this peer is identical to the public IP of the own
- * seed, construct an address using this IP; otherwise return the public address
- *
- * @see #getPublicAddress()
- * @return the alternative IP:port if present, else the public address
+ * generate a public address using a given ip. This combines the ip with the port and encloses the ip
+ * with square brackets if the ip is of typeIPv6
+ * @param ip
+ * @return an address string which can be used as host:port part of an url
*/
- public final String getClusterAddress() {
- if ( this.alternativeIP == null ) {
- return getPublicAddress();
- }
-
- final String port = this.dna.get(Seed.PORT);
+ public final String getPublicAddress(final String ip) {
+ if (ip == null) throw new RuntimeException("ip == NULL"); // that should not happen
+ final String port = this.dna.get(Seed.PORT); // we do not use getPort() here to avoid String->Integer->toString() conversion
if ( port == null || port.length() < 2 || port.length() > 5 ) {
- return null;
+ throw new RuntimeException("port not wellformed: " + port); // that should not happen
}
-
- final StringBuilder sb = new StringBuilder(this.alternativeIP.length() + port.length() + 3);
- if (this.alternativeIP.indexOf(':') >= 0) {
- // IPv6 Address!, see: http://en.wikipedia.org/wiki/IPv6_address#Literal_IPv6_addresses_in_network_resource_identifiers
- sb.append('[');
- sb.append(this.alternativeIP);
- sb.append(']');
- sb.append(':');
- sb.append(port);
+ final StringBuilder sb = new StringBuilder(ip.length() + port.length() + 3);
+ if (ip.indexOf(':') >= 0) {
+ if (!ip.startsWith("[")) sb.append('[');
+ sb.append(ip);
+ if (!ip.endsWith("]")) sb.append(']');
} else {
- sb.append(this.alternativeIP);
- sb.append(':');
- sb.append(port);
+ sb.append(ip);
}
+ sb.append(':');
+ sb.append(port);
return sb.toString();
}
- /**
- * @return the IP address of the peer represented by this yacySeed object as {@link InetAddress}
- */
- public final InetAddress getInetAddress() {
- return Domains.dnsResolve(getIP());
- }
-
/** @return the port number of this seed or <code>-1</code> if not present */
public final int getPort() {
final String port = this.dna.get(Seed.PORT);
@@ -662,12 +770,8 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> // because java thinks it must apply the UTC offset to the current time,
// to create a string that looks like our current time, it adds the local UTC offset to the
// time. To create a corrected UTC Date string, we first subtract the local UTC offset.
- final GenericFormatter my_SHORT_SECOND_FORMATTER =
- new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
- final String ls =
- my_SHORT_SECOND_FORMATTER
- .format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/));
- //System.out.println("SETTING LAST-SEEN of " + this.getName() + " to " + ls);
+ final GenericFormatter my_SHORT_SECOND_FORMATTER = new GenericFormatter(GenericFormatter.FORMAT_SHORT_SECOND, GenericFormatter.time_second); // use our own formatter to prevent concurrency locks with other processes
+ final String ls = my_SHORT_SECOND_FORMATTER.format(new Date(System.currentTimeMillis() /*- DateFormatter.UTCDiff()*/));
this.dna.put(Seed.LASTSEEN, ls);
}
@@ -1101,10 +1205,8 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> // check IP
if ( !checkOwnIP ) {
// checking of IP is omitted if we read the own seed file
- final String ipCheck = isProperIP(getIP());
- if ( ipCheck != null ) {
- return ipCheck;
- }
+ final String ip = getIP();
+ if (!isProperIP(ip)) return "not a proper IP " + ip;
}
// seedURL
@@ -1134,23 +1236,19 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed> return null;
}
- public static final String isProperIP(final String ipString) {
- // returns null if ipString is proper, a string with the cause otherwise
- if ( ipString == null ) {
- return ipString + " -> IP is null";
- }
- if ( ipString.length() < 8 ) {
- return ipString + " -> IP is too short: ";
- }
- if ( Switchboard.getSwitchboard().isAllIPMode() ) {
- return null;
- }
+ /**
+ * check if the given string containing an IP is proper. This checks also if the IP is within the given
+ * range of the network definition
+ * @param ipString
+ * @return true iff the IP is proper
+ */
+ public static final boolean isProperIP(final String ipString) {
+ if (ipString == null) return false;
+ if (ipString.length() < 3) return false;
+ if (Switchboard.getSwitchboard().isAllIPMode()) return true; // accept everyting
final boolean islocal = Domains.isLocal(ipString, null);
//if (islocal && Switchboard.getSwitchboard().isGlobalMode()) return ipString + " - local IP for global mode rejected";
- if ( !islocal && Switchboard.getSwitchboard().isIntranetMode() ) {
- return ipString + " - global IP for intranet mode rejected";
- }
- return null;
+ return islocal == Switchboard.getSwitchboard().isIntranetMode();
}
@Override
diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java index 295663643..d20a2d304 100644 --- a/source/net/yacy/peers/SeedDB.java +++ b/source/net/yacy/peers/SeedDB.java @@ -35,7 +35,8 @@ import java.util.Collection; import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.TreeMap;
+import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -208,11 +209,7 @@ public final class SeedDB implements AlternativeDomainNames { System.exit(-1);
}
}
- if (Switchboard.getSwitchboard().isIntranetMode()) {
- this.mySeed.setIP(Domains.myPublicLocalIP().getHostAddress()); // in intranet mode host address is best choice (to become senior peer)
- } else {
- this.mySeed.setIP("");// we delete the old information to see what we have now
- }
+ this.mySeed.setIPs(Switchboard.getSwitchboard().myPublicIPs());
this.mySeed.put(Seed.PEERTYPE, Seed.PEERTYPE_VIRGIN); // markup startup condition
}
@@ -229,10 +226,6 @@ public final class SeedDB implements AlternativeDomainNames { if (this.mySeed == null) {
if (sizeConnected() == 0) try {Thread.sleep(5000);} catch (final InterruptedException e) {} // wait for init
initMySeed();
- // check if my seed has an IP assigned
- if (myIP() == null || myIP().isEmpty()) {
- this.mySeed.setIP(Domains.myPublicLocalIP().getHostAddress());
- }
}
return this.mySeed;
}
@@ -246,10 +239,15 @@ public final class SeedDB implements AlternativeDomainNames { return mySeed().getName() + ".yacy";
}
- @Override
+ @Deprecated
public String myIP() {
return mySeed().getIP();
}
+
+ @Override
+ public Set<String> myIPs() {
+ return mySeed().getIPs();
+ }
@Override
public int myPort() {
@@ -349,7 +347,7 @@ public final class SeedDB implements AlternativeDomainNames { return new seedEnum(up, field, this.seedPotentialDB);
}
- public TreeMap<byte[], String> /* peer-b64-hashes/ipport */ clusterHashes(final String clusterdefinition) {
+ public TreeSet<byte[]> /* peer-b64-hashes/ipport */ clusterHashes(final String clusterdefinition) {
// collects seeds according to cluster definition string, which consists of
// comma-separated .yacy or .yacyh-domains
// the domain may be extended by an alternative address specification of the form
@@ -359,18 +357,16 @@ public final class SeedDB implements AlternativeDomainNames { // address ::= (<peername>'.yacy'|<peerhexhash>'.yacyh'){'='<ip>{':'<port}}
// clusterdef ::= {address}{','address}*
final String[] addresses = (clusterdefinition.isEmpty()) ? new String[0] : clusterdefinition.split(",");
- final TreeMap<byte[], String> clustermap = new TreeMap<byte[], String>(Base64Order.enhancedCoder);
+ final TreeSet<byte[]> clustermap = new TreeSet<>(Base64Order.enhancedCoder);
Seed seed;
- String hash, yacydom, ipport;
+ String hash, yacydom;
int p;
for (final String addresse : addresses) {
p = addresse.indexOf('=');
if (p >= 0) {
yacydom = addresse.substring(0, p);
- ipport = addresse.substring(p + 1);
} else {
yacydom = addresse;
- ipport = null;
}
if (yacydom.endsWith(".yacyh")) {
// find a peer with its hexhash
@@ -379,7 +375,7 @@ public final class SeedDB implements AlternativeDomainNames { if (seed == null) {
Network.log.warn("cluster peer '" + yacydom + "' was not found.");
} else {
- clustermap.put(ASCII.getBytes(hash), ipport);
+ clustermap.add(ASCII.getBytes(hash));
}
} else if (yacydom.endsWith(".yacy")) {
// find a peer with its name
@@ -387,7 +383,7 @@ public final class SeedDB implements AlternativeDomainNames { if (seed == null) {
Network.log.warn("cluster peer '" + yacydom + "' was not found.");
} else {
- clustermap.put(ASCII.getBytes(seed.hash), ipport);
+ clustermap.add(ASCII.getBytes(seed.hash));
}
} else {
Network.log.warn("cluster peer '" + addresse + "' has wrong syntax. the name must end with .yacy or .yacyh");
@@ -732,9 +728,7 @@ public final class SeedDB implements AlternativeDomainNames { }
// check local seed
- if (this.mySeed == null) return null;
- String s = this.mySeed.getIP();
- if (s == null || !ipString.equals(s)) return null;
+ if (this.mySeed == null || !this.mySeed.getIPs().contains(ipString)) return null;
int p = this.mySeed.getPort();
if (port > 0 && p != port) return null;
//System.out.println("*** found lookupByIP as my seed: " + peerIP.toString() + " -> " + this.mySeed.getName());
@@ -915,7 +909,7 @@ public final class SeedDB implements AlternativeDomainNames { seed = this.mySeed;
else return null;
}
- return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
+ return seed.getPublicAddress(seed.getIP()) + ((subdom == null) ? "" : ("/" + subdom));
} else if (host.endsWith(".yacy")) {
// identify subdomain
p = host.indexOf('.');
@@ -932,7 +926,7 @@ public final class SeedDB implements AlternativeDomainNames { // take local ip instead of external
return Switchboard.getSwitchboard().myPublicIP() + ":" + Switchboard.getSwitchboard().getConfig("port", "8090") + ((subdom == null) ? "" : ("/" + subdom));
}
- return seed.getPublicAddress() + ((subdom == null) ? "" : ("/" + subdom));
+ return seed.getPublicAddress(seed.getIP()) + ((subdom == null) ? "" : ("/" + subdom));
} else {
return null;
}
@@ -942,11 +936,11 @@ public final class SeedDB implements AlternativeDomainNames { // find target address
String address;
if (targetHash.equals(mySeed().hash)) {
- address = mySeed().getClusterAddress();
+ address = mySeed().getPublicAddress(mySeed().getIP());
} else {
final Seed targetSeed = getConnected(targetHash);
if (targetSeed == null) { return null; }
- address = targetSeed.getClusterAddress();
+ address = targetSeed.getPublicAddress(targetSeed.getIP());
}
if (address == null) address = "localhost" + (this.mySeed.getPort() > 0 ? ":" + this.mySeed.getPort() : "");
return address;
diff --git a/source/net/yacy/peers/Transmission.java b/source/net/yacy/peers/Transmission.java index 970c4532d..65662f295 100644 --- a/source/net/yacy/peers/Transmission.java +++ b/source/net/yacy/peers/Transmission.java @@ -268,8 +268,7 @@ public class Transmission { // get possibly newer target Info final Seed newTarget = Transmission.this.seeds.get(this.dhtTarget.hash); if (newTarget != null) { - final String oldAddress = this.dhtTarget.getPublicAddress(); - if ((oldAddress != null) && (oldAddress.equals(newTarget.getPublicAddress()))) { + if (this.dhtTarget.clash(newTarget.getIPs())) { newTarget.setFlagAcceptRemoteIndex(false); Transmission.this.seeds.update(newTarget.hash, newTarget); } else { diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index e9c783eca..c809727c6 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -64,7 +64,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Set; -import java.util.SortedMap; import java.util.SortedSet; import java.util.TreeMap; import java.util.TreeSet; @@ -265,7 +264,7 @@ public final class Switchboard extends serverSwitch { public int searchQueriesRobinsonFromLocal = 0; // absolute counter of all local queries submitted on this peer from a local or autheticated used public int searchQueriesRobinsonFromRemote = 0; // absolute counter of all local queries submitted on this peer from a remote IP without authentication public float searchQueriesGlobal = 0f; // partial counter of remote queries (1/number-of-requested-peers) - public SortedMap<byte[], String> clusterhashes; // map of peerhash(String)/alternative-local-address as ip:port or only ip (String) or null if address in seed should be used + public SortedSet<byte[]> clusterhashes; // a set of cluster hashes public List<Pattern> networkWhitelist, networkBlacklist; public FilterEngine domainList; private Dispatcher dhtDispatcher; @@ -1592,7 +1591,7 @@ public final class Switchboard extends serverSwitch { getConfig(SwitchboardConstants.CLUSTER_MODE, SwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER); if ( clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER) ) { // check if we got the request from a peer in the public cluster - return this.clusterhashes.containsKey(ASCII.getBytes(peer)); + return this.clusterhashes.contains(ASCII.getBytes(peer)); } return false; } @@ -1610,7 +1609,7 @@ public final class Switchboard extends serverSwitch { getConfig(SwitchboardConstants.CLUSTER_MODE, SwitchboardConstants.CLUSTER_MODE_PUBLIC_PEER); if ( clustermode.equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER) ) { // check if we got the request from a peer in the public cluster - return this.clusterhashes.containsKey(ASCII.getBytes(seed.hash)); + return this.clusterhashes.contains(ASCII.getBytes(seed.hash)); } return false; } @@ -2888,9 +2887,6 @@ public final class Switchboard extends serverSwitch { if ( (processCase == EventOrigin.GLOBAL_CRAWLING) && (queueEntry.initiator() != null) ) { final Seed initiatorPeer = this.peers.get(ASCII.String(queueEntry.initiator())); if ( initiatorPeer != null ) { - if ( this.clusterhashes != null ) { - initiatorPeer.setAlternativeAddress(this.clusterhashes.get(queueEntry.initiator())); - } // start a thread for receipt sending to avoid a blocking here SolrDocument sd = this.index.fulltext().getDefaultConfiguration().toSolrDocument(newEntry); new Thread(new receiptSending(initiatorPeer, new URIMetadataNode(sd)), "sending receipt to " + ASCII.String(queueEntry.initiator())).start(); @@ -3712,6 +3708,11 @@ public final class Switchboard extends serverSwitch { mySeed.setFlagAcceptRemoteCrawl(getConfigBool("crawlResponse", true)); mySeed.setFlagAcceptRemoteIndex(getConfigBool("allowReceiveIndex", true)); mySeed.setFlagSSLAvailable(this.getHttpServer() != null && this.getHttpServer().withSSL() && getConfigBool("server.https", false)); + + // set local ips + String staticIP = this.getConfig("staticIP", ""); + if (staticIP.length() > 0) mySeed.setIP(staticIP); + mySeed.setIPs(Switchboard.getSwitchboard().myPublicIPs()); } public void loadSeedLists() { diff --git a/source/net/yacy/search/query/SearchEvent.java b/source/net/yacy/search/query/SearchEvent.java index 7ac36955c..dd64b2f72 100644 --- a/source/net/yacy/search/query/SearchEvent.java +++ b/source/net/yacy/search/query/SearchEvent.java @@ -37,6 +37,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.SortedMap; +import java.util.SortedSet; import java.util.TreeMap; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -131,7 +132,7 @@ public final class SearchEvent { public final List<RemoteSearch> primarySearchThreadsL; public final List<Thread> nodeSearchThreads; public Thread[] secondarySearchThreads; - public final SortedMap<byte[], String> preselectedPeerHashes; + public final SortedSet<byte[]> preselectedPeerHashes; private final SortedMap<byte[], Integer> IACount; private final SortedMap<byte[], String> IAResults; private final SortedMap<byte[], HeuristicResult> heuristics; @@ -198,7 +199,7 @@ public final class SearchEvent { final QueryParams query, final SeedDB peers, final WorkTables workTables, - final SortedMap<byte[], String> preselectedPeerHashes, + final SortedSet<byte[]> preselectedPeerHashes, final boolean generateAbstracts, final LoaderDispatcher loader, final int remote_maxcount, diff --git a/source/net/yacy/search/query/SearchEventCache.java b/source/net/yacy/search/query/SearchEventCache.java index b55ec8682..9458bca07 100644 --- a/source/net/yacy/search/query/SearchEventCache.java +++ b/source/net/yacy/search/query/SearchEventCache.java @@ -29,7 +29,7 @@ package net.yacy.search.query; import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.SortedMap;
+import java.util.SortedSet;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.data.WorkTables;
@@ -135,7 +135,7 @@ public class SearchEventCache { final QueryParams query,
final SeedDB peers,
final WorkTables workTables,
- final SortedMap<byte[], String> preselectedPeerHashes,
+ final SortedSet<byte[]> preselectedPeerHashes,
final boolean generateAbstracts,
final LoaderDispatcher loader,
final int remote_maxcount,
diff --git a/source/net/yacy/search/snippet/TextSnippet.java b/source/net/yacy/search/snippet/TextSnippet.java index 57882eee6..767b9ab8f 100644 --- a/source/net/yacy/search/snippet/TextSnippet.java +++ b/source/net/yacy/search/snippet/TextSnippet.java @@ -26,7 +26,6 @@ package net.yacy.search.snippet; import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
@@ -48,7 +47,6 @@ import net.yacy.crawler.retrieval.Request; import net.yacy.crawler.retrieval.Response;
import net.yacy.document.Document;
import net.yacy.document.Parser;
-import net.yacy.document.SentenceReader;
import net.yacy.document.SnippetExtractor;
import net.yacy.document.WordTokenizer;
import net.yacy.document.parser.html.CharacterCoding;
diff --git a/source/net/yacy/server/http/AlternativeDomainNames.java b/source/net/yacy/server/http/AlternativeDomainNames.java index 0fabf5bc5..1d7025b9f 100644 --- a/source/net/yacy/server/http/AlternativeDomainNames.java +++ b/source/net/yacy/server/http/AlternativeDomainNames.java @@ -24,6 +24,8 @@ package net.yacy.server.http; +import java.util.Set; + public interface AlternativeDomainNames { /** @@ -42,10 +44,10 @@ public interface AlternativeDomainNames { public String myAlternativeAddress(); /** - * return the IP as string of my server address - * @return IP as string of this server + * return a set of IPs of this server + * @return IP as set of strings of this server */ - public String myIP(); + public Set<String> myIPs(); /** * return the port of my server address diff --git a/source/net/yacy/server/serverAccessTracker.java b/source/net/yacy/server/serverAccessTracker.java index d4f13817d..185fc995e 100644 --- a/source/net/yacy/server/serverAccessTracker.java +++ b/source/net/yacy/server/serverAccessTracker.java @@ -140,7 +140,9 @@ public class serverAccessTracker { track.add(new Track(System.currentTimeMillis(), accessPath)); clearTooOldAccess(track); } - if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis(); + if (Domains.isLocalhost(host)) lastLocalhostAccess = System.currentTimeMillis(); else { + System.out.println("******** Access not from localhost: " + host); + } } public static Collection<Track> accessTrack(final String host) { diff --git a/source/net/yacy/server/serverSwitch.java b/source/net/yacy/server/serverSwitch.java index 8c71614ae..ea81d3892 100644 --- a/source/net/yacy/server/serverSwitch.java +++ b/source/net/yacy/server/serverSwitch.java @@ -31,10 +31,13 @@ import java.io.IOException; import java.io.InputStreamReader;
import java.io.Reader;
import java.net.InetAddress;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Random;
+import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -50,6 +53,7 @@ import net.yacy.http.YaCyHttpServer; import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.workflow.BusyThread;
import net.yacy.kelondro.workflow.WorkflowThread;
+import net.yacy.peers.Seed;
import net.yacy.search.SwitchboardConstants;
public class serverSwitch
@@ -139,8 +143,11 @@ public class serverSwitch /**
* get my public IP, either set statically or figure out dynamic
- * @return
+ * This method is deprecated because there may be more than one public IPs of this peer,
+ * i.e. one IPv4 and one IPv6. Please use myPublicIPs() instead
+ * @return the public IP of this peer, if known
*/
+ @Deprecated
public String myPublicIP() {
// if a static IP was configured, we have to return it here ...
final String staticIP = getConfig("staticIP", "");
@@ -152,6 +159,31 @@ public class serverSwitch return null;
}
+ /**
+ * Get all my public IPs. If there was a static IP assignment, only one, that IP is returned.
+ * @return a set of IPs which are supposed to be my own public IPs
+ */
+ public Set<String> myPublicIPs() {
+ // if a static IP was configured, we have to return it here ...
+ final String staticIP = getConfig("staticIP", "");
+ if ( staticIP.length() > 0 ) {
+ HashSet<String> h = new HashSet<>();
+ h.add(staticIP);
+ return h;
+ }
+
+ Set<String> h = new LinkedHashSet<>();
+ for (InetAddress i: Domains.myPublicIPv6()) {
+ String s = i.getHostAddress();
+ if (Seed.isProperIP(s)) h.add(s);
+ }
+ for (InetAddress i: Domains.myPublicIPv4()) {
+ String s = i.getHostAddress();
+ if (Seed.isProperIP(s)) h.add(s);
+ }
+ return h;
+ }
+
// a logger for this switchboard
public void setLog(final ConcurrentLog log) {
this.log = log;
diff --git a/source/net/yacy/yacy.java b/source/net/yacy/yacy.java index 1b58a5e4b..36adfcbc4 100644 --- a/source/net/yacy/yacy.java +++ b/source/net/yacy/yacy.java @@ -530,7 +530,7 @@ public final class yacy { p.load(fis);
fis.close();
// Test for server access restriction (is implemented using Jetty IPaccessHandler which does not support IPv6
- // try to disavle IPv6
+ // try to disable IPv6
String teststr = p.getProperty("serverClient", "*");
if (!teststr.equals("*")) {
// testing on Win-8 showed this property has to be set befor Switchboard starts
@@ -569,7 +569,6 @@ public final class yacy { }
}
-
/**
* Main-method which is started by java. Checks for special arguments or
* starts up the application.
@@ -601,8 +600,7 @@ public final class yacy { if (OS.isWindows) headless = false;
if (args.length >= 1 && args[0].toLowerCase().equals("-gui")) headless = false;
System.setProperty("java.awt.headless", headless ? "true" : "false");
- // System.setProperty("java.net.preferIPv4Stack", "true"); // DO NOT PREFER IPv6, i.e. freifunk uses ipv6 only and host resolving does not work
-
+
String s = ""; for (final String a: args) s += a + " ";
yacyRelease.startParameter = s.trim();
diff --git a/startYACY.sh b/startYACY.sh index ede26fd7e..f8a20e9cb 100755 --- a/startYACY.sh +++ b/startYACY.sh @@ -6,7 +6,7 @@ PIDFILE="yacy.pid" OS="`uname`" #get javastart args -JAVA_ARGS="-server -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8"; +JAVA_ARGS="-server -Djava.awt.headless=true -Dfile.encoding=UTF-8"; #JAVA_ARGS="-verbose:gc -XX:+PrintGCTimeStamps -XX:+PrintGCDetails $JAVA_ARGS"; #check if OS is Sun Solaris or one of the OpenSolaris distributions and use different version of id if necessary diff --git a/startYACY_debug.bat b/startYACY_debug.bat index 3af009b59..28bafe8be 100644 --- a/startYACY_debug.bat +++ b/startYACY_debug.bat @@ -17,7 +17,7 @@ if exist DATA\SETTINGS\httpProxy.conf GoTo :RENAMEINDEX if exist DATA\SETTINGS\yacy.conf GoTo :GETSTARTOPTS
:STARTJAVA
-set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Dfile.encoding=UTF-8
+set javacmd=%javacmd% -XX:-UseGCOverheadLimit -Djava.awt.headless=true -Dfile.encoding=UTF-8
Rem Starting YaCy
Echo Generated classpath:%CLASSPATH%
Echo JRE Parameters:%javacmd%
|