IPv6-aware fixes

not a full IPv6 support fix, but changes towards

5 files changed, 351 insertions, 125 deletions

diff --git a/source/net/yacy/cora/protocol/Domains.java b/source/net/yacy/cora/protocol/Domains.java
index 527d6cf67..728dfd8c4 100644
--- a/source/net/yacy/cora/protocol/Domains.java
+++ b/source/net/yacy/cora/protocol/Domains.java
@@ -1184,6 +1184,8 @@ public class Domains {
     }
 
     public static String chopZoneID(final String ip) {
+        // Some peer seeds are IPv6-only and may not carry the legacy Seed.IP value.
+        if (ip == null) return null;
         final int i = ip.indexOf('%');
         return i < 0 ? ip : ip.substring(0, i);
     }
@@ -1375,4 +1377,4 @@ public class Domains {
             System.out.println("Intranet IP: " + b);
         }
     }
-}
\ No newline at end of file
+}
diff --git a/source/net/yacy/peers/Seed.java b/source/net/yacy/peers/Seed.java
index ad1976cc2..189315665 100644
--- a/source/net/yacy/peers/Seed.java
+++ b/source/net/yacy/peers/Seed.java
@@ -348,31 +348,28 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
     }

 

 

-    private Set<String> getIPv6Entries() {

-        String ip6s = this.dna.get(Seed.IP6);

-        final Set<String> set = Collections.synchronizedSet(new HashSet<String>());

-        if (ip6s == null) return set;

-        final StringTokenizer st = new StringTokenizer(ip6s, "|");

-        while (st.hasMoreTokens()) {

-            set.add(Domains.chopZoneID(st.nextToken().trim()));

-        }

-        return set;

-    }

+    private Set<String> getIPv6Entries() {
+        String ip6s = this.dna.get(Seed.IP6);
+        final Set<String> set = Collections.synchronizedSet(new HashSet<String>());
+        if (ip6s == null) return set;
+        final StringTokenizer st = new StringTokenizer(ip6s, "|");
+        while (st.hasMoreTokens()) {
+            final String ip = Domains.chopZoneID(st.nextToken().trim());
+            if (ip != null && !ip.isEmpty()) set.add(ip);
+        }
+        return set;
+    }
     

     /**

      * try to get the public IP<br>

      *

      * @return the public IP or null

      */

-    @Deprecated

-    public final String getIP() {

-        final String ipx = this.dna.get(Seed.IP); // may contain both, IPv4 or IPv6

-        if (ipx != null && !ipx.isEmpty()) return Domains.chopZoneID(ipx);

-        

-        Set<String> ip6s = getIPv6Entries();

-        if (ip6s != null && ip6s.size() > 0) return ip6s.iterator().next(); 

-        return null;

-    }

+    @Deprecated
+    public final String getIP() {
+        final Set<String> ips = getIPs();
+        return ips.isEmpty() ? null : ips.iterator().next();
+    }
 

     /**

      * Get all my public IPs. If there was a static IP assignment, only one, that IP is returned.

@@ -396,24 +393,21 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
      * count the number of IPs assgined to that peer

      * @return the number of peers in field IP (should be 1 all the time) plus the number of IPs in the IP6 field.

      */

-    public final int countIPs() {

-        final String ipx = this.dna.get(Seed.IP); // may contain both, IPv4 or IPv6

-        Set<String> ip6s = getIPv6Entries();

-        

-        if (ip6s == null || ip6s.size() == 0) {

-            return (ipx == null || ipx.isEmpty()) ? 0 : 1;

-        }

-        return (ipx == null || ipx.isEmpty()) ? ip6s.size() : ip6s.size() + 1;

-    }

+    public final int countIPs() {
+        return getIPs().size();
+    }
     

     /**

      * remove the given IP from the seed. Be careful not to remove the last IP; maybe call countIPs before calling the method.

      * @param ip

      * @return true if the IP was in the seed and had been removed. If the peer did not change, this returns false.

      */

-    public final boolean removeIP(String ip) {

-        String ipx = Domains.chopZoneID(this.dna.get(Seed.IP)); // may contain both, IPv4 or IPv6

-        Set<String> ip6s = getIPv6Entries();

+    public final boolean removeIP(String ip) {
+        // A failed publish attempt can pass a missing legacy IPv4 address for IPv6-only seeds.
+        if (ip == null || ip.isEmpty()) return false;
+        ip = Domains.chopZoneID(ip);
+        String ipx = Domains.chopZoneID(this.dna.get(Seed.IP)); // may contain both, IPv4 or IPv6
+        Set<String> ip6s = getIPv6Entries();
                 

         if (ip6s == null || ip6s.size() == 0) {

             if (ipx != null && !ipx.isEmpty() && ipx.equals(ip)) {

@@ -442,13 +436,27 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
      * @param ips

      * @return true if any of the given IPs are identical to the Seeds IP set

      */

-    public boolean clash(Set<String> ips) {

-        Set<String> myIPs = getIPs();

-        for (String s: ips) {

-            if (myIPs.contains(s) && isProperIP(s)) return true;

-        }

-        return false;

-    }

+    public boolean clash(Set<String> ips) {
+        Set<String> myIPs = getIPs();
+        for (String s: ips) {
+            if (!isProperIP(s)) continue;
+            if (myIPs.contains(Domains.chopZoneID(s))) return true;
+            for (final String myIP: myIPs) {
+                if (sameIP(myIP, s)) return true;
+            }
+        }
+        return false;
+    }
+
+    private static boolean sameIP(final String left, final String right) {
+        final String normalizedLeft = Domains.chopZoneID(left);
+        final String normalizedRight = Domains.chopZoneID(right);
+        if (normalizedLeft == null || normalizedRight == null) return false;
+        if (normalizedLeft.equals(normalizedRight)) return true;
+        final java.net.InetAddress leftAddress = Domains.dnsResolve(normalizedLeft);
+        final java.net.InetAddress rightAddress = Domains.dnsResolve(normalizedRight);
+        return leftAddress != null && leftAddress.equals(rightAddress);
+    }
     

     /**

      * try to get the peertype<br>

@@ -547,29 +555,39 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
         }

     }

     

-    /**

-     * set the Peer ip.

-     * This sets the IP and IP6 field according to the current fill state of that fields:

-     * - if no field has a content, then IP is filled with the given ip, even if that ip is of type IPv6

-     * - if IP is already set then check if this is equivalent with the given ip. If both are equal, nothing is done.

-     *   If they are not equal, the IP and IPv6 field is set according to the type if the given ip: if the given ip

-     *   is of type IPv4, then IP is set with ip, otherwise IP6 is set with the ip.

-     * ATTENTION: if the given IP is IPv6, then after the call that IP is the only one assigned to the peer!

-     * @param ip

-     */

-    public final void setIP(String ip) {

-        ip = Domains.chopZoneID(ip);

-        if (!isProperIP(ip)) return;

-        String oldIP = this.dna.get(Seed.IP);

-        String oldIP6 = this.dna.get(Seed.IP6);

-        if ((oldIP == null || oldIP.length() == 0) && (oldIP6 == null || oldIP6.length() == 0)) {

-            this.dna.put(Seed.IP, ip);

-        } else {

-            if (oldIP == null || !oldIP.equals(ip)) {

-                if (oldIP == null || oldIP.length() == 0 || ip.indexOf(':') == 0) this.dna.put(Seed.IP, ip); else this.dna.put(Seed.IP6, ip);

-            }

-        }

-    }

+    /**
+     * set the Peer ip.
+     * IPv4 stays the preferred primary address in the legacy IP field. IPv6 is added to IP6
+     * when a primary address already exists, while IPv6-only peers can still use IP as their
+     * existence address until an IPv4 address becomes known.
+     * @param ip
+     */
+    public final void setIP(String ip) {
+        ip = Domains.chopZoneID(ip);
+        if (!isProperIP(ip)) return;
+        final boolean ipv6 = ip.indexOf(':') >= 0;
+        String oldIP = Domains.chopZoneID(this.dna.get(Seed.IP));
+        final Set<String> ip6s = getIPv6Entries();
+
+        if (oldIP == null || oldIP.length() == 0) {
+            this.dna.put(Seed.IP, ip);
+            ip6s.remove(ip);
+            this.dna.put(Seed.IP6, MapTools.set2string(ip6s, "|", false));
+            return;
+        }
+
+        if (oldIP.equals(ip)) return;
+
+        if (ipv6) {
+            ip6s.add(ip);
+        } else {
+            // When IPv4 appears after an IPv6-only primary address, preserve that IPv6 in IP6.
+            if (oldIP.indexOf(':') >= 0) ip6s.add(oldIP);
+            ip6s.remove(ip);
+            this.dna.put(Seed.IP, ip);
+        }
+        this.dna.put(Seed.IP6, MapTools.set2string(ip6s, "|", false));
+    }
 

     /**

      * Set the public facing port.

@@ -1386,14 +1404,18 @@ public class Seed implements Cloneable, Comparable<Seed>, Comparator<Seed>
      * @param ipString

      * @return true if the IP is proper

      */

-    public static final boolean isProperIP(final String ipString) {

-        if (ipString == null) return false;

-        if (ipString.length() < 3) return false;

-        if (Switchboard.getSwitchboard().isAllIPMode()) return true; // accept everyting

-        final boolean islocal = Domains.isLocal(ipString, null);

-        //if (islocal && Switchboard.getSwitchboard().isGlobalMode()) return ipString + " - local IP for global mode rejected";

-        return islocal == Switchboard.getSwitchboard().isIntranetMode();

-    }

+    public static final boolean isProperIP(final String ipString) {
+        if (ipString == null) return false;
+        if (ipString.length() < 3) return false;
+        final Switchboard switchboard = Switchboard.getSwitchboard();
+        // Seed address normalization is also used in isolated tests and early object setup.
+        // When no Switchboard exists yet, keep the method limited to basic sanity checks.
+        if (switchboard == null) return true;
+        if (switchboard.isAllIPMode()) return true; // accept everyting
+        final boolean islocal = Domains.isLocal(ipString, null);
+        //if (islocal && Switchboard.getSwitchboard().isGlobalMode()) return ipString + " - local IP for global mode rejected";
+        return islocal == switchboard.isIntranetMode();
+    }
 

     /**

      * checks if the given port is within the allowed range (1-65535).

diff --git a/source/net/yacy/peers/SeedDB.java b/source/net/yacy/peers/SeedDB.java
index 38c82d210..1b4852662 100644
--- a/source/net/yacy/peers/SeedDB.java
+++ b/source/net/yacy/peers/SeedDB.java
@@ -742,63 +742,82 @@ public final class SeedDB implements AlternativeDomainNames {
         }

 

         // then try to use the cache

-        Seed seed = null;

-        final String ipString = peerIP.getHostAddress();

-

-        if (lookupConnected) {

-            try {

-                final Collection<byte[]> idx = this.seedActiveDB.select(Seed.IP, ipString);

-                for (final byte[] pk: idx) {

-                    seed = this.getConnected(pk);

-                    if (seed == null) continue;

-                    if (port > 0 && seed.getPort() != port) continue;

-                    //System.out.println("*** found lookupByIP in connected: " + peerIP.toString() + " -> " + seed.getName());

-                    return seed;

-                }

-            } catch (final IOException e ) {

-                ConcurrentLog.logException(e);

-            }

-        }

-

-        if (lookupDisconnected) {

-            try {

-                final Collection<byte[]> idx = this.seedPassiveDB.select(Seed.IP, ipString);

-                for (final byte[] pk: idx) {

-                    seed = this.getDisconnected(pk);

-                    if (seed == null) continue;

-                    if (port > 0 && seed.getPort() != port) continue;

-                    //System.out.println("*** found lookupByIP in disconnected: " + peerIP.toString() + " -> " + seed.getName());

-                    return seed;

-                }

-            } catch (final IOException e ) {

-                ConcurrentLog.logException(e);

-            }

-        }

-

-        if (lookupPotential) {

-            try {

-                final Collection<byte[]> idx = this.seedPotentialDB.select(Seed.IP, ipString);

-                for (final byte[] pk: idx) {

-                    seed = this.getPotential(pk);

-                    if (seed == null) continue;

-                    if (port > 0 && seed.getPort() != port) continue;

-                    //System.out.println("*** found lookupByIP in potential: " + peerIP.toString() + " -> " + seed.getName());

-                    return seed;

-                }

-            } catch (final IOException e ) {

-                ConcurrentLog.logException(e);

-            }

-        }

-

-        // check local seed

-        if (this.mySeed == null || !this.mySeed.getIPs().contains(ipString)) return null;

+        Seed seed = null;
+        final String ipString = peerIP.getHostAddress();
+
+        if (lookupConnected) {
+            seed = this.lookupByIndexedIP(this.seedActiveDB, ipString, port);
+            if (seed != null) return seed;
+        }
+
+        if (lookupDisconnected) {
+            seed = this.lookupByIndexedIP(this.seedPassiveDB, ipString, port);
+            if (seed != null) return seed;
+        }
+
+        if (lookupPotential) {
+            seed = this.lookupByIndexedIP(this.seedPotentialDB, ipString, port);
+            if (seed != null) return seed;
+        }
+
+        if (lookupConnected) {
+            seed = this.lookupBySeedAddresses(peerIP, ipString, port, this.seedsConnected(true, false, null, 0.0));
+            if (seed != null) return seed;
+        }
+
+        if (lookupDisconnected) {
+            seed = this.lookupBySeedAddresses(peerIP, ipString, port, this.seedsDisconnected(true, false, null, 0.0));
+            if (seed != null) return seed;
+        }
+
+        if (lookupPotential) {
+            seed = this.lookupBySeedAddresses(peerIP, ipString, port, new seedEnum(true, false, null, null, this.seedPotentialDB, 0.0));
+            if (seed != null) return seed;
+        }
+
+        // check local seed
+        if (this.mySeed == null || !seedHasIP(this.mySeed, peerIP, ipString)) return null;
         final int p = this.mySeed.getPort();

         if (port > 0 && p != port) return null;

-        //System.out.println("*** found lookupByIP as my seed: " + peerIP.toString() + " -> " + this.mySeed.getName());

-        return this.mySeed;

-    }

-

-    private ArrayList<String> storeSeedList(final File seedFile, final boolean addMySeed) throws IOException {

+        //System.out.println("*** found lookupByIP as my seed: " + peerIP.toString() + " -> " + this.mySeed.getName());
+        return this.mySeed;
+    }
+
+    private Seed lookupByIndexedIP(final MapDataMining database, final String ipString, final int port) {
+        try {
+            final Collection<byte[]> idx = database.select(Seed.IP, ipString);
+            for (final byte[] pk: idx) {
+                final Seed seed = this.get(pk, database);
+                if (seed == null) continue;
+                if (port > 0 && seed.getPort() != port) continue;
+                return seed;
+            }
+        } catch (final IOException e ) {
+            ConcurrentLog.logException(e);
+        }
+        return null;
+    }
+
+    private Seed lookupBySeedAddresses(final InetAddress peerIP, final String ipString, final int port, final Iterator<Seed> seeds) {
+        while (seeds.hasNext()) {
+            final Seed seed = seeds.next();
+            if (seed == null) continue;
+            if (port > 0 && seed.getPort() != port) continue;
+            if (seedHasIP(seed, peerIP, ipString)) return seed;
+        }
+        return null;
+    }
+
+    private static boolean seedHasIP(final Seed seed, final InetAddress peerIP, final String ipString) {
+        for (final String seedIP: seed.getIPs()) {
+            if (ipString.equals(seedIP)) return true;
+            final InetAddress resolvedSeedIP = Domains.dnsResolve(seedIP);
+            if (peerIP.equals(resolvedSeedIP)) return true;
+        }
+        return false;
+    }
+
+    private ArrayList<String> storeSeedList(final File seedFile, final boolean addMySeed) throws IOException {
         PrintWriter pw = null;

         final ArrayList<String> v = new ArrayList<>(this.seedActiveDB.size() + 1);

         try {

diff --git a/test/java/net/yacy/peers/SeedDBTest.java b/test/java/net/yacy/peers/SeedDBTest.java
new file mode 100644
index 000000000..a787ae1ba
--- /dev/null
+++ b/test/java/net/yacy/peers/SeedDBTest.java
@@ -0,0 +1,100 @@
+package net.yacy.peers;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.File;
+import java.net.InetAddress;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class SeedDBTest {
+
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    private SeedDB seedDB;
+
+    @Before
+    public void setUp() throws Exception {
+        final File networkRoot = this.temporaryFolder.newFolder("network");
+        this.seedDB = new SeedDB(
+                networkRoot,
+                "seedActive.db",
+                "seedPassive.db",
+                "seedPotential.db",
+                new File(networkRoot, SeedDB.DBFILE_OWN_SEED),
+                1,
+                4,
+                false,
+                false);
+    }
+
+    @After
+    public void tearDown() {
+        if (this.seedDB != null) this.seedDB.close();
+    }
+
+    @Test
+    public void lookupByIPFindsIPv6StoredInConnectedSeedIP6() throws Exception {
+        final Seed seed = seed("AAAAAAAAAAAA", "connected-peer", 8090);
+        seed.setIP("192.0.2.10");
+        seed.setIP("2001:db8::10");
+        this.seedDB.addConnected(seed);
+
+        final Seed found = this.seedDB.lookupByIP(InetAddress.getByName("2001:db8::10"), 8090, true, false, false);
+
+        assertEquals(seed.hash, found.hash);
+    }
+
+    @Test
+    public void lookupByIPFindsIPv6StoredInDisconnectedSeedIP6() throws Exception {
+        final Seed seed = seed("BBBBBBBBBBBB", "disconnected-peer", 8091);
+        seed.setIP("192.0.2.11");
+        seed.setIP("2001:db8::11");
+        this.seedDB.addDisconnected(seed);
+
+        final Seed found = this.seedDB.lookupByIP(InetAddress.getByName("2001:db8::11"), 8091, false, true, false);
+
+        assertEquals(seed.hash, found.hash);
+    }
+
+    @Test
+    public void lookupByIPFindsIPv6StoredInPotentialSeedIP6() throws Exception {
+        final Seed seed = seed("CCCCCCCCCCCC", "potential-peer", 8092);
+        seed.setIP("192.0.2.12");
+        seed.setIP("2001:db8::12");
+        this.seedDB.addPotential(seed);
+
+        final Seed found = this.seedDB.lookupByIP(InetAddress.getByName("2001:db8::12"), 8092, false, false, true);
+
+        assertEquals(seed.hash, found.hash);
+    }
+
+    @Test
+    public void lookupByIPHonorsPortForIPv6FallbackMatches() throws Exception {
+        final Seed seed = seed("DDDDDDDDDDDD", "port-peer", 8093);
+        seed.setIP("192.0.2.13");
+        seed.setIP("2001:db8::13");
+        this.seedDB.addConnected(seed);
+
+        final Seed found = this.seedDB.lookupByIP(InetAddress.getByName("2001:db8::13"), 8094, true, false, false);
+
+        assertNull(found);
+    }
+
+    private static Seed seed(final String hash, final String name, final int port) {
+        final ConcurrentMap<String, String> dna = new ConcurrentHashMap<String, String>();
+        final Seed seed = new Seed(hash, dna);
+        seed.setName(name);
+        seed.setType(Seed.PEERTYPE_SENIOR);
+        seed.setPort(port);
+        return seed;
+    }
+}
diff --git a/test/java/net/yacy/peers/SeedTest.java b/test/java/net/yacy/peers/SeedTest.java
new file mode 100644
index 000000000..6e221d843
--- /dev/null
+++ b/test/java/net/yacy/peers/SeedTest.java
@@ -0,0 +1,83 @@
+package net.yacy.peers;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Set;
+import java.util.HashSet;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.junit.Test;
+
+public class SeedTest {
+
+    private static Seed seed() {
+        final ConcurrentMap<String, String> dna = new ConcurrentHashMap<String, String>();
+        return new Seed("testseedhash", dna);
+    }
+
+    @Test
+    public void setIPAddsIPv6WithoutReplacingExistingIPv6Entries() {
+        final Seed seed = seed();
+        seed.setIP("192.0.2.10");
+        seed.setIP("2001:db8::1");
+        seed.setIP("2001:db8::2");
+
+        assertEquals("192.0.2.10", seed.get(Seed.IP, ""));
+        assertTrue(seed.getIPs().contains("2001:db8::1"));
+        assertTrue(seed.getIPs().contains("2001:db8::2"));
+        assertEquals(3, seed.countIPs());
+    }
+
+    @Test
+    public void setIPPreservesIPv6OnlyPrimaryWhenIPv4BecomesAvailable() {
+        final Seed seed = seed();
+        seed.setIP("2001:db8::1");
+        seed.setIP("192.0.2.10");
+
+        assertEquals("192.0.2.10", seed.get(Seed.IP, ""));
+        assertTrue(seed.getIPs().contains("2001:db8::1"));
+        assertTrue(seed.getIPs().contains("192.0.2.10"));
+        assertEquals(2, seed.countIPs());
+    }
+
+    @Test
+    public void setIPNormalizesZoneIdsAndAvoidsDuplicateIPv6Entries() {
+        final Seed seed = seed();
+        seed.setIP("192.0.2.10");
+        seed.setIP("fe80::1%en0");
+        seed.setIP("fe80::1%eth0");
+
+        final Set<String> ips = seed.getIPs();
+        assertTrue(ips.contains("fe80::1"));
+        assertFalse(ips.contains("fe80::1%en0"));
+        assertFalse(ips.contains("fe80::1%eth0"));
+        assertEquals(2, seed.countIPs());
+    }
+
+    @Test
+    public void clashMatchesIPv6WithZoneId() {
+        final Seed seed = seed();
+        seed.setIP("192.0.2.10");
+        seed.setIP("fe80::1");
+
+        final Set<String> otherIPs = new HashSet<String>();
+        otherIPs.add("fe80::1%en0");
+
+        assertTrue(seed.clash(otherIPs));
+    }
+
+    @Test
+    public void clashMatchesEquivalentIPv6TextForms() {
+        final Seed seed = seed();
+        seed.setIP("192.0.2.10");
+        seed.setIP("2001:db8::1");
+
+        final Set<String> otherIPs = new HashSet<String>();
+        otherIPs.add("2001:db8:0:0:0:0:0:1");
+
+        assertTrue(seed.clash(otherIPs));
+    }
+}