diff options
25 files changed, 407 insertions, 519 deletions
diff --git a/build.properties b/build.properties index baa9dbb8a..1d231e3d3 100644 --- a/build.properties +++ b/build.properties @@ -3,7 +3,7 @@ javacSource=1.5 javacTarget=1.5
# Release Configuration
-releaseVersion=0.612
+releaseVersion=0.613
stdReleaseFile=yacy_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
embReleaseFile=yacy_emb_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
proReleaseFile=yacy_pro_v${releaseVersion}_${DSTAMP}_${releaseNr}.tar.gz
diff --git a/defaults/yacy.init b/defaults/yacy.init index 74d82fdfe..48a66bbcc 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -558,11 +558,11 @@ performanceSpeed=100 50_localcrawl_memprereq=4194304
50_localcrawl_isPaused=false
60_remotecrawlloader_idlesleep=60000
-60_remotecrawlloader_busysleep=40000
+60_remotecrawlloader_busysleep=30000
60_remotecrawlloader_memprereq=2097152
60_remotecrawlloader_isPaused=false
-62_remotetriggeredcrawl_idlesleep=10000
-62_remotetriggeredcrawl_busysleep=1000
+62_remotetriggeredcrawl_idlesleep=30000
+62_remotetriggeredcrawl_busysleep=10000
62_remotetriggeredcrawl_memprereq=6291456
62_remotetriggeredcrawl_isPaused=false
80_indexing_idlesleep=1000
diff --git a/htroot/ConfigLanguage_p.java b/htroot/ConfigLanguage_p.java index 29e4b130a..fff681e8b 100644 --- a/htroot/ConfigLanguage_p.java +++ b/htroot/ConfigLanguage_p.java @@ -108,7 +108,6 @@ public class ConfigLanguage_p { //reread language files langFiles = listManager.getDirListing(langPath); - int i; final HashMap<String, String> langNames = translator.langMap(env); String langKey, langName; diff --git a/htroot/ConfigNetwork_p.html b/htroot/ConfigNetwork_p.html index 35f112d68..642e4d3b9 100644 --- a/htroot/ConfigNetwork_p.html +++ b/htroot/ConfigNetwork_p.html @@ -150,7 +150,7 @@ Perform web indexing upon request of another peer.<br /> This works only if you are a senior peer.<br /> <label for="acceptCrawlLimit">Load with a maximum of</label> - <input id="acceptCrawlLimit" name="acceptCrawlLimit" type="text" size="3" maxlength="3" value="#[acceptCrawlLimit]#" /> pages per minute + <input id="acceptCrawlLimit" name="acceptCrawlLimit" type="text" size="4" maxlength="4" value="#[acceptCrawlLimit]#" /> pages per minute </dd> </dl> </fieldset> diff --git a/htroot/ConfigNetwork_p.java b/htroot/ConfigNetwork_p.java index 0e52c031f..660598b0e 100644 --- a/htroot/ConfigNetwork_p.java +++ b/htroot/ConfigNetwork_p.java @@ -149,8 +149,21 @@ public class ConfigNetwork_p { newppm = Math.max(1, Integer.parseInt(post.get("acceptCrawlLimit", "1"))); } catch (final NumberFormatException e) {} final long newBusySleep = Math.max(100, 60000 / newppm); + + // propagate to crawler final serverBusyThread rct = sb.getThread(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); + sb.setConfig(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, newBusySleep); + sb.setConfig(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_IDLESLEEP, newBusySleep * 3); rct.setBusySleep(newBusySleep); + rct.setIdleSleep(newBusySleep * 3); + + // propagate to loader + final serverBusyThread rcl = sb.getThread(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER); + sb.setConfig(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_BUSYSLEEP, newBusySleep * 5); + sb.setConfig(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_CRAWL_LOADER_IDLESLEEP, newBusySleep * 10); + rcl.setBusySleep(newBusySleep * 5); + rcl.setIdleSleep(newBusySleep * 10); + sb.setConfig(plasmaSwitchboardConstants.CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP, Long.toString(newBusySleep)); sb.setConfig("cluster.peers.ipport", checkIPPortList(post.get("cluster.peers.ipport", ""))); diff --git a/htroot/IndexControlRWIs_p.java b/htroot/IndexControlRWIs_p.java index 0fe9e5e7c..6e6f7fd92 100644 --- a/htroot/IndexControlRWIs_p.java +++ b/htroot/IndexControlRWIs_p.java @@ -320,7 +320,7 @@ public class IndexControlRWIs_p { sb.webIndex.removeEntries(keyhash, urlHashes); } - if (prop.getInt("searchresult", 0) == 3) plasmaSearchAPI.listHosts(prop, keyhash, sb.webIndex.peerActions); + if (prop.getInt("searchresult", 0) == 3) plasmaSearchAPI.listHosts(prop, keyhash, sb); } diff --git a/htroot/IndexTransfer_p.java b/htroot/IndexTransfer_p.java index 68576a340..57b6c4e3a 100644 --- a/htroot/IndexTransfer_p.java +++ b/htroot/IndexTransfer_p.java @@ -39,6 +39,7 @@ import de.anomic.plasma.plasmaSwitchboard; import de.anomic.server.serverMemory;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
+import de.anomic.yacy.yacyPeerSelection;
import de.anomic.yacy.yacySeed;
public final class IndexTransfer_p {
@@ -111,7 +112,7 @@ public final class IndexTransfer_p { yacySeed seed;
int hc = 0;
if ((sb.webIndex.seedDB != null) && (sb.webIndex.seedDB.sizeConnected() > 0)) {
- final Iterator<yacySeed> e = sb.webIndex.peerActions.dhtAction.getAcceptRemoteIndexSeeds("------------");
+ final Iterator<yacySeed> e = yacyPeerSelection.getAcceptRemoteIndexSeeds(sb.webIndex.seedDB, "AAAAAAAAAAAA", sb.webIndex.seedDB.sizeConnected());
final TreeMap<String, String> hostList = new TreeMap<String, String>();
while (e.hasNext()) {
seed = e.next();
diff --git a/htroot/rct_p.java b/htroot/rct_p.java index 777614c60..034551b72 100644 --- a/htroot/rct_p.java +++ b/htroot/rct_p.java @@ -38,6 +38,7 @@ import de.anomic.server.serverSwitch; import de.anomic.xml.RSSFeed; import de.anomic.xml.RSSMessage; import de.anomic.yacy.yacyClient; +import de.anomic.yacy.yacyPeerSelection; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyURL; @@ -52,7 +53,7 @@ public class rct_p { if (post.containsKey("retrieve")) { final String peerhash = post.get("peer", null); final yacySeed seed = (peerhash == null) ? null : sb.webIndex.seedDB.getConnected(peerhash); - final RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(sb.webIndex.seedDB, seed, 10); + final RSSFeed feed = (seed == null) ? null : yacyClient.queryRemoteCrawlURLs(sb.webIndex.seedDB, seed, 20, 10000); if (feed != null) { for (final RSSMessage item: feed) { //System.out.println("URL=" + item.getLink() + ", desc=" + item.getDescription() + ", pubDate=" + item.getPubDate()); @@ -113,7 +114,7 @@ public class rct_p { yacySeed seed; int hc = 0; if (sb.webIndex.seedDB != null && sb.webIndex.seedDB.sizeConnected() > 0) { - final Iterator<yacySeed> e = sb.webIndex.peerActions.dhtAction.getProvidesRemoteCrawlURLs(); + final Iterator<yacySeed> e = yacyPeerSelection.getProvidesRemoteCrawlURLs(sb.webIndex.seedDB); while (e.hasNext()) { seed = e.next(); if (seed != null) { diff --git a/htroot/yacy/hello.java b/htroot/yacy/hello.java index 8f7e68c1b..c7466fa9a 100644 --- a/htroot/yacy/hello.java +++ b/htroot/yacy/hello.java @@ -41,6 +41,7 @@ import de.anomic.server.serverSwitch; import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNetwork;
+import de.anomic.yacy.yacyPeerSelection;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyVersion;
@@ -183,7 +184,7 @@ public final class hello { if (count > 100) { count = 100; }
// latest seeds
- final Map<String, yacySeed> ySeeds = sb.webIndex.seedDB.seedsByAge(true, count); // peerhash/yacySeed relation
+ final Map<String, yacySeed> ySeeds = yacyPeerSelection.seedsByAge(sb.webIndex.seedDB, true, count); // peerhash/yacySeed relation
// attach also my own seed
seeds.append("seed0=").append(sb.webIndex.seedDB.mySeed().genSeedStr(key)).append(serverCore.CRLF_STRING);
diff --git a/htroot/yacy/urls.java b/htroot/yacy/urls.java index 32243d102..6c7f2a0fa 100644 --- a/htroot/yacy/urls.java +++ b/htroot/yacy/urls.java @@ -58,11 +58,15 @@ public class urls { if (post.get("call", "").equals("remotecrawl")) { // perform a remote crawl url handover final int stackType = NoticedURL.STACK_TYPE_LIMIT; - int count = Math.min(100, post.getInt("count", 0)); + int maxCount = Math.min(100, post.getInt("count", 10)); + long maxTime = Math.min(20000, Math.max(1000, post.getInt("time", 10000))); + long timeout = System.currentTimeMillis() + maxTime; int c = 0; CrawlEntry entry; yacyURL referrer; - while ((count > 0) && (sb.crawlQueues.noticeURL.stackSize(stackType) > 0)) { + while ((maxCount > 0) && + (System.currentTimeMillis() < timeout) && + (sb.crawlQueues.noticeURL.stackSize(stackType) > 0)) { try { entry = sb.crawlQueues.noticeURL.pop(stackType, false, sb.webIndex.profilesActiveCrawls); } catch (final IOException e) { @@ -92,7 +96,7 @@ public class urls { prop.put("item_" + c + "_pubDate", serverDate.formatShortSecond(entry.appdate())); prop.put("item_" + c + "_guid", entry.url().hash()); c++; - count--; + maxCount--; } prop.put("item", c); prop.putHTML("response", "ok"); diff --git a/source/de/anomic/crawler/CrawlQueues.java b/source/de/anomic/crawler/CrawlQueues.java index 31411dfa3..938726f18 100644 --- a/source/de/anomic/crawler/CrawlQueues.java +++ b/source/de/anomic/crawler/CrawlQueues.java @@ -46,6 +46,7 @@ import de.anomic.server.logging.serverLog; import de.anomic.xml.RSSFeed; import de.anomic.xml.RSSMessage; import de.anomic.yacy.yacyClient; +import de.anomic.yacy.yacyPeerSelection; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyURL; @@ -318,28 +319,25 @@ public class CrawlQueues { return false; } - if (remoteTriggeredCrawlJobSize() > 0) { + if (remoteTriggeredCrawlJobSize() > 100) { if (this.log.isFine()) log.logFine("remoteCrawlLoaderJob: the remote-triggered crawl job queue is filled, omitting processing"); return false; } - /* if (coreCrawlJobSize() > 0) { if (this.log.isFine()) log.logFine("remoteCrawlLoaderJob: a local crawl is running, omitting processing"); return false; } - */ // check if we have an entry in the provider list, otherwise fill the list yacySeed seed; if (remoteCrawlProviderHashes.size() == 0) { if (sb.webIndex.seedDB != null && sb.webIndex.seedDB.sizeConnected() > 0) { - final Iterator<yacySeed> e = sb.webIndex.peerActions.dhtAction.getProvidesRemoteCrawlURLs(); + final Iterator<yacySeed> e = yacyPeerSelection.getProvidesRemoteCrawlURLs(sb.webIndex.seedDB); while (e.hasNext()) { seed = e.next(); if (seed != null) { remoteCrawlProviderHashes.add(seed.hash); - } } } @@ -363,7 +361,7 @@ public class CrawlQueues { if (seed == null) return false; // we know a peer which should provide remote crawl entries. load them now. - final RSSFeed feed = yacyClient.queryRemoteCrawlURLs(sb.webIndex.seedDB, seed, 20); + final RSSFeed feed = yacyClient.queryRemoteCrawlURLs(sb.webIndex.seedDB, seed, 30, 5000); if (feed == null) return true; // parse the rss yacyURL url, referrer; diff --git a/source/de/anomic/plasma/plasmaDHTChunk.java b/source/de/anomic/plasma/plasmaDHTChunk.java index 79ff39f87..55434faf5 100644 --- a/source/de/anomic/plasma/plasmaDHTChunk.java +++ b/source/de/anomic/plasma/plasmaDHTChunk.java @@ -51,8 +51,6 @@ public class plasmaDHTChunk { public static final int chunkStatus_INTERRUPTED = 3;
public static final int chunkStatus_COMPLETE = 4;
- public static final int peerRedundancy = 3;
-
private plasmaWordIndex wordIndex;
private serverLog log;
@@ -107,11 +105,11 @@ public class plasmaDHTChunk { return this.status;
}
- public plasmaDHTChunk(final serverLog log, final plasmaWordIndex wordIndex, final int minCount, final int maxCount, final int maxtime) {
+ public plasmaDHTChunk(final serverLog log, final plasmaWordIndex wordIndex, final int minCount, final int maxCount, final int maxtime, String startPointHash) {
try {
this.log = log;
this.wordIndex = wordIndex;
- this.startPointHash = selectTransferStart();
+ this.startPointHash = startPointHash;
if (this.log.isFine()) log.logFine("Selected hash " + this.startPointHash + " as start point for index distribution, distance = " + yacySeed.dhtDistance(this.startPointHash, wordIndex.seedDB.mySeed()));
selectTransferContainers(this.startPointHash, minCount, maxCount, maxtime);
@@ -126,25 +124,7 @@ public class plasmaDHTChunk { }
}
- public plasmaDHTChunk(final serverLog log, final plasmaWordIndex wordIndex, final int minCount, final int maxCount, final int maxtime, final String startHash) {
- try {
- this.log = log;
- this.wordIndex = wordIndex;
- if (this.log.isFine()) log.logFine("Demanded hash " + startHash + " as start point for index distribution, distance = " + yacySeed.dhtDistance(this.startPointHash, wordIndex.seedDB.mySeed()));
- selectTransferContainers(startHash, minCount, maxCount, maxtime);
-
- // count the indexes, can be smaller as expected
- this.idxCount = indexCounter();
- if (this.idxCount < minCount) {
- if (this.log.isFine()) log.logFine("Too few (" + this.idxCount + ") indexes selected for transfer.");
- this.status = chunkStatus_FAILED;
- }
- } catch (final InterruptedException e) {
- this.status = chunkStatus_INTERRUPTED;
- }
- }
-
- private String selectTransferStart() {
+ public static String selectTransferStart() {
return kelondroBase64Order.enhancedCoder.encode(serverCodings.encodeMD5Raw(Long.toString(System.currentTimeMillis()))).substring(2, 2 + yacySeedDB.commonHashLength);
}
@@ -178,7 +158,7 @@ public class plasmaDHTChunk { int wholesize;
urlCache = new HashMap<String, indexURLReference>();
- final long maximumDistanceLong = Long.MAX_VALUE / wordIndex.seedDB.sizeConnected() * peerRedundancy * 2;
+ final long maximumDistanceLong = Long.MAX_VALUE / wordIndex.seedDB.sizeConnected() * wordIndex.netRedundancy * 2;
final long timeout = (maxtime < 0) ? Long.MAX_VALUE : System.currentTimeMillis() + maxtime;
while (
(maxcount > refcount) &&
diff --git a/source/de/anomic/plasma/plasmaSearchAPI.java b/source/de/anomic/plasma/plasmaSearchAPI.java index 320a94ce3..5634bd07d 100644 --- a/source/de/anomic/plasma/plasmaSearchAPI.java +++ b/source/de/anomic/plasma/plasmaSearchAPI.java @@ -39,7 +39,7 @@ import de.anomic.index.indexURLReference; import de.anomic.kelondro.kelondroBitfield; import de.anomic.server.serverDate; import de.anomic.server.serverObjects; -import de.anomic.yacy.yacyPeerActions; +import de.anomic.yacy.yacyPeerSelection; import de.anomic.yacy.yacySeed; import de.anomic.yacy.yacyURL; @@ -68,12 +68,12 @@ public class plasmaSearchAPI { return b; } - public static void listHosts(final serverObjects prop, final String startHash, final yacyPeerActions peerActions) { + public static void listHosts(final serverObjects prop, final String startHash, final plasmaSwitchboard sb) { // list known hosts yacySeed seed; int hc = 0; prop.put("searchresult_keyhash", startHash); - final Iterator<yacySeed> e = peerActions.dhtAction.getAcceptRemoteIndexSeeds(startHash); + final Iterator<yacySeed> e = yacyPeerSelection.getAcceptRemoteIndexSeeds(sb.webIndex.seedDB, startHash, sb.webIndex.seedDB.sizeConnected()); while (e.hasNext()) { seed = e.next(); if (seed != null) { diff --git a/source/de/anomic/plasma/plasmaSearchEvent.java b/source/de/anomic/plasma/plasmaSearchEvent.java index 6e6e1ed33..d46efb4df 100644 --- a/source/de/anomic/plasma/plasmaSearchEvent.java +++ b/source/de/anomic/plasma/plasmaSearchEvent.java @@ -130,7 +130,7 @@ public final class plasmaSearchEvent { // do a global search
this.rankedCache = new plasmaSearchRankingProcess(wordIndex, query, max_results_preparation, 16);
- final int fetchpeers = 10;
+ final int fetchpeers = 12;
// the result of the fetch is then in the rcGlobal
final long timer = System.currentTimeMillis();
diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index e35bbd8e7..f53c6ccf5 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -170,6 +170,7 @@ import de.anomic.yacy.yacyClient; import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyNewsRecord;
+import de.anomic.yacy.yacyPeerSelection;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacyTray;
import de.anomic.yacy.yacyURL;
@@ -305,7 +306,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack. log.logConfig("Starting Indexing Management");
final String networkName = getConfig(plasmaSwitchboardConstants.NETWORK_NAME, "");
final boolean useCommons = getConfigBool("index.storeCommons", false);
- webIndex = new plasmaWordIndex(networkName, log, indexPrimaryPath, indexSecondaryPath, wordCacheMaxCount, useCommons);
+ final int redundancy = (int) sb.getConfigLong("network.unit.dhtredundancy.senior", 1);
+ webIndex = new plasmaWordIndex(networkName, log, indexPrimaryPath, indexSecondaryPath, wordCacheMaxCount, useCommons, redundancy);
crawlResults = new ResultURLs();
// start yacy core
@@ -740,7 +742,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack. final File indexSecondaryPath = (getConfig(plasmaSwitchboardConstants.INDEX_SECONDARY_PATH, "").length() == 0) ? indexPrimaryPath : new File(getConfig(plasmaSwitchboardConstants.INDEX_SECONDARY_PATH, ""));
final int wordCacheMaxCount = (int) getConfigLong(plasmaSwitchboardConstants.WORDCACHE_MAX_COUNT, 20000);
final boolean useCommons = getConfigBool("index.storeCommons", false);
- this.webIndex = new plasmaWordIndex(getConfig(plasmaSwitchboardConstants.NETWORK_NAME, ""), getLog(), indexPrimaryPath, indexSecondaryPath, wordCacheMaxCount, useCommons);
+ final int redundancy = (int) sb.getConfigLong("network.unit.dhtredundancy.senior", 1);
+ this.webIndex = new plasmaWordIndex(getConfig(plasmaSwitchboardConstants.NETWORK_NAME, ""), getLog(), indexPrimaryPath, indexSecondaryPath, wordCacheMaxCount, useCommons, redundancy);
}
// start up crawl jobs
continueCrawlJob(plasmaSwitchboardConstants.CRAWLJOB_LOCAL_CRAWL);
@@ -1192,7 +1195,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack. )) {
// generate new chunk
final int minChunkSize = (int) getConfigLong(plasmaSwitchboardConstants.INDEX_DIST_CHUNK_SIZE_MIN, 30);
- dhtTransferChunk = new plasmaDHTChunk(this.log, webIndex, minChunkSize, dhtTransferIndexCount, 5000);
+ dhtTransferChunk = new plasmaDHTChunk(this.log, webIndex, minChunkSize, dhtTransferIndexCount, 5000, plasmaDHTChunk.selectTransferStart());
doneSomething = true;
}
@@ -2005,11 +2008,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack. try {
// find a list of DHT-peers
- final ArrayList<yacySeed> seeds = webIndex.peerActions.dhtAction.getDHTTargets(webIndex.seedDB, log, peerCount, 9, dhtChunk.firstContainer().getWordHash(), dhtChunk.lastContainer().getWordHash());
- if (seeds.size() < peerCount) {
- log.logWarning("found not enough (" + seeds.size() + ") peers for distribution for dhtchunk [" + dhtChunk.firstContainer().getWordHash() + " .. " + dhtChunk.lastContainer().getWordHash() + "]");
- return false;
- }
+ if (log != null) log.logInfo("Collecting DHT target peers for first_hash = " + dhtChunk.firstContainer().getWordHash() + ", last_hash = " + dhtChunk.lastContainer().getWordHash());
+ final Iterator<yacySeed> seedIter = yacyPeerSelection.getAcceptRemoteIndexSeeds(webIndex.seedDB, dhtChunk.lastContainer().getWordHash(), peerCount + 9);
// send away the indexes to all these peers
int hc1 = 0;
@@ -2019,8 +2019,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack. final int timeout = (int)getConfigLong(plasmaSwitchboardConstants.INDEX_DIST_TIMEOUT, 60000);
final int retries = 0;
- // starting up multiple DHT transfer threads
- final Iterator<yacySeed> seedIter = seeds.iterator();
+ // starting up multiple DHT transfer threads
+ yacySeed seed;
+ long firstdist, lastdist;
final ArrayList<plasmaDHTTransfer> transfer = new ArrayList<plasmaDHTTransfer>(peerCount);
while (hc1 < peerCount && (transfer.size() > 0 || seedIter.hasNext())) {
@@ -2031,7 +2032,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch<IndexingStack. checkInterruption();
if (seedIter.hasNext()) {
- final plasmaDHTTransfer t = new plasmaDHTTransfer(log, webIndex.seedDB, webIndex.peerActions, seedIter.next(), dhtChunk,gzipBody,timeout,retries);
+ seed = seedIter.next();
+ firstdist = yacySeed.dhtDistance(dhtChunk.firstContainer().getWordHash(), seed);
+ lastdist = yacySeed.dhtDistance(dhtChunk.lastContainer().getWordHash(), seed);
+ if (log != null) log.logInfo("Selected DHT target peer " + seed.getName() + ":" + seed.hash + ", distance2first = " + firstdist + ", distance2last = " + lastdist);
+ final plasmaDHTTransfer t = new plasmaDHTTransfer(log, webIndex.seedDB, webIndex.peerActions, seed, dhtChunk, gzipBody, timeout, retries);
t.start();
transfer.add(t);
} else {
diff --git a/source/de/anomic/plasma/plasmaWordIndex.java b/source/de/anomic/plasma/plasmaWordIndex.java index 0720402d0..e19952a6a 100644 --- a/source/de/anomic/plasma/plasmaWordIndex.java +++ b/source/de/anomic/plasma/plasmaWordIndex.java @@ -67,9 +67,9 @@ import de.anomic.server.logging.serverLog; import de.anomic.tools.iso639;
import de.anomic.xml.RSSFeed;
import de.anomic.xml.RSSMessage;
-import de.anomic.yacy.yacyDHTAction;
import de.anomic.yacy.yacyNewsPool;
import de.anomic.yacy.yacyPeerActions;
+import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.yacyURL;
@@ -114,13 +114,15 @@ public final class plasmaWordIndex implements indexRI { public CrawlProfile.entry defaultMediaSnippetLocalProfile, defaultMediaSnippetGlobalProfile;
private final File queuesRoot;
public yacyPeerActions peerActions;
+ public int netRedundancy;
- public plasmaWordIndex(final String networkName, final serverLog log, final File indexPrimaryRoot, final File indexSecondaryRoot, final int entityCacheMaxSize, boolean useCommons) {
+ public plasmaWordIndex(final String networkName, final serverLog log, final File indexPrimaryRoot, final File indexSecondaryRoot, final int entityCacheMaxSize, boolean useCommons, int redundancy) {
if (networkName == null || networkName.length() == 0) {
log.logSevere("no network name given - shutting down");
System.exit(0);
}
this.log = log;
+ this.netRedundancy = redundancy;
this.primaryRoot = new File(indexPrimaryRoot, networkName);
this.secondaryRoot = new File(indexSecondaryRoot, networkName);
File indexPrimaryTextLocation = new File(this.primaryRoot, "TEXT");
@@ -494,7 +496,7 @@ public final class plasmaWordIndex implements indexRI { public void addEntry(final String wordHash, final indexRWIRowEntry entry, final long updateTime, boolean dhtInCase) {
// set dhtInCase depending on wordHash
- if ((!dhtInCase) && (yacyDHTAction.shallBeOwnWord(seedDB, wordHash))) dhtInCase = true;
+ if ((!dhtInCase) && (yacySeed.shallBeOwnWord(seedDB, wordHash, this.netRedundancy))) dhtInCase = true;
// add the entry
if (dhtInCase) {
@@ -514,7 +516,7 @@ public final class plasmaWordIndex implements indexRI { assert (entries.row().objectsize == indexRWIRowEntry.urlEntryRow.objectsize);
// set dhtInCase depending on wordHash
- if ((!dhtInCase) && (yacyDHTAction.shallBeOwnWord(seedDB, entries.getWordHash()))) dhtInCase = true;
+ if ((!dhtInCase) && (yacySeed.shallBeOwnWord(seedDB, entries.getWordHash(), this.netRedundancy))) dhtInCase = true;
// add the entry
if (dhtInCase) {
diff --git a/source/de/anomic/yacy/yacyClient.java b/source/de/anomic/yacy/yacyClient.java index 970e6ea0d..862684830 100644 --- a/source/de/anomic/yacy/yacyClient.java +++ b/source/de/anomic/yacy/yacyClient.java @@ -377,7 +377,7 @@ public final class yacyClient { }
}
- public static RSSFeed queryRemoteCrawlURLs(final yacySeedDB seedDB, final yacySeed target, final int count) {
+ public static RSSFeed queryRemoteCrawlURLs(final yacySeedDB seedDB, final yacySeed target, final int maxCount, final long maxTime) {
// returns a list of
if (target == null) { return null; }
@@ -385,7 +385,8 @@ public final class yacyClient { final String salt = crypt.randomSalt();
final List<Part> post = yacyNetwork.basicRequestPost(plasmaSwitchboard.getSwitchboard(), target.hash, salt);
post.add(new DefaultCharsetStringPart("call", "remotecrawl"));
- post.add(new DefaultCharsetStringPart("count", Integer.toString(count)));
+ post.add(new DefaultCharsetStringPart("count", Integer.toString(maxCount)));
+ post.add(new DefaultCharsetStringPart("time", Long.toString(maxTime)));
// send request
try {
diff --git a/source/de/anomic/yacy/yacyCore.java b/source/de/anomic/yacy/yacyCore.java index f38bbc9ea..53906e317 100644 --- a/source/de/anomic/yacy/yacyCore.java +++ b/source/de/anomic/yacy/yacyCore.java @@ -305,7 +305,7 @@ public class yacyCore { if (sb.webIndex.seedDB.mySeed().get(yacySeed.PEERTYPE, yacySeed.PEERTYPE_VIRGIN).equals(yacySeed.PEERTYPE_VIRGIN)) {
if (attempts > PING_INITIAL) { attempts = PING_INITIAL; }
final Map<String, String> ch = plasmaSwitchboard.getSwitchboard().clusterhashes;
- seeds = sb.webIndex.seedDB.seedsByAge(true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
+ seeds = yacyPeerSelection.seedsByAge(sb.webIndex.seedDB, true, attempts - ((ch == null) ? 0 : ch.size())); // best for fast connection
// add also all peers from cluster if this is a public robinson cluster
if (ch != null) {
final Iterator<Map.Entry<String, String>> i = ch.entrySet().iterator();
@@ -332,7 +332,7 @@ public class yacyCore { } else {
if (attempts > PING_MIN_RUNNING) { attempts = PING_MIN_RUNNING; }
}
- seeds = sb.webIndex.seedDB.seedsByAge(false, attempts); // best for seed list maintenance/cleaning
+ seeds = yacyPeerSelection.seedsByAge(sb.webIndex.seedDB, false, attempts); // best for seed list maintenance/cleaning
}
if ((seeds == null) || seeds.size() == 0) { return 0; }
diff --git a/source/de/anomic/yacy/yacyDHTAction.java b/source/de/anomic/yacy/yacyDHTAction.java deleted file mode 100644 index ebcfe0b90..000000000 --- a/source/de/anomic/yacy/yacyDHTAction.java +++ /dev/null @@ -1,350 +0,0 @@ -// yacyDHTAction.java
-// -------------------------------------
-// (C) by Michael Peter Christen; mc@yacy.net
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2005
-//
-// $LastChangedDate$
-// $LastChangedRevision$
-// $LastChangedBy$
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-
-package de.anomic.yacy;
-
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import de.anomic.kelondro.kelondroBase64Order;
-import de.anomic.kelondro.kelondroCloneableIterator;
-import de.anomic.kelondro.kelondroCloneableMapIterator;
-import de.anomic.kelondro.kelondroException;
-import de.anomic.kelondro.kelondroMScoreCluster;
-import de.anomic.kelondro.kelondroRotateIterator;
-import de.anomic.server.logging.serverLog;
-
-public class yacyDHTAction {
-
- protected yacySeedDB seedDB;
- protected kelondroMScoreCluster<String> seedCrawlReady;
-
- public yacyDHTAction(final yacySeedDB seedDB) {
- this.seedDB = seedDB;
- this.seedCrawlReady = new kelondroMScoreCluster<String>();
- // init crawl-ready table
- try {
- final Iterator<yacySeed> en = seedDB.seedsConnected(true, false, null, (float) 0.0);
- yacySeed ys;
- while (en.hasNext()) {
- ys = en.next();
- if ((ys != null) && (ys.getVersion() >= ((float) 0.3))) seedCrawlReady.setScore(ys.hash, yacyCore.yacyTime());
- }
- } catch (final IllegalArgumentException e) {
- }
- }
-
- public void close() {
- // the seedDB should be cleared elsewhere
- seedCrawlReady = null;
- }
-
- public Iterator<yacySeed> getDHTSeeds(final boolean up, final String firstHash, final float minVersion) {
- // enumerates seed-type objects: all seeds with starting point in the middle, rotating at the end/beginning
- return new seedDHTEnum(up, firstHash, minVersion);
- }
-
- class seedDHTEnum implements Iterator<yacySeed> {
-
- Iterator<yacySeed> e1, e2;
- boolean up;
- int steps;
- float minVersion;
-
- public seedDHTEnum(final boolean up, final String firstHash, final float minVersion) {
- this.steps = seedDB.sizeConnected();
- this.up = up;
- this.minVersion = minVersion;
- this.e1 = seedDB.seedsConnected(up, false, firstHash, minVersion);
- this.e2 = null;
- }
-
- public boolean hasNext() {
- return (steps > 0) && ((e2 == null) || (e2.hasNext()));
- }
-
- public yacySeed next() {
- if (steps == 0) return null;
- steps--;
-
- if (e1 == null || !e1.hasNext()) {
- if (e2 == null) {
- e1 = null;
- e2 = seedDB.seedsConnected(up, false, null, minVersion);
- }
- return e2.next();
- }
-
- final yacySeed n = e1.next();
- if (!(e1.hasNext())) {
- e1 = null;
- e2 = seedDB.seedsConnected(up, false, null, minVersion);
- }
- return n;
- }
-
- public void remove() {
- throw new UnsupportedOperationException();
- }
- }
-
- public Iterator<yacySeed> getProvidesRemoteCrawlURLs() {
- return new providesRemoteCrawlURLsEnum();
- }
-
- class providesRemoteCrawlURLsEnum implements Iterator<yacySeed> {
-
- Iterator<yacySeed> se;
- yacySeed nextSeed;
-
- public providesRemoteCrawlURLsEnum() {
- se = getDHTSeeds(true, null, yacyVersion.YACY_POVIDES_REMOTECRAWL_LISTS);
- nextSeed = nextInternal();
- }
-
- public boolean hasNext() {
- return nextSeed != null;
- }
-
- private yacySeed nextInternal() {
- yacySeed s;
- try {
- while (se.hasNext()) {
- s = se.next();
- if (s == null) return null;
- if (s.getLong(yacySeed.RCOUNT, 0) > 0) return s;
- }
- } catch (final kelondroException e) {
- System.out.println("DEBUG providesRemoteCrawlURLsEnum:" + e.getMessage());
- yacyCore.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db.");
- seedDB.resetActiveTable();
- return null;
- }
- return null;
- }
-
- public yacySeed next() {
- final yacySeed next = nextSeed;
- nextSeed = nextInternal();
- return next;
- }
-
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- }
-
- public Iterator<yacySeed> getAcceptRemoteIndexSeeds(final String starthash) {
- // returns an enumeration of yacySeed-Objects
- // that have the AcceptRemoteIndex-Flag set
- // the seeds are enumerated in the right order according DHT
- return new acceptRemoteIndexSeedEnum(starthash);
- }
-
- class acceptRemoteIndexSeedEnum implements Iterator<yacySeed> {
-
- Iterator<yacySeed> se;
- yacySeed nextSeed;
-
- public acceptRemoteIndexSeedEnum(final String starthash) {
- se = getDHTSeeds(true, starthash, yacyVersion.YACY_HANDLES_COLLECTION_INDEX);
- nextSeed = nextInternal();
- }
-
- public boolean hasNext() {
- return nextSeed != null;
- }
-
- private yacySeed nextInternal() {
- yacySeed s;
- try {
- while (se.hasNext()) {
- s = se.next();
- if (s == null) return null;
- if (s.getFlagAcceptRemoteIndex()) return s;
- }
- } catch (final kelondroException e) {
- System.out.println("DEBUG acceptRemoteIndexSeedEnum:" + e.getMessage());
- yacyCore.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db.");
- seedDB.resetActiveTable();
- return null;
- }
- return null;
- }
-
- public yacySeed next() {
- final yacySeed next = nextSeed;
- nextSeed = nextInternal();
- return next;
- }
-
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- }
-
- public Iterator<yacySeed> getAcceptRemoteCrawlSeeds(final String starthash, final boolean available) {
- return new acceptRemoteCrawlSeedEnum(starthash, available);
- }
-
- class acceptRemoteCrawlSeedEnum implements Iterator<yacySeed> {
-
- Iterator<yacySeed> se;
- yacySeed nextSeed;
- boolean available;
-
- public acceptRemoteCrawlSeedEnum(final String starthash, final boolean available) {
- this.se = getDHTSeeds(true, starthash, (float) 0.0);
- this.available = available;
- nextSeed = nextInternal();
- }
-
- public boolean hasNext() {
- return nextSeed != null;
- }
-
- private yacySeed nextInternal() {
- yacySeed s;
- while (se.hasNext()) {
- s = se.next();
- if (s == null) return null;
- s.available = seedCrawlReady.getScore(s.hash);
- if (available) {
- if (seedCrawlReady.getScore(s.hash) < yacyCore.yacyTime()) return s;
- } else {
- if (seedCrawlReady.getScore(s.hash) > yacyCore.yacyTime()) return s;
- }
- }
- return null;
- }
-
- public yacySeed next() {
- final yacySeed next = nextSeed;
- nextSeed = nextInternal();
- return next;
- }
-
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- }
-
- public synchronized yacySeed getGlobalCrawlSeed(final String urlHash) {
- Iterator<yacySeed> e = getAcceptRemoteCrawlSeeds(urlHash, true);
- yacySeed seed;
- if (e.hasNext()) seed = e.next(); else seed = null;
- e = null;
- return seed;
- }
-
- public synchronized yacySeed getPublicClusterCrawlSeed(final String urlHash, final TreeMap<String, String> clusterhashes) {
- // clusterhashes is a String(hash)/String(IP) - mapping
- final kelondroCloneableIterator<String> i = new kelondroRotateIterator<String>(new kelondroCloneableMapIterator<String>(clusterhashes, urlHash), null, clusterhashes.size());
- String hash;
- int count = clusterhashes.size(); // counter to ensure termination
- while ((i.hasNext()) && (count-- > 0)) {
- hash = i.next();
- final yacySeed seed = seedDB.getConnected(hash);
- if (seed == null) continue;
- seed.setAlternativeAddress(clusterhashes.get(hash));
- return seed;
- }
- return null;
- }
-
- public void setCrawlTime(final String seedHash, int newYacyTime) {
- if (newYacyTime < yacyCore.yacyTime()) newYacyTime = yacyCore.yacyTime();
- seedCrawlReady.setScore(seedHash, newYacyTime);
- }
-
- public void setCrawlDelay(final String seedHash, final int newDelay) {
- seedCrawlReady.setScore(seedHash, yacyCore.yacyTime() + newDelay);
- }
-
- public void processPeerArrival(final yacySeed peer, final boolean direct) {
- if (peer.getVersion() >= ((float) 0.3)) {
- if (!(seedCrawlReady.existsScore(peer.hash))) seedCrawlReady.setScore(peer.hash, yacyCore.yacyTime());
- } else {
- seedCrawlReady.deleteScore(peer.hash);
- }
- }
-
- public void processPeerDeparture(final yacySeed peer) {
- seedCrawlReady.deleteScore(peer.hash);
- }
-
- public static boolean shallBeOwnWord(final yacySeedDB seedDB, final String wordhash) {
- if (seedDB == null) return false;
- if (seedDB.mySeed().isPotential()) return false;
- final long distance = yacySeed.dhtDistance(wordhash, seedDB.mySeed());
- final long max = Long.MAX_VALUE / seedDB.sizeConnected() * 2;
- //System.out.println("Distance for " + wordhash + ": " + distance + "; max is " + max);
- return (distance > 0) && (distance <= max);
- }
-
- public synchronized ArrayList<yacySeed> getDHTTargets(final yacySeedDB seedDB, final serverLog log, final int primaryPeerCount, final int reservePeerCount, final String firstKey, final String lastKey) {
- // find a list of DHT-peers
- assert firstKey != null;
- assert lastKey != null;
- assert seedDB != null;
- assert seedDB.mySeed() != null;
- assert seedDB.mySeed().hash != null;
- /*
- assert
- !(kelondroBase64Order.enhancedCoder.cardinal(firstKey.getBytes()) < kelondroBase64Order.enhancedCoder.cardinal(yacyCore.seedDB.mySeed.hash.getBytes()) &&
- kelondroBase64Order.enhancedCoder.cardinal(lastKey.getBytes()) > kelondroBase64Order.enhancedCoder.cardinal(yacyCore.seedDB.mySeed.hash.getBytes()));
- */
- final ArrayList<yacySeed> seeds = new ArrayList<yacySeed>();
- yacySeed seed;
- //double ownDistance = Math.min(yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, firstKey), yacyDHTAction.dhtDistance(yacyCore.seedDB.mySeed.hash, lastKey));
- //double maxDistance = Math.min(ownDistance, maxDist);
-
- long firstdist, lastdist;
- Iterator<yacySeed> e = this.getAcceptRemoteIndexSeeds(lastKey);
- final TreeSet<String> doublecheck = new TreeSet<String>(kelondroBase64Order.enhancedComparator);
- int maxloop = Math.min(100, seedDB.sizeConnected()); // to ensure termination
- if (log != null) log.logInfo("Collecting DHT target peers for first_hash = " + firstKey + ", last_hash = " + lastKey);
- while ((e.hasNext()) && (seeds.size() < (primaryPeerCount + reservePeerCount)) && (maxloop-- > 0)) {
- seed = e.next();
- if (seed == null || seed.hash == null) continue;
- firstdist = yacySeed.dhtDistance(firstKey, seed);
- lastdist = yacySeed.dhtDistance(lastKey, seed);
- if (doublecheck.contains(seed.hash)) {
- if (log != null && yacyCore.log.isFine()) log.logFine("Discarded double DHT target peer " + seed.getName() + ":" + seed.hash + ", distance2first = " + firstdist + ", distance2last = " + lastdist);
- } else {
- if (log != null) log.logInfo("Selected " + ((seeds.size() < primaryPeerCount) ? "primary" : "reserve") + " DHT target peer " + seed.getName() + ":" + seed.hash + ", distance2first = " + firstdist + ", distance2last = " + lastdist);
- seeds.add(seed);
- doublecheck.add(seed.hash);
- }
- }
- e = null; // finish enumeration
-
- return seeds;
- }
-}
diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index 69ad2ddb3..a8460650d 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -1,7 +1,7 @@ // yacyPeerActions.java
// -------------------------------------
// (C) by Michael Peter Christen; mc@yacy.net
-// first published on http://www.anomic.de
+// first published on http://yacy.net
// Frankfurt, Germany, 2005
//
// $LastChangedDate$
@@ -38,7 +38,6 @@ public class yacyPeerActions { private final yacySeedDB seedDB;
private HashMap<String, String> userAgents;
public long disconnects;
- public yacyDHTAction dhtAction;
private final yacyNewsPool newsPool;
public yacyPeerActions(final yacySeedDB seedDB, final yacyNewsPool newsPool) {
@@ -46,14 +45,12 @@ public class yacyPeerActions { this.newsPool = newsPool;
this.userAgents = new HashMap<String, String>();
this.disconnects = 0;
- this.dhtAction = new yacyDHTAction(seedDB);
}
public void close() {
// the seedDB and newsPool should be cleared elsewhere
if (userAgents != null) userAgents.clear();
userAgents = null;
- if (dhtAction != null) dhtAction.close();
}
public synchronized boolean connectPeer(final yacySeed seed, final boolean direct) {
@@ -198,7 +195,6 @@ public class yacyPeerActions { final boolean res = connectPeer(peer, direct);
if (res) {
// perform all actions if peer is effective new
- dhtAction.processPeerArrival(peer, direct);
this.processPeerArrival(peer, direct);
RSSFeed.channels(RSSFeed.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " joined the network", "", ""));
}
@@ -214,8 +210,6 @@ public class yacyPeerActions { peer.put("dct", Long.toString(System.currentTimeMillis()));
seedDB.addDisconnected(peer); // update info
}
- // perform all actions
- dhtAction.processPeerDeparture(peer);
RSSFeed.channels(RSSFeed.PEERNEWS).addMessage(new RSSMessage(peer.getName() + " left the network", "", ""));
}
@@ -259,4 +253,5 @@ public class yacyPeerActions { final String userAgent = userAgents.get(IP);
return (userAgent == null) ? "" : userAgent;
}
+
}
diff --git a/source/de/anomic/yacy/yacyPeerSelection.java b/source/de/anomic/yacy/yacyPeerSelection.java new file mode 100644 index 000000000..368516999 --- /dev/null +++ b/source/de/anomic/yacy/yacyPeerSelection.java @@ -0,0 +1,276 @@ +// yacyPeerSelection.java +// ------------------------------------- +// (C) by Michael Peter Christen; mc@yacy.net +// first published 05.11.2008 on http://yacy.net +// Frankfurt, Germany, 2008 +// +// $LastChangedDate: 2008-09-03 02:30:21 +0200 (Mi, 03 Sep 2008) $ +// $LastChangedRevision: 5102 $ +// $LastChangedBy: orbiter $ +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.yacy; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; + +import de.anomic.kelondro.kelondroException; +import de.anomic.kelondro.kelondroMScoreCluster; +import de.anomic.server.serverDate; +import de.anomic.server.logging.serverLog; + + +/* + * this package is a collection of peer selection iterations that had been + * part of yacyPeerActions, yacyDHTActions and yacySeedDB + */ + +public class yacyPeerSelection { + + public static void selectDHTPositions(final yacySeedDB seedDB, String wordhash, int redundancy, HashMap<String, yacySeed> regularSeeds, kelondroMScoreCluster<String> ranking) { + // this method is called from the search target computation + long[] dhtVerticalTargets = yacySeed.dhtPositions(wordhash, yacySeed.partitionExponent); + yacySeed seed; + long distance; + for (int v = 0; v < dhtVerticalTargets.length; v++) { + wordhash = yacySeed.positionToHash(dhtVerticalTargets[v]); + Iterator<yacySeed> dhtEnum = getAcceptRemoteIndexSeeds(seedDB, wordhash, redundancy); + int c = Math.min(seedDB.sizeConnected(), redundancy); + int cc = 3; // select a maximum of 3, this is enough redundancy + while (dhtEnum.hasNext() && c > 0 && cc-- > 0) { + seed = dhtEnum.next(); + if (seed == null || seed.hash == null) continue; + distance = yacySeed.dhtDistance(wordhash, seed); + if (!seed.getFlagAcceptRemoteIndex()) continue; // probably a robinson peer + if (serverLog.isFine("PLASMA")) serverLog.logFine("PLASMA", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/" + distance + " for wordhash " + wordhash + ", score " + c); + ranking.addScore(seed.hash, 2 * c); + regularSeeds.put(seed.hash, seed); + c--; + } + } + } + + public static boolean verifyIfOwnWord(final yacySeedDB seedDB, final String wordhash, int redundancy) { + String myHash = seedDB.mySeed().hash; + long[] dhtVerticalTargets = yacySeed.dhtPositions(wordhash, yacySeed.partitionExponent); + for (int v = 0; v < dhtVerticalTargets.length; v++) { + Iterator<yacySeed> dhtEnum = getAcceptRemoteIndexSeeds(seedDB, yacySeed.positionToHash(dhtVerticalTargets[v]), redundancy); + while (dhtEnum.hasNext()) { + if (dhtEnum.next().equals(myHash)) return true; + } + } + return false; + } + + public static Iterator<yacySeed> getAcceptRemoteIndexSeeds(yacySeedDB seedDB, final String starthash, int max) { + // returns an enumeration of yacySeed-Objects + // that have the AcceptRemoteIndex-Flag set + // the seeds are enumerated in the right order according DHT + return new acceptRemoteIndexSeedEnum(seedDB, starthash, Math.max(max, seedDB.sizeConnected())); + } + + private static class acceptRemoteIndexSeedEnum implements Iterator<yacySeed> { + + private Iterator<yacySeed> se; + private yacySeed nextSeed; + private yacySeedDB seedDB; + private HashSet<String> doublecheck; + private int remaining; + + public acceptRemoteIndexSeedEnum(yacySeedDB seedDB, final String starthash, int max) { + this.seedDB = seedDB; + this.se = getDHTSeeds(seedDB, starthash, yacyVersion.YACY_HANDLES_COLLECTION_INDEX); + this.nextSeed = nextInternal(); + this.doublecheck = new HashSet<String>(); + this.remaining = max; + } + + public boolean hasNext() { + return nextSeed != null; + } + + private yacySeed nextInternal() { + if (this.remaining <= 0) return null; + yacySeed s; + try { + while (se.hasNext()) { + s = se.next(); + if (s == null) return null; + if (doublecheck.contains(s.hash)) return null; + this.doublecheck.add(s.hash); + if (s.getFlagAcceptRemoteIndex()) { + this.remaining--; + return s; + } + } + } catch (final kelondroException e) { + System.out.println("DEBUG acceptRemoteIndexSeedEnum:" + e.getMessage()); + yacyCore.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db."); + seedDB.resetActiveTable(); + return null; + } + return null; + } + + public yacySeed next() { + final yacySeed next = nextSeed; + nextSeed = nextInternal(); + return next; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + + } + + public static Iterator<yacySeed> getDHTSeeds(yacySeedDB seedDB, final String firstHash, final float minVersion) { + // enumerates seed-type objects: all seeds with starting point in the middle, rotating at the end/beginning + return new seedDHTEnum(seedDB, firstHash, minVersion); + } + + private static class seedDHTEnum implements Iterator<yacySeed> { + + Iterator<yacySeed> e1, e2; + int steps; + float minVersion; + yacySeedDB seedDB; + + public seedDHTEnum(yacySeedDB seedDB, final String firstHash, final float minVersion) { + this.seedDB = seedDB; + this.steps = seedDB.sizeConnected(); + this.minVersion = minVersion; + this.e1 = seedDB.seedsConnected(true, false, firstHash, minVersion); + this.e2 = null; + } + + public boolean hasNext() { + return (steps > 0) && ((e2 == null) || (e2.hasNext())); + } + + public yacySeed next() { + if (steps == 0) return null; + steps--; + + if (e1 == null || !e1.hasNext()) { + if (e2 == null) { + e1 = null; + e2 = seedDB.seedsConnected(true, false, null, minVersion); + } + return e2.next(); + } + + final yacySeed n = e1.next(); + if (!(e1.hasNext())) { + e1 = null; + e2 = seedDB.seedsConnected(true, false, null, minVersion); + } + return n; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + } + + public static Iterator<yacySeed> getProvidesRemoteCrawlURLs(yacySeedDB seedDB) { + return new providesRemoteCrawlURLsEnum(seedDB); + } + + private static class providesRemoteCrawlURLsEnum implements Iterator<yacySeed> { + + Iterator<yacySeed> se; + yacySeed nextSeed; + yacySeedDB seedDB; + + public providesRemoteCrawlURLsEnum(yacySeedDB seedDB) { + this.seedDB = seedDB; + se = getDHTSeeds(seedDB, null, yacyVersion.YACY_POVIDES_REMOTECRAWL_LISTS); + nextSeed = nextInternal(); + } + + public boolean hasNext() { + return nextSeed != null; + } + + private yacySeed nextInternal() { + yacySeed s; + try { + while (se.hasNext()) { + s = se.next(); + if (s == null) return null; + if (s.getLong(yacySeed.RCOUNT, 0) > 0) return s; + } + } catch (final kelondroException e) { + System.out.println("DEBUG providesRemoteCrawlURLsEnum:" + e.getMessage()); + yacyCore.log.logSevere("database inconsistency (" + e.getMessage() + "), re-set of db."); + seedDB.resetActiveTable(); + return null; + } + return null; + } + + public yacySeed next() { + final yacySeed next = nextSeed; + nextSeed = nextInternal(); + return next; + } + + public void remove() { + throw new UnsupportedOperationException(); + } + + } + + public static HashMap<String, yacySeed> seedsByAge(yacySeedDB seedDB, final boolean up, int count) { + // returns a peerhash/yacySeed relation + // to get most recent peers, set up = true; for oldest peers, set up = false + + if (count > seedDB.sizeConnected()) count = seedDB.sizeConnected(); + + // fill a score object + final kelondroMScoreCluster<String> seedScore = new kelondroMScoreCluster<String>(); + yacySeed ys; + long absage; + final Iterator<yacySeed> s = seedDB.seedsConnected(true, false, null, (float) 0.0); + int searchcount = 1000; + if (searchcount > seedDB.sizeConnected()) searchcount = seedDB.sizeConnected(); + try { + while ((s.hasNext()) && (searchcount-- > 0)) { + ys = s.next(); + if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try { + absage = Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC()); + seedScore.addScore(ys.hash, (int) absage); // the higher absage, the older is the peer + } catch (final Exception e) {} + } + + // result is now in the score object; create a result vector + final HashMap<String, yacySeed> result = new HashMap<String, yacySeed>(); + final Iterator<String> it = seedScore.scores(up); + int c = 0; + while ((c < count) && (it.hasNext())) { + c++; + ys = seedDB.getConnected(it.next()); + if ((ys != null) && (ys.hash != null)) result.put(ys.hash, ys); + } + return result; + } catch (final kelondroException e) { + yacyCore.log.logSevere("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e); + return null; + } + } +} diff --git a/source/de/anomic/yacy/yacySearch.java b/source/de/anomic/yacy/yacySearch.java index 8f2bc0a96..b049111a5 100644 --- a/source/de/anomic/yacy/yacySearch.java +++ b/source/de/anomic/yacy/yacySearch.java @@ -169,7 +169,7 @@ public class yacySearch extends Thread { //return (yacySeed[]) l.toArray();
}
- private static yacySeed[] selectSearchTargets(final yacySeedDB seedDB, final yacyPeerActions peerActions, final Set<String> wordhashes, int seedcount) {
+ private static yacySeed[] selectSearchTargets(final yacySeedDB seedDB, final Set<String> wordhashes, int seedcount, int redundancy) {
// find out a specific number of seeds, that would be relevant for the given word hash(es)
// the result is ordered by relevance: [0] is most relevant
// the seedcount is the maximum number of wanted results
@@ -182,31 +182,16 @@ public class yacySearch extends Thread { final kelondroMScoreCluster<String> ranking = new kelondroMScoreCluster<String>();
final HashMap<String, yacySeed> regularSeeds = new HashMap<String, yacySeed>();
final HashMap<String, yacySeed> matchingSeeds = new HashMap<String, yacySeed>();
- final HashMap<String, yacySeed> robinsonSeeds = new HashMap<String, yacySeed>();
yacySeed seed;
Iterator<yacySeed> dhtEnum;
- int c;
- String wordhash;
- long distance;
Iterator<String> iter = wordhashes.iterator();
while (iter.hasNext()) {
- wordhash = iter.next();
- dhtEnum = peerActions.dhtAction.getDHTSeeds(true, wordhash, (float) 0.0);
- c = seedcount;
- while (dhtEnum.hasNext() && c > 0) {
- seed = dhtEnum.next();
- if (seed == null || seed.hash == null) continue;
- distance = yacySeed.dhtDistance(wordhash, seed);
- if (!seed.getFlagAcceptRemoteIndex()) continue; // probably a robinson peer
- if (serverLog.isFine("PLASMA")) serverLog.logFine("PLASMA", "selectPeers/DHTorder: " + seed.hash + ":" + seed.getName() + "/" + distance + " for wordhash " + wordhash + ", score " + c);
- ranking.addScore(seed.hash, c--);
- regularSeeds.put(seed.hash, seed);
- }
+ yacyPeerSelection.selectDHTPositions(seedDB, iter.next(), redundancy, regularSeeds, ranking);
}
// put in seeds according to size of peer
dhtEnum = seedDB.seedsSortedConnected(false, yacySeed.ICOUNT);
- c = Math.min(seedDB.sizeConnected(), seedcount);
+ int c = Math.min(seedDB.sizeConnected(), seedcount);
int score;
while (dhtEnum.hasNext() && c > 0) {
seed = dhtEnum.next();
@@ -225,37 +210,22 @@ public class yacySearch extends Thread { while (dhtEnum.hasNext()) {
seed = dhtEnum.next();
if (seed == null) continue;
- if (seed.getFlagAcceptRemoteIndex()) {
- // enhance ranking for regular peers
- if (seed.matchPeerTags(wordhashes)) { // access robinson peers with matching tag
- serverLog.logInfo("PLASMA", "selectPeers/PeerTags: " + seed.hash + ":" + seed.getName() + ", is specialized peer for " + seed.getPeerTags().toString());
- regularSeeds.remove(seed.hash);
- ranking.deleteScore(seed.hash);
- matchingSeeds.put(seed.hash, seed);
- }
- if (seed.getAge() < 1) { // the 'workshop feature'
- serverLog.logInfo("PLASMA", "selectPeers/Age: " + seed.hash + ":" + seed.getName() + ", is newbie, age = " + seed.getAge());
- regularSeeds.remove(seed.hash);
- ranking.deleteScore(seed.hash);
- matchingSeeds.put(seed.hash, seed);
- }
- } else {
- // this is a robinson peer
- // in case the peer has more than a million urls, take it as search target
- if (seed.getLinkCount() > 1000000 || seed.matchPeerTags(wordhashes)) {
- regularSeeds.remove(seed.hash);
- ranking.deleteScore(seed.hash);
- if (seed.matchPeerTags(wordhashes))
- matchingSeeds.put(seed.hash, seed);
- else
- robinsonSeeds.put(seed.hash, seed);
- }
+ if (seed.matchPeerTags(wordhashes)) {
+ serverLog.logInfo("PLASMA", "selectPeers/PeerTags: " + seed.hash + ":" + seed.getName() + ", is specialized peer for " + seed.getPeerTags().toString());
+ regularSeeds.remove(seed.hash);
+ ranking.deleteScore(seed.hash);
+ matchingSeeds.put(seed.hash, seed);
+ } else if (seed.getFlagAcceptRemoteIndex() && seed.getAge() < 1) { // the 'workshop feature'
+ serverLog.logInfo("PLASMA", "selectPeers/Age: " + seed.hash + ":" + seed.getName() + ", is newbie, age = " + seed.getAge());
+ regularSeeds.remove(seed.hash);
+ ranking.deleteScore(seed.hash);
+ matchingSeeds.put(seed.hash, seed);
}
}
// evaluate the ranking score and select seeds
seedcount = Math.min(ranking.size(), seedcount);
- final yacySeed[] result = new yacySeed[seedcount + robinsonSeeds.size() + matchingSeeds.size()];
+ final yacySeed[] result = new yacySeed[seedcount + matchingSeeds.size()];
c = 0;
iter = ranking.scores(false); // higher are better
while (iter.hasNext() && c < seedcount) {
@@ -264,11 +234,8 @@ public class yacySearch extends Thread { serverLog.logInfo("PLASMA", "selectPeers/_dht_: " + seed.hash + ":" + seed.getName() + " is choice " + c);
result[c++] = seed;
}
- for (final yacySeed s: robinsonSeeds.values()) {
- serverLog.logInfo("PLASMA", "selectPeers/_robinson_: " + s.hash + ":" + s.getName() + " is choice " + c);
- result[c++] = s;
- }
for (final yacySeed s: matchingSeeds.values()) {
+ s.selectscore = c;
serverLog.logInfo("PLASMA", "selectPeers/_match_: " + s.hash + ":" + s.getName() + " is choice " + c);
result[c++] = s;
}
@@ -295,7 +262,7 @@ public class yacySearch extends Thread { // prepare seed targets and threads
assert language != null;
- final yacySeed[] targetPeers = (clusterselection == null) ? selectSearchTargets(wordIndex.seedDB, wordIndex.peerActions, plasmaSearchQuery.hashes2Set(wordhashes), targets) : selectClusterPeers(wordIndex.seedDB, clusterselection);
+ final yacySeed[] targetPeers = (clusterselection == null) ? selectSearchTargets(wordIndex.seedDB, plasmaSearchQuery.hashes2Set(wordhashes), targets, wordIndex.netRedundancy) : selectClusterPeers(wordIndex.seedDB, clusterselection);
if (targetPeers == null) return new yacySearch[0];
targets = targetPeers.length;
if (targets == 0) return new yacySearch[0];
diff --git a/source/de/anomic/yacy/yacySeed.java b/source/de/anomic/yacy/yacySeed.java index 19d1e62ed..2baf3909a 100644 --- a/source/de/anomic/yacy/yacySeed.java +++ b/source/de/anomic/yacy/yacySeed.java @@ -67,6 +67,7 @@ import de.anomic.tools.crypt; public class yacySeed implements Cloneable {
+ public static final int partitionExponent = 1;
public static final int maxsize = 4096;
/**
* <b>substance</b> "sI" (send index/words)
@@ -731,6 +732,7 @@ public class yacySeed implements Cloneable { }
public final static long dhtPosition(final String wordHash, final String urlHash, final int e) {
+ // this creates 1^^e different positions for the same word hash (according to url hash)
assert wordHash != null;
assert urlHash != null;
if (urlHash == null || e < 1) return dhtPosition(wordHash);
@@ -782,6 +784,10 @@ public class yacySeed implements Cloneable { assert from != null;
final long toPos = dhtPosition(to);
final long fromPos = dhtPosition(from);
+ return dhtDistance(fromPos, toPos);
+ }
+
+ public final static long dhtDistance(final long fromPos, final long toPos) {
final long d = toPos - fromPos;
return (d >= 0) ? d : (d + Long.MAX_VALUE) + 1;
}
@@ -836,7 +842,7 @@ public class yacySeed implements Cloneable { return gaps;
}
- private static String positionToHash(final double t) {
+ static String positionToHash(final double t) {
// transform the position of a peer position into a close peer hash
assert t >= 0.0 : "t = " + t;
assert t < 1.0 : "t = " + t;
@@ -844,7 +850,7 @@ public class yacySeed implements Cloneable { return new String(kelondroBase64Order.enhancedCoder.uncardinal((long) (((double) Long.MAX_VALUE) * t))) + "AA";
}
- private static String positionToHash(final long l) {
+ public static String positionToHash(final long l) {
// transform the position of a peer position into a close peer hash
return new String(kelondroBase64Order.enhancedCoder.uncardinal(l)) + "AA";
@@ -1006,6 +1012,37 @@ public class yacySeed implements Cloneable { }
}
+
+
+ private static int guessedOwn = 0;
+ private static int verifiedOwn = 0;
+
+ public static boolean shallBeOwnWord(final yacySeedDB seedDB, final String wordhash, int redundancy) {
+ if (!guessIfOwnWord(seedDB, wordhash)) return false;
+ guessedOwn++;
+ if (yacyPeerSelection.verifyIfOwnWord(seedDB, wordhash, redundancy)) {
+ verifiedOwn++;
+ System.out.println("*** DEBUG shallBeOwnWord: true. verified/guessed ration = " + verifiedOwn + "/" + guessedOwn);
+ return true;
+ } else {
+ System.out.println("*** DEBUG shallBeOwnWord: false. verified/guessed ration = " + verifiedOwn + "/" + guessedOwn);
+ return false;
+ }
+ }
+
+ private static boolean guessIfOwnWord(final yacySeedDB seedDB, final String wordhash) {
+ if (seedDB == null) return false;
+ if (seedDB.mySeed().isPotential()) return false;
+ final long[] targets = yacySeed.dhtPositions(wordhash, yacySeed.partitionExponent);
+ final long mypos = yacySeed.dhtPosition(seedDB.mySeed().hash);
+ for (int i = 0; i < targets.length; i++) {
+ long distance = yacySeed.dhtDistance(targets[i], mypos);
+ if (distance <= 0) continue;
+ if (distance <= Long.MAX_VALUE / seedDB.sizeConnected() * 2) return true;
+ }
+ return false;
+ }
+
public static void main(String[] args) {
// java -classpath classes de.anomic.yacy.yacySeed hHJBztzcFn76
// java -classpath classes de.anomic.yacy.yacySeed hHJBztzcFG76 M8hgtrHG6g12 3
diff --git a/source/de/anomic/yacy/yacySeedDB.java b/source/de/anomic/yacy/yacySeedDB.java index e8b923d3a..79d5a61bc 100644 --- a/source/de/anomic/yacy/yacySeedDB.java +++ b/source/de/anomic/yacy/yacySeedDB.java @@ -34,7 +34,6 @@ import java.lang.reflect.Method; import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
@@ -50,11 +49,9 @@ import de.anomic.http.httpdAlternativeDomainNames; import de.anomic.kelondro.kelondroBLOBHeap;
import de.anomic.kelondro.kelondroBase64Order;
import de.anomic.kelondro.kelondroException;
-import de.anomic.kelondro.kelondroMScoreCluster;
import de.anomic.kelondro.kelondroMapDataMining;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.server.serverCore;
-import de.anomic.server.serverDate;
import de.anomic.server.serverDomains;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverSwitch;
@@ -368,45 +365,6 @@ public final class yacySeedDB implements httpdAlternativeDomainNames { return e.next();
}
- public HashMap<String, yacySeed> seedsByAge(final boolean up, int count) {
- // returns a peerhash/yacySeed relation
- // to get most recent peers, set up = true; for oldest peers, set up = false
-
- if (count > sizeConnected()) count = sizeConnected();
-
- // fill a score object
- final kelondroMScoreCluster<String> seedScore = new kelondroMScoreCluster<String>();
- yacySeed ys;
- long absage;
- final Iterator<yacySeed> s = seedsConnected(true, false, null, (float) 0.0);
- int searchcount = 1000;
- if (searchcount > sizeConnected()) searchcount = sizeConnected();
- try {
- while ((s.hasNext()) && (searchcount-- > 0)) {
- ys = s.next();
- if ((ys != null) && (ys.get(yacySeed.LASTSEEN, "").length() > 10)) try {
- absage = Math.abs(System.currentTimeMillis() + serverDate.dayMillis - ys.getLastSeenUTC());
- seedScore.addScore(ys.hash, (int) absage); // the higher absage, the older is the peer
- } catch (final Exception e) {}
- }
-
- // result is now in the score object; create a result vector
- final HashMap<String, yacySeed> result = new HashMap<String, yacySeed>();
- final Iterator<String> it = seedScore.scores(up);
- int c = 0;
- while ((c < count) && (it.hasNext())) {
- c++;
- ys = getConnected(it.next());
- if ((ys != null) && (ys.hash != null)) result.put(ys.hash, ys);
- }
- return result;
- } catch (final kelondroException e) {
- seedActiveDB = resetSeedTable(seedActiveDB, seedActiveDBFile);
- if (yacyCore.log.isFine()) yacyCore.log.logFine("Internal Error at yacySeedDB.seedsByAge: " + e.getMessage(), e);
- return null;
- }
- }
-
public int sizeConnected() {
return seedActiveDB.size();
}
diff --git a/source/yacy.java b/source/yacy.java index 27ad1b0e7..674c24c51 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -674,7 +674,7 @@ public final class yacy { final int cacheMem = (int)(serverMemory.max() - serverMemory.total());
if (cacheMem < 2048000) throw new OutOfMemoryError("Not enough memory available to start clean up.");
- final plasmaWordIndex wordIndex = new plasmaWordIndex(networkName, log, indexPrimaryRoot, indexSecondaryRoot, 10000, false);
+ final plasmaWordIndex wordIndex = new plasmaWordIndex(networkName, log, indexPrimaryRoot, indexSecondaryRoot, 10000, false, 1);
final Iterator<indexContainer> indexContainerIterator = wordIndex.wordContainers("AAAAAAAAAAAA", false, false);
long urlCounter = 0, wordCounter = 0;
@@ -865,7 +865,7 @@ public final class yacy { try {
Iterator<indexContainer> indexContainerIterator = null;
if (resource.equals("all")) {
- WordIndex = new plasmaWordIndex("freeworld", log, indexPrimaryRoot, indexSecondaryRoot, 10000, false);
+ WordIndex = new plasmaWordIndex("freeworld", log, indexPrimaryRoot, indexSecondaryRoot, 10000, false, 1);
indexContainerIterator = WordIndex.wordContainers(wordChunkStartHash, false, false);
}
int counter = 0;
|