diff options
48 files changed, 1428 insertions, 2933 deletions
diff --git a/htroot/CrawlProfileEditor_p.java b/htroot/CrawlProfileEditor_p.java index 860d83002..7f3347962 100644 --- a/htroot/CrawlProfileEditor_p.java +++ b/htroot/CrawlProfileEditor_p.java @@ -90,7 +90,7 @@ public class CrawlProfileEditor_p { sb.profilesPassiveCrawls.newEntry(sb.profilesActiveCrawls.getEntry(handle).map()); sb.profilesActiveCrawls.removeEntry(handle); // delete all entries from the crawl queue that are deleted here - sb.noticeURL.removeByProfileHandle(handle); + sb.crawlQueues.noticeURL.removeByProfileHandle(handle); } if (post.containsKey("delete")) { // deletion of a terminated crawl profile diff --git a/htroot/CrawlURLFetchStack_p.java b/htroot/CrawlURLFetchStack_p.java index 3d63260f5..5f319753c 100644 --- a/htroot/CrawlURLFetchStack_p.java +++ b/htroot/CrawlURLFetchStack_p.java @@ -172,15 +172,15 @@ public class CrawlURLFetchStack_p { } } else if (post.containsKey("shiftlcq")) { - final int count = Math.min(post.getInt("shiftloc", 0), sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE)); - final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_CORE, getURLFetcherStack(env), count); + final int count = Math.min(post.getInt("shiftloc", 0), sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE)); + final int failed = shiftFromNotice(sb.crawlQueues.noticeURL, plasmaCrawlNURL.STACK_TYPE_CORE, getURLFetcherStack(env), count); prop.put("shiftloc", "1"); prop.put("shiftloc_value", count - failed); prop.put("shiftloc_failed", failed); } else if (post.containsKey("shiftrcq")) { final int count = post.getInt("shiftrem", 0); - final int failed = shiftFromNotice(sb.noticeURL, plasmaCrawlNURL.STACK_TYPE_LIMIT, getURLFetcherStack(env), count); + final int failed = shiftFromNotice(sb.crawlQueues.noticeURL, plasmaCrawlNURL.STACK_TYPE_LIMIT, getURLFetcherStack(env), count); prop.put("shiftrem", "1"); prop.put("shiftrem_value", count - failed); prop.put("shiftrem_failed", failed); @@ -235,10 +235,10 @@ public class CrawlURLFetchStack_p { prop.put("totalFetched", getURLFetcherStack(env).getPopped()); prop.put("totalAdded", getURLFetcherStack(env).getPushed()); prop.put("maxSize", maxURLsPerFetch); - prop.put("locurls", sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE)); - prop.put("remurls", sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT)); - prop.put("locurlsVal", Math.min(sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE), 500)); - prop.put("remurlsVal", Math.min(sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT), 500)); + prop.put("locurls", sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE)); + prop.put("remurls", sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT)); + prop.put("locurlsVal", Math.min(sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE), 500)); + prop.put("remurlsVal", Math.min(sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT), 500)); return prop; } diff --git a/htroot/CrawlURLFetch_p.java b/htroot/CrawlURLFetch_p.java index 14a830ea3..107703cab 100644 --- a/htroot/CrawlURLFetch_p.java +++ b/htroot/CrawlURLFetch_p.java @@ -465,17 +465,18 @@ public class CrawlURLFetch_p { return getListServletURL(ys.getPublicAddress(), MODE_LIST, this.count, yacyCore.seedDB.mySeed().hash); } - private int stackURLs(String[] urls) throws InterruptedException { + private int stackURLs(ArrayList /*of yacyURL*/ urls) throws InterruptedException { this.lastFailed = 0; this.lastFetchedURLs = 0; this.failed.clear(); if (urls == null) return 0; String reason; - for (int i=0; i<urls.length && !isInterrupted(); i++) { - if (urls[i].trim().length() == 0) continue; - reason = this.sb.sbStackCrawlThread.stackCrawl( - urls[i], + yacyURL url; + for (int i = 0; i < urls.size() && !isInterrupted(); i++) { + url = (yacyURL) urls.get(i); + reason = this.sb.crawlStacker.stackCrawl( + url, null, yacyCore.seedDB.mySeed().hash, null, @@ -483,28 +484,26 @@ public class CrawlURLFetch_p { this.profile.generalDepth(), this.profile); if (reason == null) { - serverLog.logFine(this.getName(), "stacked " + urls[i]); + serverLog.logFine(this.getName(), "stacked " + url); this.lastFetchedURLs++; } else { - serverLog.logFine(this.getName(), "error on stacking " + urls[i] + ": " + reason); + serverLog.logFine(this.getName(), "error on stacking " + url + ": " + reason); this.lastFailed++; totalFailed++; - this.failed.put(urls[i], reason); - try { - plasmaCrawlZURL.Entry ee = this.sb.errorURL.newEntry( - new yacyURL(urls[i], null), + this.failed.put(url, reason); + plasmaCrawlZURL.Entry ee = this.sb.crawlQueues.errorURL.newEntry( + url, reason); - ee.store(); - this.sb.errorURL.stackPushEntry(ee); - } catch (MalformedURLException e) { } + ee.store(); + this.sb.crawlQueues.errorURL.push(ee); } } return this.lastFetchedURLs; } - private String[] getURLs(yacyURL url) { + private ArrayList /*of yacyURL */ getURLs(yacyURL url) { if (url == null) return null; - String[] r = null; + ArrayList a = new ArrayList(); try { httpc con = new httpc( url.getHost(), @@ -528,15 +527,17 @@ public class CrawlURLFetch_p { String encoding = res.responseHeader.getCharacterEncoding(); if (encoding == null) encoding = "US-ASCII"; - r = parseText(new String(sbb.getBytes(), encoding)); + String[] s = (new String(sbb.getBytes(), encoding)).split("\n"); + for (int i = 0; i < s.length; i++) { + try { + a.add(new yacyURL(s[i], null)); + } catch (MalformedURLException e) {} + } } con.close(); } catch (IOException e) { } - return r; + return a; } - private static String[] parseText(String text) { - return text.split("\n"); - } } } diff --git a/htroot/IndexCreateIndexingQueue_p.java b/htroot/IndexCreateIndexingQueue_p.java index 707048202..26ca76446 100644 --- a/htroot/IndexCreateIndexingQueue_p.java +++ b/htroot/IndexCreateIndexingQueue_p.java @@ -77,7 +77,7 @@ public class IndexCreateIndexingQueue_p { }
if (post.containsKey("clearRejected")) {
- switchboard.errorURL.clearStack();
+ switchboard.crawlQueues.errorURL.clearStack();
}
if (post.containsKey("moreRejected")) {
showRejectedCount = Integer.parseInt(post.get("showRejected", "10"));
@@ -88,7 +88,7 @@ public class IndexCreateIndexingQueue_p { plasmaSwitchboardQueue.Entry entry = null;
while ((entry = switchboard.sbQueue.pop()) != null) {
if ((entry != null) && (entry.profile() != null) && (!(entry.profile().storeHTCache()))) {
- plasmaHTCache.deleteFile(entry.url());
+ plasmaHTCache.deleteURLfromCache(entry.url());
}
}
switchboard.sbQueue.clear(); // reset file to clean up content completely
@@ -161,11 +161,11 @@ public class IndexCreateIndexingQueue_p { }
// failure cases
- if (switchboard.errorURL.stackSize() != 0) {
- if (showRejectedCount > switchboard.errorURL.stackSize()) showRejectedCount = switchboard.errorURL.stackSize();
+ if (switchboard.crawlQueues.errorURL.stackSize() != 0) {
+ if (showRejectedCount > switchboard.crawlQueues.errorURL.stackSize()) showRejectedCount = switchboard.crawlQueues.errorURL.stackSize();
prop.put("rejected", "1");
- prop.putNum("rejected_num", switchboard.errorURL.stackSize());
- if (showRejectedCount != switchboard.errorURL.stackSize()) {
+ prop.putNum("rejected_num", switchboard.crawlQueues.errorURL.stackSize());
+ if (showRejectedCount != switchboard.crawlQueues.errorURL.stackSize()) {
prop.put("rejected_only-latest", "1");
prop.putNum("rejected_only-latest_num", showRejectedCount);
prop.putNum("rejected_only-latest_newnum", ((int) (showRejectedCount * 1.5)));
@@ -178,9 +178,9 @@ public class IndexCreateIndexingQueue_p { plasmaCrawlZURL.Entry entry;
yacySeed initiatorSeed, executorSeed;
int j=0;
- for (int i = switchboard.errorURL.stackSize() - 1; i >= (switchboard.errorURL.stackSize() - showRejectedCount); i--) {
+ for (int i = switchboard.crawlQueues.errorURL.stackSize() - 1; i >= (switchboard.crawlQueues.errorURL.stackSize() - showRejectedCount); i--) {
try {
- entry = switchboard.errorURL.stackPopEntry(i);
+ entry = switchboard.crawlQueues.errorURL.top(i);
url = entry.url();
if (url == null) continue;
diff --git a/htroot/IndexCreateLoaderQueue_p.html b/htroot/IndexCreateLoaderQueue_p.html index ba8727f4a..c68116386 100644 --- a/htroot/IndexCreateLoaderQueue_p.html +++ b/htroot/IndexCreateLoaderQueue_p.html @@ -22,13 +22,15 @@ </colgroup>
<tr class="TableHeader">
<th>Initiator</th>
- <th>Depth</th>
+ <th>Depth</th> + <th>Status</th>
<th>URL</th>
</tr>
#{list}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#">
<td>#[initiator]#</td>
- <td>#[depth]#</td>
+ <td>#[depth]#</td> + <td>#[status]#</td>
<td><a href="#[url]#">#[url]#</a></td>
</tr>
#{/list}#
diff --git a/htroot/IndexCreateLoaderQueue_p.java b/htroot/IndexCreateLoaderQueue_p.java index b0e1c1e5d..ce52169f0 100644 --- a/htroot/IndexCreateLoaderQueue_p.java +++ b/htroot/IndexCreateLoaderQueue_p.java @@ -44,9 +44,8 @@ // if the shell's current path is HTROOT
import de.anomic.http.httpHeader;
-import de.anomic.plasma.plasmaCrawlLoaderMessage;
+import de.anomic.plasma.plasmaCrawlEntry;
import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.plasma.crawler.plasmaCrawlWorker;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
@@ -60,32 +59,27 @@ public class IndexCreateLoaderQueue_p { serverObjects prop = new serverObjects();
- if (switchboard.cacheLoader.size() == 0) {
+ if (switchboard.crawlQueues.size() == 0) {
prop.put("loader-set", "0");
} else {
prop.put("loader-set", "1");
boolean dark = true;
-
- ThreadGroup loaderThreads = switchboard.cacheLoader.threadStatus();
- int threadCount = loaderThreads.activeCount();
- Thread[] threadList = new Thread[threadCount*2];
- threadCount = loaderThreads.enumerate(threadList);
+ plasmaCrawlEntry[] w = switchboard.crawlQueues.activeWorker();
yacySeed initiator;
- int i, count = 0;
- for (i = 0; i < threadCount; i++) {
- plasmaCrawlWorker theWorker = (plasmaCrawlWorker)threadList[i];
- plasmaCrawlLoaderMessage theMsg = theWorker.getMessage();
- if (theMsg == null) continue;
+ int count = 0;
+ for (int i = 0; i < w.length; i++) {
+ if (w[i] == null) continue;
- initiator = yacyCore.seedDB.getConnected(theMsg.initiator);
+ initiator = yacyCore.seedDB.getConnected(w[i].initiator());
prop.put("loader-set_list_"+count+"_dark", dark ? "1" : "0");
prop.put("loader-set_list_"+count+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
- prop.put("loader-set_list_"+count+"_depth", theMsg.depth );
- prop.put("loader-set_list_"+count+"_url", theMsg.url.toNormalform(false, true)); // null pointer exception here !!! maybe url = null; check reason.
+ prop.put("loader-set_list_"+count+"_depth", w[i].depth());
+ prop.put("loader-set_list_"+count+"_status", w[i].getStatus());
+ prop.put("loader-set_list_"+count+"_url", w[i].url().toNormalform(true, false));
dark = !dark;
count++;
}
- prop.put("loader-set_list", count );
+ prop.put("loader-set_list", count);
prop.put("loader-set_num", count);
}
diff --git a/htroot/IndexCreateWWWGlobalQueue_p.java b/htroot/IndexCreateWWWGlobalQueue_p.java index 7b563181d..a184b89a1 100644 --- a/htroot/IndexCreateWWWGlobalQueue_p.java +++ b/htroot/IndexCreateWWWGlobalQueue_p.java @@ -79,8 +79,8 @@ public class IndexCreateWWWGlobalQueue_p { }
if (post.containsKey("clearcrawlqueue")) {
- int c = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
- switchboard.noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_LIMIT);
+ int c = switchboard.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
+ switchboard.crawlQueues.noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_LIMIT);
try { switchboard.cleanProfiles(); } catch (InterruptedException e) { /* Ignore this */}
/*
int c = 0;
@@ -93,18 +93,18 @@ public class IndexCreateWWWGlobalQueue_p { prop.putNum("info_numEntries", c);
} else if (post.containsKey("deleteEntry")) {
String urlHash = (String) post.get("deleteEntry");
- switchboard.noticeURL.removeByURLHash(urlHash);
+ switchboard.crawlQueues.noticeURL.removeByURLHash(urlHash);
prop.put("LOCATION","");
return prop;
}
}
- int stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
+ int stackSize = switchboard.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
if (stackSize == 0) {
prop.put("crawler-queue", "0");
} else {
prop.put("crawler-queue", "1");
- plasmaCrawlEntry[] crawlerList = switchboard.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, showLimit);
+ plasmaCrawlEntry[] crawlerList = switchboard.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, showLimit);
plasmaCrawlEntry urle;
boolean dark = true;
diff --git a/htroot/IndexCreateWWWLocalQueue_p.java b/htroot/IndexCreateWWWLocalQueue_p.java index dc87fb3ff..5a6504717 100644 --- a/htroot/IndexCreateWWWLocalQueue_p.java +++ b/htroot/IndexCreateWWWLocalQueue_p.java @@ -79,7 +79,7 @@ public class IndexCreateWWWLocalQueue_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
serverObjects prop = new serverObjects();
int showLimit = 100;
@@ -96,9 +96,9 @@ public class IndexCreateWWWLocalQueue_p { String pattern = post.get("pattern", ".*").trim();
final int option = post.getInt("option", INVALID);
if (pattern.equals(".*")) {
- c = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
- switchboard.noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_CORE);
- try { switchboard.cleanProfiles(); } catch (InterruptedException e) {/* ignore this */}
+ c = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
+ sb.crawlQueues.noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_CORE);
+ try { sb.cleanProfiles(); } catch (InterruptedException e) {/* ignore this */}
} else if (option > INVALID) {
Pattern compiledPattern = null;
try {
@@ -108,7 +108,7 @@ public class IndexCreateWWWLocalQueue_p { if (option == PROFILE) {
// search and delete the crawl profile (_much_ faster, independant of queue size)
// XXX: what to do about the annoying LOST PROFILE messages in the log?
- Iterator it = switchboard.profilesActiveCrawls.profiles(true);
+ Iterator it = sb.profilesActiveCrawls.profiles(true);
plasmaCrawlProfile.entry entry;
while (it.hasNext()) {
entry = (plasmaCrawlProfile.entry)it.next();
@@ -119,12 +119,12 @@ public class IndexCreateWWWLocalQueue_p { name.equals(plasmaSwitchboard.CRAWL_PROFILE_SNIPPET_MEDIA))
continue;
if (compiledPattern.matcher(name).find()) {
- switchboard.profilesActiveCrawls.removeEntry(entry.handle());
+ sb.profilesActiveCrawls.removeEntry(entry.handle());
}
}
} else {
// iterating through the list of URLs
- Iterator iter = switchboard.noticeURL.iterator(plasmaCrawlNURL.STACK_TYPE_CORE);
+ Iterator iter = sb.crawlQueues.noticeURL.iterator(plasmaCrawlNURL.STACK_TYPE_CORE);
plasmaCrawlEntry entry;
while (iter.hasNext()) {
if ((entry = (plasmaCrawlEntry) iter.next()) == null) continue;
@@ -144,7 +144,7 @@ public class IndexCreateWWWLocalQueue_p { if (value != null) {
Matcher matcher = compiledPattern.matcher(value);
if (matcher.find()) {
- switchboard.noticeURL.removeByURLHash(entry.url().hash());
+ sb.crawlQueues.noticeURL.removeByURLHash(entry.url().hash());
}
}
}
@@ -158,18 +158,18 @@ public class IndexCreateWWWLocalQueue_p { prop.putNum("info_numEntries", c);
} else if (post.containsKey("deleteEntry")) {
String urlHash = (String) post.get("deleteEntry");
- switchboard.noticeURL.removeByURLHash(urlHash);
+ sb.crawlQueues.noticeURL.removeByURLHash(urlHash);
prop.put("LOCATION","");
return prop;
}
}
- int showNum = 0, stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
+ int showNum = 0, stackSize = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
if (stackSize == 0) {
prop.put("crawler-queue", "0");
} else {
prop.put("crawler-queue", "1");
- plasmaCrawlEntry[] crawlerList = switchboard.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, (int) (showLimit * 1.20));
+ plasmaCrawlEntry[] crawlerList = sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, (int) (showLimit * 1.20));
plasmaCrawlEntry urle;
boolean dark = true;
@@ -182,7 +182,7 @@ public class IndexCreateWWWLocalQueue_p { if ((urle != null)&&(urle.url()!=null)) {
initiator = yacyCore.seedDB.getConnected(urle.initiator());
profileHandle = urle.profileHandle();
- profileEntry = (profileHandle == null) ? null : switchboard.profilesActiveCrawls.getEntry(profileHandle);
+ profileEntry = (profileHandle == null) ? null : sb.profilesActiveCrawls.getEntry(profileHandle);
prop.put("crawler-queue_list_"+showNum+"_dark", dark ? "1" : "0");
prop.put("crawler-queue_list_"+showNum+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()) );
prop.put("crawler-queue_list_"+showNum+"_profile", ((profileEntry == null) ? "unknown" : profileEntry.name()));
diff --git a/htroot/IndexCreateWWWRemoteQueue_p.java b/htroot/IndexCreateWWWRemoteQueue_p.java index dd2be646d..a874e0474 100644 --- a/htroot/IndexCreateWWWRemoteQueue_p.java +++ b/htroot/IndexCreateWWWRemoteQueue_p.java @@ -79,8 +79,8 @@ public class IndexCreateWWWRemoteQueue_p { } if (post.containsKey("clearcrawlqueue")) { - int c = sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE); - sb.noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_REMOTE); + int c = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE); + sb.crawlQueues.noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_REMOTE); try { sb.cleanProfiles(); } catch (InterruptedException e) { /* Ignore this */} /* int c = 0; @@ -93,18 +93,18 @@ public class IndexCreateWWWRemoteQueue_p { prop.putNum("info_numEntries", c); } else if (post.containsKey("deleteEntry")) { String urlHash = (String) post.get("deleteEntry"); - sb.noticeURL.removeByURLHash(urlHash); + sb.crawlQueues.noticeURL.removeByURLHash(urlHash); prop.put("LOCATION",""); return prop; } } - int stackSize = sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE); + int stackSize = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE); if (stackSize == 0) { prop.put("crawler-queue", "0"); } else { prop.put("crawler-queue", "1"); - plasmaCrawlEntry[] crawlerList = sb.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_REMOTE, showLimit); + plasmaCrawlEntry[] crawlerList = sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_REMOTE, showLimit); plasmaCrawlEntry urle; boolean dark = true; diff --git a/htroot/PerformanceQueues_p.java b/htroot/PerformanceQueues_p.java index 80813f74c..d8f47e1bb 100644 --- a/htroot/PerformanceQueues_p.java +++ b/htroot/PerformanceQueues_p.java @@ -47,7 +47,6 @@ import java.io.File; import java.util.Iterator;
import java.util.Map;
-import org.apache.commons.pool.impl.GenericKeyedObjectPool;
import org.apache.commons.pool.impl.GenericObjectPool;
import de.anomic.http.httpHeader;
@@ -203,22 +202,13 @@ public class PerformanceQueues_p { * configuring the crawler pool
*/
// getting the current crawler pool configuration
- GenericKeyedObjectPool.Config crawlerPoolConfig = switchboard.cacheLoader.getPoolConfig();
int maxActive = Integer.parseInt(post.get("Crawler Pool_maxActive","8"));
- int maxIdle = Integer.parseInt(post.get("Crawler Pool_maxIdle","4"));
- int minIdle = 0; // Integer.parseInt(post.get("Crawler Pool_minIdle","0"));
-
- //crawlerPoolConfig.minIdle = (minIdle > maxIdle) ? maxIdle/2 : minIdle;
- crawlerPoolConfig.maxIdle = (maxIdle > maxActive) ? maxActive/2 : maxIdle;
- crawlerPoolConfig.maxActive = maxActive;
// accept new crawler pool settings
plasmaSwitchboard.crawlSlots = maxActive;
- switchboard.cacheLoader.setPoolConfig(crawlerPoolConfig);
// storing the new values into configfile
switchboard.setConfig("crawler.MaxActiveThreads",maxActive);
- switchboard.setConfig("crawler.MaxIdleThreads",maxIdle);
//switchboard.setConfig("crawler.MinIdleThreads",minIdle);
/*
@@ -231,6 +221,8 @@ public class PerformanceQueues_p { } catch (NumberFormatException e) {
maxActive = 8;
}
+ int maxIdle = 0;
+ int minIdle = 0;
try {
maxIdle = Integer.parseInt(post.get("httpd Session Pool_maxIdle","4"));
} catch (NumberFormatException e) {
@@ -253,36 +245,6 @@ public class PerformanceQueues_p { switchboard.setConfig("httpdMaxIdleSessions",maxIdle);
switchboard.setConfig("httpdMinIdleSessions",minIdle);
- /*
- * Configuring the crawlStacker pool
- */
- GenericObjectPool.Config stackerPoolConfig = switchboard.sbStackCrawlThread.getPoolConfig();
- try {
- maxActive = Integer.parseInt(post.get("CrawlStacker Session Pool_maxActive","10"));
- } catch (NumberFormatException e) {
- maxActive = 10;
- }
- try {
- maxIdle = Integer.parseInt(post.get("CrawlStacker Session Pool_maxIdle","10"));
- } catch (NumberFormatException e) {
- maxIdle = 10;
- }
- try {
- minIdle = Integer.parseInt(post.get("CrawlStacker Session Pool_minIdle","5"));
- } catch (NumberFormatException e) {
- minIdle = 5;
- }
-
- stackerPoolConfig.minIdle = (minIdle > maxIdle) ? maxIdle/2 : minIdle;
- stackerPoolConfig.maxIdle = (maxIdle > maxActive) ? maxActive/2 : maxIdle;
- stackerPoolConfig.maxActive = maxActive;
-
- switchboard.sbStackCrawlThread.setPoolConfig(stackerPoolConfig);
-
- // storing the new values into configfile
- switchboard.setConfig("stacker.MaxActiveThreads",maxActive);
- switchboard.setConfig("stacker.MaxIdleThreads",maxIdle);
- switchboard.setConfig("stacker.MinIdleThreads",minIdle);
}
if ((post != null) && (post.containsKey("PrioritySubmit"))) {
@@ -313,15 +275,14 @@ public class PerformanceQueues_p { prop.put("onlineCautionDelay", switchboard.getConfigLong("onlineCautionDelay", 30000));
prop.putNum("onlineCautionDelayCurrent", System.currentTimeMillis() - switchboard.proxyLastAccess);
- // table thread pool settings
- GenericKeyedObjectPool.Config crawlerPoolConfig = switchboard.cacheLoader.getPoolConfig();
- prop.put("pool_0_name", "Crawler Pool");
- prop.put("pool_0_maxActive", crawlerPoolConfig.maxActive);
- prop.put("pool_0_maxIdle", crawlerPoolConfig.maxIdle);
- prop.put("pool_0_minIdleConfigurable", "0");
- prop.put("pool_0_minIdle", "0");
- prop.put("pool_0_numActive", switchboard.cacheLoader.getNumActiveWorker());
- prop.put("pool_0_numIdle", switchboard.cacheLoader.getNumIdleWorker());
+ // table thread pool settings + prop.put("pool_0_name","Crawler Pool");
+ prop.put("pool_0_maxActive", switchboard.getConfigLong("crawler.MaxActiveThreads", 0));
+ prop.put("pool_0_maxIdle", 0);
+ prop.put("pool_0_minIdleConfigurable",0);
+ prop.put("pool_0_minIdle", 0);
+ prop.put("pool_0_numActive",switchboard.crawlQueues.size());
+ prop.put("pool_0_numIdle", 0); serverThread httpd = switchboard.getThread("10_httpd");
GenericObjectPool.Config httpdPoolConfig = ((serverCore)httpd).getPoolConfig();
@@ -332,16 +293,8 @@ public class PerformanceQueues_p { prop.put("pool_1_minIdle", httpdPoolConfig.minIdle);
prop.put("pool_1_numActive", ((serverCore)httpd).getActiveSessionCount());
prop.put("pool_1_numIdle", ((serverCore)httpd).getIdleSessionCount());
-
- GenericObjectPool.Config stackerPoolConfig = switchboard.sbStackCrawlThread.getPoolConfig();
- prop.putHTML("pool_2_name", "CrawlStacker Session Pool");
- prop.put("pool_2_maxActive", stackerPoolConfig.maxActive);
- prop.put("pool_2_maxIdle", stackerPoolConfig.maxIdle);
- prop.put("pool_2_minIdleConfigurable", "1");
- prop.put("pool_2_minIdle", stackerPoolConfig.minIdle);
- prop.put("pool_2_numActive", switchboard.sbStackCrawlThread.getNumActiveWorker());
- prop.put("pool_2_numIdle", switchboard.sbStackCrawlThread.getNumIdleWorker());
- prop.put("pool", "3");
+ + prop.put("pool", "2"); long curr_prio = switchboard.getConfigLong("javastart_priority",0);
prop.put("priority_normal",(curr_prio==0) ? "1" : "0");
diff --git a/htroot/QuickCrawlLink_p.java b/htroot/QuickCrawlLink_p.java index 421d98bcf..8e0e41d97 100644 --- a/htroot/QuickCrawlLink_p.java +++ b/htroot/QuickCrawlLink_p.java @@ -140,8 +140,8 @@ public class QuickCrawlLink_p { String urlhash = crawlingStartURL.hash();
switchboard.wordIndex.loadedURL.remove(urlhash);
- switchboard.noticeURL.removeByURLHash(urlhash);
- switchboard.errorURL.remove(urlhash);
+ switchboard.crawlQueues.noticeURL.removeByURLHash(urlhash);
+ switchboard.crawlQueues.errorURL.remove(urlhash);
// create crawling profile
plasmaCrawlProfile.entry pe = null;
@@ -177,8 +177,8 @@ public class QuickCrawlLink_p { // stack URL
String reasonString = null;
try {
- reasonString = switchboard.sbStackCrawlThread.stackCrawl(
- crawlingStart,
+ reasonString = switchboard.crawlStacker.stackCrawl(
+ crawlingStartURL,
null,
yacyCore.seedDB.mySeed().hash,
(title==null)?"CRAWLING-ROOT":title,
diff --git a/htroot/Status.java b/htroot/Status.java index ecb3cb0b7..5439ff389 100644 --- a/htroot/Status.java +++ b/htroot/Status.java @@ -334,7 +334,7 @@ public class Status { prop.putNum("indexingQueueMax", indexingMaxCount);
prop.put("indexingQueuePercent",(indexingPercent>100) ? 100 : indexingPercent);
- int loaderJobCount = sb.cacheLoader.size();
+ int loaderJobCount = sb.crawlQueues.size();
int loaderMaxCount = plasmaSwitchboard.crawlSlots;
int loaderPercent = (loaderMaxCount==0)?0:loaderJobCount*100/loaderMaxCount;
prop.putNum("loaderQueueSize", loaderJobCount);
@@ -350,7 +350,7 @@ public class Status { prop.putNum("globalCrawlTriggerQueueSize", sb.getThread(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER).getJobCount());
prop.put("globalCrawlTriggerPaused",sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER) ? "1" : "0");
- prop.putNum("stackCrawlQueueSize", sb.sbStackCrawlThread.size());
+ prop.putNum("stackCrawlQueueSize", sb.crawlStacker.size());
// return rewrite properties
prop.put("date",(new Date()).toString());
diff --git a/htroot/ViewFile.java b/htroot/ViewFile.java index 1f3c6a262..eb7a4fc48 100644 --- a/htroot/ViewFile.java +++ b/htroot/ViewFile.java @@ -64,7 +64,6 @@ import de.anomic.plasma.plasmaParserDocument; import de.anomic.plasma.plasmaSnippetCache;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.cache.IResourceInfo;
-import de.anomic.plasma.crawler.plasmaCrawlerException;
import de.anomic.plasma.parser.ParserException;
import de.anomic.server.serverFileUtils;
import de.anomic.server.serverObjects;
@@ -169,8 +168,8 @@ public class ViewFile { if (resource == null) {
plasmaHTCache.Entry entry = null;
try {
- entry = plasmaSnippetCache.loadResourceFromWeb(url, 5000, false, true);
- } catch (plasmaCrawlerException e) {
+ entry = sb.crawlQueues.loadResourceFromWeb(url, 5000, false, true);
+ } catch (Exception e) {
prop.put("error", "4");
prop.putHTML("error_errorText", e.getMessage());
prop.put("viewMode", VIEW_MODE_NO_TEXT);
diff --git a/htroot/WatchCrawler_p.java b/htroot/WatchCrawler_p.java index b186b3535..f7affeae2 100644 --- a/htroot/WatchCrawler_p.java +++ b/htroot/WatchCrawler_p.java @@ -71,7 +71,7 @@ public class WatchCrawler_p { } else { prop.put("info", "0"); - if ((post.containsKey("autoforward")) && (switchboard.coreCrawlJobSize() == 0)) { + if ((post.containsKey("autoforward")) && (switchboard.crawlQueues.coreCrawlJobSize() == 0)) { prop.put("forwardToCrawlStart", "1"); } @@ -180,10 +180,11 @@ public class WatchCrawler_p { // stack request // first delete old entry, if exists - String urlhash = (new yacyURL(crawlingStart, null)).hash(); + yacyURL url = new yacyURL(crawlingStart, null); + String urlhash = url.hash(); switchboard.wordIndex.loadedURL.remove(urlhash); - switchboard.noticeURL.removeByURLHash(urlhash); - switchboard.errorURL.remove(urlhash); + switchboard.crawlQueues.noticeURL.removeByURLHash(urlhash); + switchboard.crawlQueues.errorURL.remove(urlhash); // stack url switchboard.profilesPassiveCrawls.removeEntry(crawlingStartURL.hash()); // if there is an old entry, delete it @@ -194,7 +195,7 @@ public class WatchCrawler_p { crawlingQ, indexText, indexMedia, storeHTCache, true, crawlOrder, xsstopw, xdstopw, xpstopw); - String reasonString = switchboard.sbStackCrawlThread.stackCrawl(crawlingStart, null, yacyCore.seedDB.mySeed().hash, "CRAWLING-ROOT", new Date(), 0, pe); + String reasonString = switchboard.crawlStacker.stackCrawl(url, null, yacyCore.seedDB.mySeed().hash, "CRAWLING-ROOT", new Date(), 0, pe); if (reasonString == null) { // liftoff! @@ -224,9 +225,9 @@ public class WatchCrawler_p { prop.putHTML("info_crawlingURL", ((String) post.get("crawlingURL"))); prop.putHTML("info_reasonString", reasonString); - plasmaCrawlZURL.Entry ee = switchboard.errorURL.newEntry(crawlingStartURL, reasonString); + plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(crawlingStartURL, reasonString); ee.store(); - switchboard.errorURL.stackPushEntry(ee); + switchboard.crawlQueues.errorURL.push(ee); } } catch (PatternSyntaxException e) { prop.put("info", "4"); //crawlfilter does not match url @@ -297,7 +298,7 @@ public class WatchCrawler_p { } // enqueuing the url for crawling - switchboard.sbStackCrawlThread.enqueue( + switchboard.crawlStacker.enqueueEntry( nexturlURL, null, yacyCore.seedDB.mySeed().hash, diff --git a/htroot/xml/queues_p.java b/htroot/xml/queues_p.java index 0473dc96a..0661ead95 100644 --- a/htroot/xml/queues_p.java +++ b/htroot/xml/queues_p.java @@ -55,11 +55,9 @@ import java.util.Locale; import de.anomic.http.httpHeader;
import de.anomic.plasma.plasmaCrawlEntry;
-import de.anomic.plasma.plasmaCrawlLoaderMessage;
import de.anomic.plasma.plasmaCrawlNURL;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.plasmaSwitchboardQueue;
-import de.anomic.plasma.crawler.http.CrawlWorker;
import de.anomic.server.serverObjects;
import de.anomic.server.serverSwitch;
import de.anomic.yacy.yacyCore;
@@ -78,7 +76,7 @@ public class queues_p { public static serverObjects respond(httpHeader header, serverObjects post, serverSwitch env) {
// return variable that accumulates replacements
- plasmaSwitchboard switchboard = (plasmaSwitchboard) env;
+ plasmaSwitchboard sb = (plasmaSwitchboard) env;
//wikiCode wikiTransformer = new wikiCode(switchboard);
serverObjects prop = new serverObjects();
if (post == null || !post.containsKey("html"))
@@ -88,12 +86,12 @@ public class queues_p { yacySeed initiator;
- //indexing queue
- prop.putNum("indexingSize", switchboard.getThread(plasmaSwitchboard.INDEXER).getJobCount()+switchboard.indexingTasksInProcess.size());
+ //indexing queue + prop.putNum("indexingSize", sb.getThread(plasmaSwitchboard.INDEXER).getJobCount()+sb.indexingTasksInProcess.size());
prop.putNum("indexingMax", plasmaSwitchboard.indexingSlots);
- prop.putNum("urlpublictextSize", switchboard.wordIndex.loadedURL.size());
- prop.putNum("rwipublictextSize", switchboard.wordIndex.size());
- if ((switchboard.sbQueue.size() == 0) && (switchboard.indexingTasksInProcess.size() == 0)) {
+ prop.putNum("urlpublictextSize", sb.wordIndex.loadedURL.size());
+ prop.putNum("rwipublictextSize", sb.wordIndex.size());
+ if ((sb.sbQueue.size() == 0) && (sb.indexingTasksInProcess.size() == 0)) { prop.put("list", "0"); //is empty
} else {
plasmaSwitchboardQueue.Entry pcentry;
@@ -103,14 +101,14 @@ public class queues_p { ArrayList entryList = new ArrayList();
// getting all entries that are currently in process
- synchronized (switchboard.indexingTasksInProcess) {
- inProcessCount = switchboard.indexingTasksInProcess.size();
- entryList.addAll(switchboard.indexingTasksInProcess.values());
+ synchronized (sb.indexingTasksInProcess) {
+ inProcessCount = sb.indexingTasksInProcess.size();
+ entryList.addAll(sb.indexingTasksInProcess.values());
}
// getting all enqueued entries
- if ((switchboard.sbQueue.size() > 0)) {
- Iterator i1 = switchboard.sbQueue.entryIterator(false);
+ if ((sb.sbQueue.size() > 0)) {
+ Iterator i1 = sb.sbQueue.entryIterator(false);
while (i1.hasNext()) entryList.add((plasmaSwitchboardQueue.Entry) i1.next());
}
@@ -130,7 +128,7 @@ public class queues_p { prop.put("list-indexing_"+i+"_depth", pcentry.depth());
prop.put("list-indexing_"+i+"_modified", pcentry.getModificationDate());
prop.putHTML("list-indexing_"+i+"_anchor", (pcentry.anchorName()==null) ? "" : pcentry.anchorName(), true);
- prop.put("list-indexing_"+i+"_url", pcentry.url().toNormalform(false, true));
+ prop.putHTML("list-indexing_"+i+"_url", pcentry.url().toNormalform(false, true), true);
prop.putNum("list-indexing_"+i+"_size", entrySize);
prop.put("list-indexing_"+i+"_inProcess", (inProcess) ? "1" : "0");
prop.put("list-indexing_"+i+"_hash", pcentry.urlHash());
@@ -140,45 +138,42 @@ public class queues_p { prop.put("list-indexing", ok);
}
- //loader queue
- prop.put("loaderSize", switchboard.cacheLoader.size());
- prop.put("loaderMax", plasmaSwitchboard.crawlSlots);
- if (switchboard.cacheLoader.size() == 0) {
+ //loader queue + prop.put("loaderSize", Integer.toString(sb.crawlQueues.size()));
+ prop.put("loaderMax", Integer.toString(plasmaSwitchboard.crawlSlots)); + if (sb.crawlQueues.size() == 0) {
prop.put("list-loader", "0");
} else {
- ThreadGroup loaderThreads = switchboard.cacheLoader.threadStatus();
- Thread[] threadList = new Thread[loaderThreads.activeCount()*2];
- int size = loaderThreads.enumerate(threadList);
-
- int i, count = 0;
- for (i = 0; i < size; i++) {
- CrawlWorker theWorker = (CrawlWorker)threadList[i];
- plasmaCrawlLoaderMessage theMsg = theWorker.theMsg;
- if (theMsg == null) continue;
- prop.put("list-loader_"+count+"_profile", theMsg.profile.name());
- initiator = yacyCore.seedDB.getConnected(theMsg.initiator);
+ plasmaCrawlEntry[] w = sb.crawlQueues.activeWorker();
+ int count = 0;
+ for (int i = 0; i < w.length; i++) {
+ if (w[i] == null) continue;
+ prop.put("list-loader_"+count+"_profile", w[i].profileHandle());
+ initiator = yacyCore.seedDB.getConnected(w[i].initiator());
prop.put("list-loader_"+count+"_initiator", ((initiator == null) ? "proxy" : initiator.getName()));
- prop.put("list-loader_"+count+"_depth", theMsg.depth );
- prop.put("list-loader_"+count+"_url", theMsg.url.toString()); // null pointer exception here !!! maybe url = null; check reason.
+ prop.put("list-loader_"+count+"_depth", w[i].depth());
+ prop.putHTML("list-loader_"+count+"_url", w[i].url().toString(), true);
count++;
}
- prop.put("list-loader", count );
+ prop.put("list-loader", count);
}
- //local crawl queue
- prop.putNum("localCrawlSize", switchboard.getThread(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL).getJobCount());
- prop.put("localCrawlState", switchboard.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
- int stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
- addNTable(prop, "list-local", switchboard.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
+ //local crawl queue + prop.putNum("localCrawlSize", Integer.toString(sb.getThread(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL).getJobCount()));
+ prop.put("localCrawlState", sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL) ? STATE_PAUSED : STATE_RUNNING);
+ int stackSize = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
+ addNTable(prop, "list-local", sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_CORE, Math.min(10, stackSize)));
+ - //global crawl queue
- prop.putNum("remoteCrawlSize", switchboard.getThread(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER).getJobCount());
- prop.put("remoteCrawlState", switchboard.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER) ? STATE_PAUSED : STATE_RUNNING);
- stackSize = switchboard.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
+ //global crawl queue + prop.putNum("remoteCrawlSize", Integer.toString(sb.getThread(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER).getJobCount()));
+ prop.put("remoteCrawlState", sb.crawlJobIsPaused(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER) ? STATE_PAUSED : STATE_RUNNING);
+ stackSize = sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
+ if (stackSize == 0) {
prop.put("list-remote", "0");
} else {
- addNTable(prop, "list-remote", switchboard.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, Math.min(10, stackSize)));
+ addNTable(prop, "list-remote", sb.crawlQueues.noticeURL.top(plasmaCrawlNURL.STACK_TYPE_LIMIT, Math.min(10, stackSize)));
}
// return rewrite properties
diff --git a/htroot/yacy/crawlOrder.java b/htroot/yacy/crawlOrder.java index 2170403aa..a56efea78 100644 --- a/htroot/yacy/crawlOrder.java +++ b/htroot/yacy/crawlOrder.java @@ -129,7 +129,7 @@ public final class crawlOrder { delay = "3600"; // may request one hour later again
} else try {
yacySeed requester = yacyCore.seedDB.getConnected(iam);
- int queuesize = switchboard.coreCrawlJobSize() + switchboard.limitCrawlTriggerJobSize() + switchboard.remoteTriggeredCrawlJobSize() + switchboard.queueSize();
+ int queuesize = switchboard.crawlQueues.coreCrawlJobSize() + switchboard.crawlQueues.limitCrawlTriggerJobSize() + switchboard.crawlQueues.remoteTriggeredCrawlJobSize() + switchboard.queueSize();
if (requester == null) {
response = "denied";
reason = "unknown-client";
@@ -180,7 +180,8 @@ public final class crawlOrder { // old method: only one url
// normalizing URL
- String newURL = new yacyURL((String) urlv.get(0), null).toNormalform(true, true);
+ yacyURL url = new yacyURL((String) urlv.get(0), null);
+ String newURL = url.toNormalform(true, true);
if (!newURL.equals(urlv.get(0))) {
env.getLog().logWarning("crawlOrder: Received not normalized URL " + urlv.get(0));
}
@@ -197,7 +198,7 @@ public final class crawlOrder { // adding URL to noticeURL Queue
env.getLog().logFinest("crawlOrder: a: url='" + newURL + "'");
- stackresult = stack(switchboard, newURL, refURL, iam, youare);
+ stackresult = stack(switchboard, url, refURL, iam, youare);
response = (String) stackresult[0];
reason = (String) stackresult[1];
lurl = (String) stackresult[2];
@@ -209,12 +210,13 @@ public final class crawlOrder { //int rejectedCount = 0;
for (int i = 0; i < count; i++) {
env.getLog().logFinest("crawlOrder: b: url='" + (String) urlv.get(i) + "'");
-
- stackresult = stack(switchboard, (String) urlv.get(i), (String) refv.get(i), iam, youare);
- response = (String) stackresult[0];
- prop.put("list_" + i + "_job", (String) stackresult[0] + "," + (String) stackresult[1]);
- prop.put("list_" + i + "_lurl", (String) stackresult[2]);
- prop.put("list_" + i + "_count", i);
+ try {
+ stackresult = stack(switchboard, new yacyURL((String) urlv.get(i), null), (String) refv.get(i), iam, youare);
+ response = (String) stackresult[0];
+ prop.put("list_" + i + "_job", (String) stackresult[0] + "," + (String) stackresult[1]);
+ prop.put("list_" + i + "_lurl", (String) stackresult[2]);
+ prop.put("list_" + i + "_count", i);
+ } catch (MalformedURLException e) {}
}
prop.put("list", count);
response = "enqueued";
@@ -242,13 +244,13 @@ public final class crawlOrder { return prop;
}
- private static Object[] stack(plasmaSwitchboard switchboard, String url, String referrer, String iam, String youare) {
+ private static Object[] stack(plasmaSwitchboard switchboard, yacyURL url, String referrer, String iam, String youare) {
String response, reason, lurl;
// stack url
switchboard.getLog().logFinest("crawlOrder: stack: url='" + url + "'");
String reasonString = null;
try {
- reasonString = switchboard.sbStackCrawlThread.stackCrawl(url, referrer, iam, "REMOTE-CRAWLING", new Date(), 0, switchboard.defaultRemoteProfile);
+ reasonString = switchboard.crawlStacker.stackCrawl(url, referrer, iam, "REMOTE-CRAWLING", new Date(), 0, switchboard.defaultRemoteProfile);
} catch (InterruptedException e) {
reasonString = "Shutdown in progress";
}
@@ -262,11 +264,7 @@ public final class crawlOrder { reason = reasonString;
// send lurl-Entry as response
indexURLEntry entry;
- try {
- entry = switchboard.wordIndex.loadedURL.load((new yacyURL(url, null)).hash(), null);
- } catch (MalformedURLException e) {
- entry = null;
- }
+ entry = switchboard.wordIndex.loadedURL.load(url.hash(), null);
if (entry == null) {
response = "rejected";
lurl = "";
diff --git a/htroot/yacy/crawlReceipt.java b/htroot/yacy/crawlReceipt.java index 4f0bbe1ac..eaf825ce0 100644 --- a/htroot/yacy/crawlReceipt.java +++ b/htroot/yacy/crawlReceipt.java @@ -157,7 +157,7 @@ public final class crawlReceipt { // put new entry into database
switchboard.wordIndex.loadedURL.store(entry);
switchboard.wordIndex.loadedURL.stack(entry, youare, iam, 1);
- switchboard.delegatedURL.remove(entry.hash()); // the delegated work has been done
+ switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work has been done
log.logInfo("crawlReceipt: RECEIVED RECEIPT from " + otherPeerName + " for URL " + entry.hash() + ":" + comp.url().toNormalform(false, true));
// ready for more
@@ -169,10 +169,10 @@ public final class crawlReceipt { return prop;
}
- switchboard.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
- plasmaCrawlZURL.Entry ee = switchboard.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason);
+ switchboard.crawlQueues.delegatedURL.remove(entry.hash()); // the delegated work is transformed into an error case
+ plasmaCrawlZURL.Entry ee = switchboard.crawlQueues.errorURL.newEntry(entry.toBalancerEntry(), youare, null, 0, result + ":" + reason);
ee.store();
- switchboard.errorURL.stackPushEntry(ee);
+ switchboard.crawlQueues.errorURL.push(ee);
//switchboard.noticeURL.remove(receivedUrlhash);
prop.put("delay", "3600");
return prop;
diff --git a/htroot/yacy/urls.java b/htroot/yacy/urls.java index 4229e5818..336c45e4d 100644 --- a/htroot/yacy/urls.java +++ b/htroot/yacy/urls.java @@ -60,9 +60,9 @@ public class urls { int count = Math.min(100, post.getInt("count", 0)); int c = 0; plasmaCrawlEntry entry; - while ((count > 0) && (sb.noticeURL.stackSize(stackType) > 0)) { + while ((count > 0) && (sb.crawlQueues.noticeURL.stackSize(stackType) > 0)) { try { - entry = sb.noticeURL.pop(stackType, false); + entry = sb.crawlQueues.noticeURL.pop(stackType, false); } catch (IOException e) { break; } diff --git a/source/de/anomic/data/SitemapParser.java b/source/de/anomic/data/SitemapParser.java index d446bbc1c..8bb15650d 100644 --- a/source/de/anomic/data/SitemapParser.java +++ b/source/de/anomic/data/SitemapParser.java @@ -274,12 +274,12 @@ public class SitemapParser extends DefaultHandler { if (this.nextURL == null) return; // get the url hash - String nexturlhash; + String nexturlhash = null; + yacyURL url = null; try { - nexturlhash = (new yacyURL(this.nextURL, null)).hash(); - } catch (MalformedURLException e1) { - nexturlhash = null; - } + url = new yacyURL(this.nextURL, null); + nexturlhash = url.hash(); + } catch (MalformedURLException e1) {} // check if the url is known and needs to be recrawled if (this.lastMod != null) { @@ -299,8 +299,8 @@ public class SitemapParser extends DefaultHandler { // URL needs to crawled String error = null; try { - error = this.switchboard.sbStackCrawlThread.stackCrawl( - this.nextURL, + error = this.switchboard.crawlStacker.stackCrawl( + url, null, // this.siteMapURL.toString(), yacyCore.seedDB.mySeed().hash, this.nextURL, @@ -317,9 +317,9 @@ public class SitemapParser extends DefaultHandler { this.logger.logInfo("The URL '" + this.nextURL + "' can not be crawled. Reason: " + error); // insert URL into the error DB - plasmaCrawlZURL.Entry ee = this.switchboard.errorURL.newEntry(new yacyURL(this.nextURL, null), error); + plasmaCrawlZURL.Entry ee = this.switchboard.crawlQueues.errorURL.newEntry(new yacyURL(this.nextURL, null), error); ee.store(); - this.switchboard.errorURL.stackPushEntry(ee); + this.switchboard.crawlQueues.errorURL.push(ee); } catch (MalformedURLException e) {/* ignore this */ } } else { this.logger.logInfo("New URL '" + this.nextURL + "' added for crawling."); diff --git a/source/de/anomic/http/httpdProxyHandler.java b/source/de/anomic/http/httpdProxyHandler.java index 1e6dfaefd..1fe76fb47 100644 --- a/source/de/anomic/http/httpdProxyHandler.java +++ b/source/de/anomic/http/httpdProxyHandler.java @@ -558,7 +558,7 @@ public final class httpdProxyHandler { if ((cacheFile.isFile()) && (cachedResponseHeader != null)) {
// delete the cache
sizeBeforeDelete = cacheFile.length();
- plasmaHTCache.deleteFile(url);
+ plasmaHTCache.deleteURLfromCache(url);
conProp.setProperty(httpHeader.CONNECTION_PROP_PROXY_RESPOND_CODE,"TCP_REFRESH_MISS");
}
diff --git a/source/de/anomic/icap/icapd.java b/source/de/anomic/icap/icapd.java index de7ab0813..caaf03b24 100644 --- a/source/de/anomic/icap/icapd.java +++ b/source/de/anomic/icap/icapd.java @@ -402,7 +402,7 @@ public class icapd implements serverHandler { // if the file already exits we delete it
if (cacheFile.isFile()) {
- plasmaHTCache.deleteFile(httpRequestURL);
+ plasmaHTCache.deleteURLfromCache(httpRequestURL);
}
// we write the new cache entry to file system directly
cacheFile.getParentFile().mkdirs();
diff --git a/source/de/anomic/plasma/cache/ftp/ResourceInfo.java b/source/de/anomic/plasma/cache/ftp/ResourceInfo.java index 76c7fb63e..e9dae385a 100644 --- a/source/de/anomic/plasma/cache/ftp/ResourceInfo.java +++ b/source/de/anomic/plasma/cache/ftp/ResourceInfo.java @@ -46,7 +46,6 @@ package de.anomic.plasma.cache.ftp;
-import java.net.MalformedURLException;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
@@ -80,7 +79,7 @@ public class ResourceInfo implements IResourceInfo { this.propertyMap = new HashMap(objectInfo);
}
- public ResourceInfo(yacyURL objectURL, String refererUrl, String mimeType, Date fileDate) {
+ public ResourceInfo(yacyURL objectURL, yacyURL refererUrl, String mimeType, Date fileDate) {
if (objectURL == null) throw new NullPointerException();
// generating the url hash
@@ -110,11 +109,7 @@ public class ResourceInfo implements IResourceInfo { }
public yacyURL getRefererUrl() {
- try {
- return (this.propertyMap == null) ? null : new yacyURL((String)this.propertyMap.get(REFERER), null);
- } catch (MalformedURLException e) {
- return null;
- }
+ return (this.propertyMap == null) ? null : ((yacyURL) this.propertyMap.get(REFERER));
}
public yacyURL getUrl() {
diff --git a/source/de/anomic/plasma/crawler/AbstractCrawlWorker.java b/source/de/anomic/plasma/crawler/AbstractCrawlWorker.java deleted file mode 100644 index d7a8fc4f2..000000000 --- a/source/de/anomic/plasma/crawler/AbstractCrawlWorker.java +++ /dev/null @@ -1,316 +0,0 @@ -// AbstractCrawlWorker.java
-// -------------------------------------
-// part of YACY
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2006
-//
-// This file ist contributed by Martin Thelian
-//
-// $LastChangedDate: 2006-02-20 23:57:42 +0100 (Mo, 20 Feb 2006) $
-// $LastChangedRevision: 1715 $
-// $LastChangedBy: theli $
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
-
-
-package de.anomic.plasma.crawler;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.util.Date;
-
-import de.anomic.plasma.plasmaCrawlEntry;
-import de.anomic.plasma.plasmaCrawlLoaderMessage;
-import de.anomic.plasma.plasmaCrawlProfile;
-import de.anomic.plasma.plasmaCrawlZURL;
-import de.anomic.plasma.plasmaHTCache;
-import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.server.logging.serverLog;
-import de.anomic.yacy.yacyCore;
-import de.anomic.yacy.yacyURL;
-
-public abstract class AbstractCrawlWorker extends Thread implements plasmaCrawlWorker {
-
- /**
- * The protocol that is supported by this crawler
- * e.g. <code>http</code>, <code>ftp</code>, etc.
- */
- protected String protocol;
-
- /* ============================================================
- * Variables for thread pool management
- * ============================================================ */
- public boolean destroyed = false;
- protected boolean running = false;
- protected boolean stopped = false;
- /**
- * Specifies that the execution of the current crawl job has finished
- */
- protected boolean done = false;
-
-
- /* ============================================================
- * Crawl job specific variables
- * ============================================================ */
- public plasmaCrawlLoaderMessage theMsg;
- protected yacyURL url;
- protected String name;
- protected String refererURLString;
- protected String initiator;
- protected int depth;
- protected long startdate;
- protected plasmaCrawlProfile.entry profile;
- protected boolean acceptAllContent;
- protected boolean keepInMemory;
-
- protected String errorMessage;
-
- /**
- * The crawler thread pool
- */
- protected final plasmaCrawlerPool myPool;
-
- /**
- * reference to the plasma switchboard
- */
- protected final plasmaSwitchboard sb;
-
- /**
- * Logging class
- */
- protected final serverLog log;
-
-
- /**
- * Constructor of this class
- * @param theTG the crawl worker thread group
- * @param thePool the crawl worker thread pool
- * @param theSb plasma switchboard
- * @param theCacheManager cache manager
- * @param theLog server log
- */
- public AbstractCrawlWorker(
- ThreadGroup theTG,
- plasmaCrawlerPool thePool,
- plasmaSwitchboard theSb,
- serverLog theLog
- ) {
- super(theTG,plasmaCrawlWorker.threadBaseName + "_created");
-
- this.myPool = thePool;
- this.sb = theSb;
- this.log = theLog;
- }
-
- public void setNameTrailer(String trailer) {
- this.setName(plasmaCrawlWorker.threadBaseName + trailer);
- }
-
- public plasmaCrawlLoaderMessage getMessage() {
- return this.theMsg;
- }
-
- public abstract void close();
-
- public long getDuration() {
- final long startDate = this.startdate;
- return (startDate != 0) ? System.currentTimeMillis() - startDate : 0;
- }
-
- public void run() {
- this.running = true;
-
- try {
- // The thread keeps running.
- while (!this.stopped && !this.isInterrupted()) {
- if (this.done) {
- if (this.myPool != null && !this.myPool.isClosed) {
- synchronized (this) {
- // return thread back into pool
- this.myPool.returnObject(this.protocol,this);
-
- // We are waiting for a new task now.
- if (!this.stopped && !this.destroyed && !this.isInterrupted()) {
- this.wait();
- }
- }
- } else {
- this.stopped = true;
- }
- } else {
- try {
- // executing the new task
- execute();
- } finally {
- // free memory
- reset();
- }
- }
- }
- } catch (InterruptedException ex) {
- serverLog.logFiner("CRAWLER-POOL","Interruption of thread '" + this.getName() + "' detected.");
- } finally {
- if (this.myPool != null && !this.destroyed)
- this.myPool.invalidateObject(this.protocol,this);
- }
- }
-
- public void execute() {
-
- plasmaHTCache.Entry loadedResource = null;
- try {
- // setting threadname
- this.setName(plasmaCrawlWorker.threadBaseName + "_" + this.url);
-
- // load some configuration variables
- init();
-
- // loading resource
- loadedResource = load();
- } catch (IOException e) {
- //throw e;
- } finally {
- // setting the error message (if available)
- if (this.errorMessage != null) {
- this.theMsg.setError(this.errorMessage);
- }
-
- // store a reference to the result in the message object
- // this is e.g. needed by the snippet fetcher
- //
- // Note: this is always called, even on empty results.
- // Otherwise the caller will block forever
- this.theMsg.setResult(loadedResource);
-
- // signal that this worker thread has finished the job
- this.done = true;
- }
- }
-
- public void execute(plasmaCrawlLoaderMessage theNewMsg) {
- synchronized (this) {
-
- this.theMsg = theNewMsg;
-
- this.url = theNewMsg.url;
- this.name = theNewMsg.name;
- this.refererURLString = theNewMsg.referer;
- this.initiator = theNewMsg.initiator;
- this.depth = theNewMsg.depth;
- this.profile = theNewMsg.profile;
- this.acceptAllContent = theNewMsg.acceptAllContent;
- this.keepInMemory = theNewMsg.keepInMemory;
-
- this.startdate = System.currentTimeMillis();
-
- this.done = false;
-
- if (!this.running) {
- // if the thread is not running until yet, we need to start it now
- this.start();
- } else {
- // inform the thread about the new crawl job
- this.notifyAll();
- }
- }
- }
-
- public void setStopped(boolean isStopped) {
- this.stopped = isStopped;
- }
-
- public void setDestroyed(boolean isDestroyed) {
- this.destroyed = isDestroyed;
- }
-
- public boolean isRunning() {
- return this.running;
- }
-
- public void reset() {
- this.theMsg = null;
-
- this.url = null;
- this.name = null;
- this.refererURLString = null;
- this.initiator = null;
- this.depth = 0;
- this.startdate = 0;
- this.profile = null;
- this.acceptAllContent = false;
- this.keepInMemory = false;
-
- this.errorMessage = null;
- }
-
- protected void addURLtoErrorDB(String failreason) {
- // remember error message
- this.errorMessage = failreason;
-
- // convert the referrer URL into a hash value
- String referrerHash;
- try {
- referrerHash = (this.refererURLString == null) ? null : (new yacyURL(this.refererURLString, null)).hash();
- } catch (MalformedURLException e) {
- referrerHash = null;
- }
-
- // create a new errorURL DB entry
- plasmaCrawlEntry bentry = new plasmaCrawlEntry(
- this.initiator,
- this.url,
- referrerHash,
- this.name,
- new Date(),
- this.profile.handle(),
- this.depth,
- 0,
- 0);
- plasmaCrawlZURL.Entry ee = this.sb.errorURL.newEntry(
- bentry, yacyCore.seedDB.mySeed().hash, null,
- 0, (failreason==null)?"Unknown reason":failreason);
-
- // store the entry
- ee.store();
-
- // push it onto the stack
- this.sb.errorURL.stackPushEntry(ee);
-
- // delete the cache file
- File cacheFile = plasmaHTCache.getCachePath(this.url);
- if (cacheFile.exists()) cacheFile.delete();
- }
-}
diff --git a/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java b/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java new file mode 100644 index 000000000..ef4174acb --- /dev/null +++ b/source/de/anomic/plasma/crawler/plasmaCrawlQueues.java @@ -0,0 +1,574 @@ +// plasmaCrawlQueues.java +// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 29.10.2007 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $ +// $LastChangedRevision: 1986 $ +// $LastChangedBy: orbiter $ +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package de.anomic.plasma.crawler; + +import java.io.File; +import java.io.IOException; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; + +import de.anomic.data.robotsParser; +import de.anomic.index.indexURLEntry; +import de.anomic.plasma.plasmaCrawlEntry; +import de.anomic.plasma.plasmaCrawlNURL; +import de.anomic.plasma.plasmaCrawlProfile; +import de.anomic.plasma.plasmaCrawlZURL; +import de.anomic.plasma.plasmaHTCache; +import de.anomic.plasma.plasmaSwitchboard; +import de.anomic.server.logging.serverLog; +import de.anomic.tools.crypt; +import de.anomic.yacy.yacyClient; +import de.anomic.yacy.yacyCore; +import de.anomic.yacy.yacySeed; +import de.anomic.yacy.yacyURL; + +public class plasmaCrawlQueues { + + private plasmaSwitchboard sb; + private serverLog log; + private HashMap workers; // mapping from url hash to Worker thread object + private plasmaProtocolLoader loader; + + public plasmaCrawlNURL noticeURL; + public plasmaCrawlZURL errorURL, delegatedURL; + + public plasmaCrawlQueues(plasmaSwitchboard sb, File plasmaPath) { + this.sb = sb; + this.log = new serverLog("CRAWLER"); + this.workers = new HashMap(); + this.loader = new plasmaProtocolLoader(sb, log); + + // start crawling management + log.logConfig("Starting Crawling Management"); + noticeURL = new plasmaCrawlNURL(plasmaPath); + //errorURL = new plasmaCrawlZURL(); // fresh error DB each startup; can be hold in RAM and reduces IO; + errorURL = new plasmaCrawlZURL(plasmaPath, "urlError1.db", true); + delegatedURL = new plasmaCrawlZURL(plasmaPath, "urlDelegated1.db", false); + + } + + public String urlExists(String hash) { + // tests if hash occurrs in any database + // if it exists, the name of the database is returned, + // if it not exists, null is returned + if (noticeURL.existsInStack(hash)) return "crawler"; + if (delegatedURL.exists(hash)) return "delegated"; + if (errorURL.exists(hash)) return "errors"; + if (workers.containsKey(new Integer(hash.hashCode()))) return "workers"; + return null; + } + + public void urlRemove(String hash) { + noticeURL.removeByURLHash(hash); + delegatedURL.remove(hash); + errorURL.remove(hash); + } + + public yacyURL getURL(String urlhash) { + if (urlhash.equals(yacyURL.dummyHash)) return null; + plasmaCrawlEntry ne = (plasmaCrawlEntry) workers.get(new Integer(urlhash.hashCode())); + if (ne != null) return ne.url(); + ne = noticeURL.get(urlhash); + if (ne != null) return ne.url(); + plasmaCrawlZURL.Entry ee = delegatedURL.getEntry(urlhash); + if (ee != null) return ee.url(); + ee = errorURL.getEntry(urlhash); + if (ee != null) return ee.url(); + return null; + } + + public void close() { + // wait for all workers to finish + Iterator i = workers.values().iterator(); + while (i.hasNext()) ((Thread) i.next()).interrupt(); + // TODO: wait some more time until all threads are finished + } + + public plasmaCrawlEntry[] activeWorker() { + synchronized (workers) { + plasmaCrawlEntry[] w = new plasmaCrawlEntry[workers.size()]; + int i = 0; + Iterator j = workers.values().iterator(); + while (j.hasNext()) { + w[i++] = ((crawlWorker) j.next()).entry; + } + return w; + } + } + + public boolean isSupportedProtocol(String protocol) { + return loader.isSupportedProtocol(protocol); + } + + public int coreCrawlJobSize() { + return noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE); + } + + public boolean coreCrawlJob() { + if (noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) == 0) { + //log.logDebug("CoreCrawl: queue is empty"); + return false; + } + if (sb.sbQueue.size() >= plasmaSwitchboard.indexingSlots) { + log.logFine("CoreCrawl: too many processes in indexing queue, dismissed (" + + "sbQueueSize=" + sb.sbQueue.size() + ")"); + return false; + } + if (this.size() >= plasmaSwitchboard.crawlSlots) { + log.logFine("CoreCrawl: too many processes in loader queue, dismissed (" + + "cacheLoader=" + this.size() + ")"); + return false; + } + if (sb.onlineCaution()) { + log.logFine("CoreCrawl: online caution, omitting processing"); + return false; + } + // if the server is busy, we do crawling more slowly + //if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {} + + // if crawling was paused we have to wait until we wer notified to continue + Object[] status = (Object[]) sb.crawlJobsStatus.get(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL); + synchronized(status[plasmaSwitchboard.CRAWLJOB_SYNC]) { + if (((Boolean)status[plasmaSwitchboard.CRAWLJOB_STATUS]).booleanValue()) { + try { + status[plasmaSwitchboard.CRAWLJOB_SYNC].wait(); + } + catch (InterruptedException e){ return false;} + } + } + + // do a local crawl + plasmaCrawlEntry urlEntry = null; + while (urlEntry == null && noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) > 0) { + String stats = "LOCALCRAWL[" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_OVERHANG) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE) + "]"; + try { + urlEntry = noticeURL.pop(plasmaCrawlNURL.STACK_TYPE_CORE, true); + String profileHandle = urlEntry.profileHandle(); + // System.out.println("DEBUG plasmaSwitchboard.processCrawling: + // profileHandle = " + profileHandle + ", urlEntry.url = " + urlEntry.url()); + if (profileHandle == null) { + log.logSevere(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); + return true; + } + plasmaCrawlProfile.entry profile = sb.profilesActiveCrawls.getEntry(profileHandle); + if (profile == null) { + log.logWarning(stats + ": LOST PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); + return true; + } + + // check if the protocol is supported + yacyURL url = urlEntry.url(); + String urlProtocol = url.getProtocol(); + if (!this.sb.crawlQueues.isSupportedProtocol(urlProtocol)) { + this.log.logSevere("Unsupported protocol in URL '" + url.toString()); + return true; + } + + log.logFine("LOCALCRAWL: URL=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter=" + profile.generalFilter() + + ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false"))); + + processLocalCrawling(urlEntry, stats); + return true; + } catch (IOException e) { + log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e); + if (e.getMessage().indexOf("hash is null") > 0) noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_CORE); + } + } + return true; + } + + + public int limitCrawlTriggerJobSize() { + return noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT); + } + + public boolean limitCrawlTriggerJob() { + if (noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) == 0) { + //log.logDebug("LimitCrawl: queue is empty"); + return false; + } + boolean robinsonPrivateCase = ((sb.isRobinsonMode()) && + (!sb.getConfig(plasmaSwitchboard.CLUSTER_MODE, "").equals(plasmaSwitchboard.CLUSTER_MODE_PUBLIC_CLUSTER)) && + (!sb.getConfig(plasmaSwitchboard.CLUSTER_MODE, "").equals(plasmaSwitchboard.CLUSTER_MODE_PRIVATE_CLUSTER))); + + if ((robinsonPrivateCase) || ((coreCrawlJobSize() <= 20) && (limitCrawlTriggerJobSize() > 10))) { + // it is not efficient if the core crawl job is empty and we have too much to do + // move some tasks to the core crawl job + int toshift = 10; // this cannot be a big number because the balancer makes a forced waiting if it cannot balance + if (toshift > limitCrawlTriggerJobSize()) toshift = limitCrawlTriggerJobSize(); + for (int i = 0; i < toshift; i++) { + noticeURL.shift(plasmaCrawlNURL.STACK_TYPE_LIMIT, plasmaCrawlNURL.STACK_TYPE_CORE); + } + log.logInfo("shifted " + toshift + " jobs from global crawl to local crawl (coreCrawlJobSize()=" + coreCrawlJobSize() + ", limitCrawlTriggerJobSize()=" + limitCrawlTriggerJobSize() + ", cluster.mode=" + sb.getConfig(plasmaSwitchboard.CLUSTER_MODE, "") + ", robinsonMode=" + ((sb.isRobinsonMode()) ? "on" : "off")); + if (robinsonPrivateCase) return false; + } + + // check local indexing queues + // in case the placing of remote crawl fails, there must be space in the local queue to work off the remote crawl + if (sb.sbQueue.size() >= plasmaSwitchboard.indexingSlots * 2) { + log.logFine("LimitCrawl: too many processes in indexing queue, dismissed (" + + "sbQueueSize=" + sb.sbQueue.size() + ")"); + return false; + } + if (this.size() >= plasmaSwitchboard.crawlSlots) { + log.logFine("LimitCrawl: too many processes in loader queue, dismissed (" + + "cacheLoader=" + this.size() + ")"); + return false; + } + if (sb.onlineCaution()) { + log.logFine("LimitCrawl: online caution, omitting processing"); + return false; + } + + // if crawling was paused we have to wait until we were notified to continue + Object[] status = (Object[]) sb.crawlJobsStatus.get(plasmaSwitchboard.CRAWLJOB_GLOBAL_CRAWL_TRIGGER); + synchronized(status[plasmaSwitchboard.CRAWLJOB_SYNC]) { + if (((Boolean)status[plasmaSwitchboard.CRAWLJOB_STATUS]).booleanValue()) { + try { + status[plasmaSwitchboard.CRAWLJOB_SYNC].wait(); + } + catch (InterruptedException e){ return false;} + } + } + + // start a global crawl, if possible + String stats = "REMOTECRAWLTRIGGER[" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_OVERHANG) + ", " + + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE) + "]"; + try { + plasmaCrawlEntry urlEntry = noticeURL.pop(plasmaCrawlNURL.STACK_TYPE_LIMIT, true); + String profileHandle = urlEntry.profileHandle(); + // System.out.println("DEBUG plasmaSwitchboard.processCrawling: + // profileHandle = " + profileHandle + ", urlEntry.url = " + urlEntry.url()); + plasmaCrawlProfile.entry profile = sb.profilesActiveCrawls.getEntry(profileHandle); + if (profile == null) { + log.logWarning(stats + ": LOST PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); + return true; + } + + // check if the protocol is supported + yacyURL url = urlEntry.url(); + String urlProtocol = url.getProtocol(); + if (!this.sb.crawlQueues.isSupportedProtocol(urlProtocol)) { + this.log.logSevere("Unsupported protocol in URL '" + url.toString()); + return true; + } + + log.logFine("plasmaSwitchboard.limitCrawlTriggerJob: url=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter=" + + profile.generalFilter() + ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false"))); + + boolean tryRemote = ((noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) != 0) || (sb.sbQueue.size() != 0)) && + (profile.remoteIndexing()) && + (urlEntry.initiator() != null) && + // (!(urlEntry.initiator().equals(indexURL.dummyHash))) && + ((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())); + if (tryRemote) { + // checking robots.txt for http(s) resources + if ((urlProtocol.equals("http") || urlProtocol.equals("https")) && robotsParser.isDisallowed(url)) { + this.log.logFine("Crawling of URL '" + url.toString() + "' disallowed by robots.txt."); + return true; + } + boolean success = processRemoteCrawlTrigger(urlEntry); + if (success) return true; + } + + processLocalCrawling(urlEntry, stats); // emergency case, work off the crawl locally + return true; + } catch (IOException e) { + log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e); + if (e.getMessage().indexOf("hash is null") > 0) noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_LIMIT); + return true; // if we return a false here we will block everything + } + } + + public int remoteTriggeredCrawlJobSize() { + return noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE); + } + + public boolean remoteTriggeredCrawlJob() { + // work off crawl requests that had been placed by other peers to our crawl stack + + // do nothing if either there are private processes to be done + // or there is no global crawl on the stack + if (noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE) == 0) { + //log.logDebug("GlobalCrawl: queue is empty"); + return false; + } + if (sb.sbQueue.size() >= plasmaSwitchboard.indexingSlots) { + log.logFine("GlobalCrawl: too many processes in indexing queue, dismissed (" + + "sbQueueSize=" + sb.sbQueue.size() + ")"); + return false; + } + if (this.size() >= plasmaSwitchboard.crawlSlots) { + log.logFine("GlobalCrawl: too many processes in loader queue, dismissed (" + + "cacheLoader=" + this.size() + ")"); + return false; + } + if (sb.onlineCaution()) { + log.logFine("GlobalCrawl: online caution, omitting processing"); + return false; + } + + // if crawling was paused we have to wait until we wer notified to continue + Object[] status = (Object[]) sb.crawlJobsStatus.get(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL); + synchronized(status[plasmaSwitchboard.CRAWLJOB_SYNC]) { + if (((Boolean)status[plasmaSwitchboard.CRAWLJOB_STATUS]).booleanValue()) { + try { + status[plasmaSwitchboard.CRAWLJOB_SYNC].wait(); + } + catch (InterruptedException e){ return false;} + } + } + + // we don't want to crawl a global URL globally, since WE are the global part. (from this point of view) + String stats = "REMOTETRIGGEREDCRAWL[" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_OVERHANG) + ", " + + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE) + "]"; + try { + plasmaCrawlEntry urlEntry = noticeURL.pop(plasmaCrawlNURL.STACK_TYPE_REMOTE, true); + String profileHandle = urlEntry.profileHandle(); + // System.out.println("DEBUG plasmaSwitchboard.processCrawling: + // profileHandle = " + profileHandle + ", urlEntry.url = " + + // urlEntry.url()); + plasmaCrawlProfile.entry profile = sb.profilesActiveCrawls.getEntry(profileHandle); + + if (profile == null) { + log.logWarning(stats + ": LOST PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url()); + return false; + } + + // check if the protocol is supported + yacyURL url = urlEntry.url(); + String urlProtocol = url.getProtocol(); + if (!this.sb.crawlQueues.isSupportedProtocol(urlProtocol)) { + this.log.logSevere("Unsupported protocol in URL '" + url.toString()); + return true; + } + + log.logFine("plasmaSwitchboard.remoteTriggeredCrawlJob: url=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter=" + + profile.generalFilter() + ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false"))); + + processLocalCrawling(urlEntry, stats); + return true; + } catch (IOException e) { + log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e); + if (e.getMessage().indexOf("hash is null") > 0) noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_REMOTE); + return true; + } + } + + private void processLocalCrawling(plasmaCrawlEntry entry, String stats) { + // work off one Crawl stack entry + if ((entry == null) || (entry.url() == null)) { + log.logInfo(stats + ": urlEntry = null"); + return; + } + + synchronized (this.workers) { + crawlWorker w = new crawlWorker(entry); + synchronized (workers) { + workers.put(new Integer(entry.hashCode()), w); + } + } + + log.logInfo(stats + ": enqueued for load " + entry.url() + " [" + entry.url().hash() + "]"); + return; + } + + private boolean processRemoteCrawlTrigger(plasmaCrawlEntry urlEntry) { + // if this returns true, then the urlEntry is considered as stored somewhere and the case is finished + // if this returns false, the urlEntry will be enqueued to the local crawl again + + // wrong access + if (urlEntry == null) { + log.logInfo("REMOTECRAWLTRIGGER[" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE) + "]: urlEntry=null"); + return true; // superfluous request; true correct in this context because the urlEntry shall not be tracked any more + } + + // check url + if (urlEntry.url() == null) { + log.logFine("ERROR: plasmaSwitchboard.processRemoteCrawlTrigger - url is null. name=" + urlEntry.name()); + return true; // same case as above: no more consideration + } + + // are we qualified for a remote crawl? + if ((yacyCore.seedDB.mySeed() == null) || (yacyCore.seedDB.mySeed().isJunior())) { + log.logFine("plasmaSwitchboard.processRemoteCrawlTrigger: no permission"); + return false; // no, we must crawl this page ourselves + } + + // check if peer for remote crawl is available + yacySeed remoteSeed = ((sb.isPublicRobinson()) && (sb.getConfig("cluster.mode", "").equals("publiccluster"))) ? + yacyCore.dhtAgent.getPublicClusterCrawlSeed(urlEntry.url().hash(), sb.clusterhashes) : + yacyCore.dhtAgent.getGlobalCrawlSeed(urlEntry.url().hash()); + if (remoteSeed == null) { + log.logFine("plasmaSwitchboard.processRemoteCrawlTrigger: no remote crawl seed available"); + return false; + } + + // do the request + HashMap page = yacyClient.crawlOrder(remoteSeed, urlEntry.url(), sb.getURL(urlEntry.referrerhash()), 6000); + if (page == null) { + log.logSevere(plasmaSwitchboard.STR_REMOTECRAWLTRIGGER + remoteSeed.getName() + " FAILED. URL CANNOT BE RETRIEVED from referrer hash: " + urlEntry.referrerhash()); + return false; + } + + // check if we got contact to peer and the peer respondet + if ((page == null) || (page.get("delay") == null)) { + log.logInfo("CRAWL: REMOTE CRAWL TO PEER " + remoteSeed.getName() + " FAILED. CAUSE: unknown (URL=" + urlEntry.url().toString() + "). Removed peer."); + yacyCore.peerActions.peerDeparture(remoteSeed, "remote crawl to peer failed; peer answered unappropriate"); + return false; // no response from peer, we will crawl this ourself + } + + String response = (String) page.get("response"); + log.logFine("plasmaSwitchboard.processRemoteCrawlTrigger: remoteSeed=" + + remoteSeed.getName() + ", url=" + urlEntry.url().toString() + + ", response=" + page.toString()); // DEBUG + + // we received an answer and we are told to wait a specific time until we shall ask again for another crawl + int newdelay = Integer.parseInt((String) page.get("delay")); + yacyCore.dhtAgent.setCrawlDelay(remoteSeed.hash, newdelay); + if (response.equals("stacked")) { + // success, the remote peer accepted the crawl + log.logInfo(plasmaSwitchboard.STR_REMOTECRAWLTRIGGER + remoteSeed.getName() + + " PLACED URL=" + urlEntry.url().toString() + + "; NEW DELAY=" + newdelay); + // track this remote crawl + delegatedURL.newEntry(urlEntry, remoteSeed.hash, new Date(), 0, response).store(); + return true; + } + + // check other cases: the remote peer may respond that it already knows that url + if (response.equals("double")) { + // in case the peer answers double, it transmits the complete lurl data + String lurl = (String) page.get("lurl"); + if ((lurl != null) && (lurl.length() != 0)) { + String propStr = crypt.simpleDecode(lurl, (String) page.get("key")); + indexURLEntry entry = sb.wordIndex.loadedURL.newEntry(propStr); + try { + sb.wordIndex.loadedURL.store(entry); + sb.wordIndex.loadedURL.stack(entry, yacyCore.seedDB.mySeed().hash, remoteSeed.hash, 1); // *** ueberfluessig/doppelt? + // noticeURL.remove(entry.hash()); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + log.logInfo(plasmaSwitchboard.STR_REMOTECRAWLTRIGGER + remoteSeed.getName() + + " SUPERFLUOUS. CAUSE: " + page.get("reason") + + " (URL=" + urlEntry.url().toString() + + "). URL IS CONSIDERED AS 'LOADED!'"); + return true; + } else { + log.logInfo(plasmaSwitchboard.STR_REMOTECRAWLTRIGGER + remoteSeed.getName() + + " REJECTED. CAUSE: bad lurl response / " + page.get("reason") + " (URL=" + + urlEntry.url().toString() + ")"); + remoteSeed.setFlagAcceptRemoteCrawl(false); + yacyCore.seedDB.update(remoteSeed.hash, remoteSeed); + return false; + } + } + + log.logInfo(plasmaSwitchboard.STR_REMOTECRAWLTRIGGER + remoteSeed.getName() + + " DENIED. RESPONSE=" + response + ", CAUSE=" + + page.get("reason") + ", URL=" + urlEntry.url().toString()); + remoteSeed.setFlagAcceptRemoteCrawl(false); + yacyCore.seedDB.update(remoteSeed.hash, remoteSeed); + return false; + } + + public plasmaHTCache.Entry loadResourceFromWeb( + yacyURL url, + int socketTimeout, + boolean keepInMemory, + boolean forText + ) { + + plasmaCrawlEntry centry = new plasmaCrawlEntry( + yacyCore.seedDB.mySeed().hash, + url, + null, + "", + new Date(), + (forText) ? sb.defaultTextSnippetProfile.handle() : sb.defaultMediaSnippetProfile.handle(), // crawl profile + 0, + 0, + 0); + + return loader.load(centry); + } + + public int size() { + return workers.size(); + } + + protected class crawlWorker extends Thread { + + public plasmaCrawlEntry entry; + + public crawlWorker(plasmaCrawlEntry entry) { + this.entry = entry; + this.entry.setStatus("worker-initialized"); + this.start(); + } + + public void run() { + try { + // checking robots.txt for http(s) resources + this.entry.setStatus("worker-checkingrobots"); + if ((entry.url().getProtocol().equals("http") || entry.url().getProtocol().equals("https")) && robotsParser.isDisallowed(entry.url())) { + log.logFine("Crawling of URL '" + entry.url().toString() + "' disallowed by robots.txt."); + plasmaCrawlZURL.Entry eentry = errorURL.newEntry(this.entry.url(), "denied by robots.txt"); + eentry.store(); + errorURL.push(eentry); + } else { + // starting a load from the internet + this.entry.setStatus("worker-loading"); + String result = loader.process(this.entry); + if (result != null) { + plasmaCrawlZURL.Entry eentry = errorURL.newEntry(this.entry.url(), "cannot load: " + result); + eentry.store(); + errorURL.push(eentry); + } else { + this.entry.setStatus("worker-processed"); + } + } + } catch (Exception e) { + plasmaCrawlZURL.Entry eentry = errorURL.newEntry(this.entry.url(), e.getMessage() + " - in worker"); + eentry.store(); + errorURL.push(eentry); + e.printStackTrace(); + } finally { + synchronized (workers) { + workers.remove(new Integer(entry.hashCode())); + } + this.entry.setStatus("worker-finalized"); + } + } + + } + +} diff --git a/source/de/anomic/plasma/crawler/plasmaCrawlWorker.java b/source/de/anomic/plasma/crawler/plasmaCrawlWorker.java deleted file mode 100644 index 0eb0f0d79..000000000 --- a/source/de/anomic/plasma/crawler/plasmaCrawlWorker.java +++ /dev/null @@ -1,74 +0,0 @@ -// plasmaCrawlWorker.java
-// -------------------------------------
-// part of YACY
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2006
-//
-// This file ist contributed by Martin Thelian
-//
-// $LastChangedDate: 2006-02-20 23:57:42 +0100 (Mo, 20 Feb 2006) $
-// $LastChangedRevision: 1715 $
-// $LastChangedBy: theli $
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
-
-
-package de.anomic.plasma.crawler;
-
-import java.io.IOException;
-
-import de.anomic.plasma.plasmaCrawlLoaderMessage;
-import de.anomic.plasma.plasmaHTCache;
-
-
-public interface plasmaCrawlWorker {
-
- public static final String threadBaseName = "CrawlerWorker";
-
- public void setNameTrailer(String trailer);
-
- public void setStopped(boolean isStopped);
- public void setDestroyed(boolean isDestroyed);
-
- public plasmaCrawlLoaderMessage getMessage();
-
- public void reset();
- public void execute();
- public void execute(plasmaCrawlLoaderMessage theNewMsg);
- public void init();
-
- public void close();
- public plasmaHTCache.Entry load() throws IOException;
-}
diff --git a/source/de/anomic/plasma/crawler/plasmaCrawlerException.java b/source/de/anomic/plasma/crawler/plasmaCrawlerException.java deleted file mode 100644 index 4cd925d4f..000000000 --- a/source/de/anomic/plasma/crawler/plasmaCrawlerException.java +++ /dev/null @@ -1,12 +0,0 @@ -package de.anomic.plasma.crawler;
-
-import java.io.IOException;
-
-public class plasmaCrawlerException extends IOException {
-
- private static final long serialVersionUID = 1L;
-
- public plasmaCrawlerException(String errorMsg) {
- super(errorMsg);
- }
-}
diff --git a/source/de/anomic/plasma/crawler/plasmaCrawlerFactory.java b/source/de/anomic/plasma/crawler/plasmaCrawlerFactory.java deleted file mode 100644 index e56cb3e6f..000000000 --- a/source/de/anomic/plasma/crawler/plasmaCrawlerFactory.java +++ /dev/null @@ -1,172 +0,0 @@ -// plasmaCrawlerFactory.java
-// -------------------------------------
-// part of YACY
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2006
-//
-// This file ist contributed by Martin Thelian
-//
-// $LastChangedDate: 2006-02-20 23:57:42 +0100 (Mo, 20 Feb 2006) $
-// $LastChangedRevision: 1715 $
-// $LastChangedBy: theli $
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
-
-package de.anomic.plasma.crawler;
-
-import java.lang.reflect.Constructor;
-
-import org.apache.commons.pool.KeyedPoolableObjectFactory;
-
-import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.server.logging.serverLog;
-
-public final class plasmaCrawlerFactory implements KeyedPoolableObjectFactory {
-
- private plasmaCrawlerPool thePool;
- private final ThreadGroup theThreadGroup;
- private final serverLog theLog;
- private final plasmaSwitchboard sb;
-
- public plasmaCrawlerFactory(
- ThreadGroup threadGroup,
- plasmaSwitchboard theSb,
- serverLog log
- ) {
-
- super();
-
- if (threadGroup == null)
- throw new IllegalArgumentException("The threadgroup object must not be null.");
-
- this.theThreadGroup = threadGroup;
- this.sb = theSb;
- this.theLog = log;
- }
-
- public void setPool(plasmaCrawlerPool pool) {
- this.thePool = pool;
- }
-
- public Object makeObject(Object key) throws Exception {
- return makeObject(key, true);
- }
-
- /**
- * @see org.apache.commons.pool.PoolableObjectFactory#makeObject()
- */
- public Object makeObject(Object key, boolean usePool) throws Exception {
- if (!(key instanceof String))
- throw new IllegalArgumentException("The object key must be of type string.");
-
- // getting the class name
- String className = this.getClass().getPackage().getName() + "." + key + ".CrawlWorker";
-
- // loading class by name
- Class moduleClass = Class.forName(className);
-
- // getting the constructor
- Constructor classConstructor = moduleClass.getConstructor( new Class[] {
- ThreadGroup.class,
- plasmaCrawlerPool.class,
- plasmaSwitchboard.class,
- serverLog.class
- } );
-
- // instantiating class
- plasmaCrawlWorker theCrawlWorker = (plasmaCrawlWorker) classConstructor.newInstance(new Object[] {
- this.theThreadGroup,
- (usePool)?this.thePool:null,
- this.sb,
- this.theLog
- });
-
- // return the newly created object
- return theCrawlWorker;
-
-// return new plasmaCrawlWorker(
-// this.theThreadGroup,
-// this.thePool,
-// this.sb,
-// this.cacheManager,
-// this.theLog);
- }
-
- /**
- * @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object)
- */
- public void destroyObject(Object key, Object obj) {
- if (obj == null) return;
- if (obj instanceof plasmaCrawlWorker) {
- plasmaCrawlWorker theWorker = (plasmaCrawlWorker) obj;
- synchronized(theWorker) {
- theWorker.setDestroyed(true);
- theWorker.setNameTrailer("_destroyed");
- theWorker.setStopped(true);
- ((Thread)theWorker).interrupt();
- }
- }
- }
-
- /**
- * @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object)
- */
- public boolean validateObject(Object key, Object obj) {
- return true;
- }
-
- /**
- * @param obj
- *
- */
- public void activateObject(Object key, Object obj) {
- //log.debug(" activateObject...");
- }
-
- /**
- * @param obj
- *
- */
-
- public void passivateObject(Object key, Object obj) {
- //log.debug(" passivateObject..." + obj);
- /*
- if (obj instanceof plasmaCrawlWorker) {
- plasmaCrawlWorker theWorker = (plasmaCrawlWorker) obj;
- }
- */
- }
-
-}
diff --git a/source/de/anomic/plasma/crawler/plasmaCrawlerMsgQueue.java b/source/de/anomic/plasma/crawler/plasmaCrawlerMsgQueue.java deleted file mode 100644 index b263835ef..000000000 --- a/source/de/anomic/plasma/crawler/plasmaCrawlerMsgQueue.java +++ /dev/null @@ -1,127 +0,0 @@ -// plasmaCrawlerMsgQueue.java
-// -------------------------------------
-// part of YACY
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2006
-//
-// This file ist contributed by Martin Thelian
-//
-// $LastChangedDate: 2006-02-20 23:57:42 +0100 (Mo, 20 Feb 2006) $
-// $LastChangedRevision: 1715 $
-// $LastChangedBy: theli $
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
-
-package de.anomic.plasma.crawler;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-
-import de.anomic.plasma.plasmaCrawlLoaderMessage;
-import de.anomic.server.serverSemaphore;
-
-public class plasmaCrawlerMsgQueue {
- private final serverSemaphore readSync;
- private final serverSemaphore writeSync;
- private final ArrayList messageList;
-
- public plasmaCrawlerMsgQueue() {
- this.readSync = new serverSemaphore (0);
- this.writeSync = new serverSemaphore (1);
-
- this.messageList = new ArrayList(10);
- }
-
- /**
- *
- * @param newMessage
- * @throws MessageQueueLockedException
- * @throws InterruptedException
- */
- public void addMessage(plasmaCrawlLoaderMessage newMessage)
- throws InterruptedException, NullPointerException
- {
- if (newMessage == null) throw new NullPointerException();
-
- this.writeSync.P();
-
- boolean insertionDoneSuccessfully = false;
- synchronized(this.messageList) {
- insertionDoneSuccessfully = this.messageList.add(newMessage);
- }
-
- if (insertionDoneSuccessfully) {
- this.sortMessages();
- this.readSync.V();
- }
-
- this.writeSync.V();
- }
-
- public plasmaCrawlLoaderMessage waitForMessage() throws InterruptedException {
- this.readSync.P();
- this.writeSync.P();
-
- plasmaCrawlLoaderMessage newMessage = null;
- synchronized(this.messageList) {
- newMessage = (plasmaCrawlLoaderMessage) this.messageList.remove(0);
- }
-
- this.writeSync.V();
- return newMessage;
- }
-
- protected void sortMessages() {
- Collections.sort(this.messageList, new Comparator() {
- public int compare(Object o1, Object o2)
- {
- plasmaCrawlLoaderMessage message1 = (plasmaCrawlLoaderMessage) o1;
- plasmaCrawlLoaderMessage message2 = (plasmaCrawlLoaderMessage) o2;
-
- int message1Priority = message1.crawlingPriority;
- int message2Priority = message2.crawlingPriority;
-
- if (message1Priority > message2Priority){
- return -1;
- } else if (message1Priority < message2Priority) {
- return 1;
- } else {
- return 0;
- }
- }
- });
- }
-}
\ No newline at end of file diff --git a/source/de/anomic/plasma/crawler/plasmaCrawlerPool.java b/source/de/anomic/plasma/crawler/plasmaCrawlerPool.java deleted file mode 100644 index 7b52106ee..000000000 --- a/source/de/anomic/plasma/crawler/plasmaCrawlerPool.java +++ /dev/null @@ -1,158 +0,0 @@ -// plasmaCrawlerPool.java
-// -------------------------------------
-// part of YACY
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2006
-//
-// This file ist contributed by Martin Thelian
-//
-// $LastChangedDate: 2006-02-20 23:57:42 +0100 (Mo, 20 Feb 2006) $
-// $LastChangedRevision: 1715 $
-// $LastChangedBy: theli $
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
-
-
-package de.anomic.plasma.crawler;
-
-import org.apache.commons.pool.impl.GenericKeyedObjectPool;
-
-import de.anomic.server.logging.serverLog;
-
-public final class plasmaCrawlerPool extends GenericKeyedObjectPool {
-
- private plasmaCrawlerFactory theFactory;
- private final ThreadGroup theThreadGroup;
- public boolean isClosed = false;
-
- public plasmaCrawlerPool(plasmaCrawlerFactory objFactory, GenericKeyedObjectPool.Config config, ThreadGroup threadGroup) {
- super(objFactory, config);
- this.theFactory = objFactory;
- this.theThreadGroup = threadGroup;
- objFactory.setPool(this);
- }
-
- public plasmaCrawlerFactory getFactory() {
- return this.theFactory;
- }
-
- public Object borrowObject(Object key) throws Exception {
- return super.borrowObject(key);
- }
-
- public void returnObject(Object key,Object obj) {
- if (obj == null) return;
- if (obj instanceof plasmaCrawlWorker) {
- try {
- ((plasmaCrawlWorker)obj).setNameTrailer("_inPool");
- super.returnObject(key,obj);
- } catch (Exception e) {
- ((plasmaCrawlWorker)obj).setStopped(true);
- serverLog.logSevere("CRAWLER-POOL","Unable to return crawler thread to pool.",e);
- }
- } else {
- serverLog.logSevere("CRAWLER-POOL","Object of wrong type '" + obj.getClass().getName() +
- "' returned to pool.");
- }
- }
-
- public void invalidateObject(Object key,Object obj) {
- if (obj == null) return;
- if (this.isClosed) return;
- if (obj instanceof plasmaCrawlWorker) {
- try {
- ((plasmaCrawlWorker)obj).setNameTrailer("_invalidated");
- ((plasmaCrawlWorker)obj).setStopped(true);
- super.invalidateObject(key,obj);
- } catch (Exception e) {
- serverLog.logSevere("CRAWLER-POOL","Unable to invalidate crawling thread.",e);
- }
- }
- }
-
- public synchronized void close() throws Exception {
- try {
- /*
- * shutdown all still running session threads ...
- */
- this.isClosed = true;
-
- /* waiting for all threads to finish */
- int threadCount = this.theThreadGroup.activeCount();
- Thread[] threadList = new Thread[threadCount];
- threadCount = this.theThreadGroup.enumerate(threadList);
-
- // signaling shutdown to all still running or pooled threads ...
- serverLog.logInfo("CRAWLER","Signaling shutdown to " + threadCount + " remaining crawler threads ...");
- for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) {
- ((plasmaCrawlWorker)threadList[currentThreadIdx]).setStopped(true);
- }
-
- // giving the crawlers some time to finish shutdown
- try { Thread.sleep(500); } catch(Exception e) {/* Ignore this. Shutdown in progress */}
-
- // sending interrupted signal to all remaining threads
- serverLog.logInfo("CRAWLER","Sending interruption signal to " + this.theThreadGroup.activeCount() + " remaining crawler threads ...");
- this.theThreadGroup.interrupt();
-
- // aborting all crawlers by closing all still open httpc sockets
- serverLog.logInfo("CRAWLER","Trying to abort " + this.theThreadGroup.activeCount() + " remaining crawler threads ...");
- for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) {
- Thread currentThread = threadList[currentThreadIdx];
- if (currentThread.isAlive()) {
- serverLog.logInfo("CRAWLER","Trying to shutdown crawler thread '" + currentThread.getName() + "' [" + currentThreadIdx + "].");
- ((plasmaCrawlWorker)currentThread).close();
- }
- }
-
- serverLog.logInfo("CRAWLER","Waiting for " + this.theThreadGroup.activeCount() + " remaining crawler threads to finish shutdown ...");
- for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) {
- Thread currentThread = threadList[currentThreadIdx];
- if (currentThread.isAlive()) {
- serverLog.logInfo("CRAWLER","Waiting for crawler thread '" + currentThread.getName() + "' [" + currentThreadIdx + "] to finish shutdown.");
- try { currentThread.join(500); } catch (InterruptedException ex) {/* Ignore this. Shutdown in progress */}
- }
- }
- serverLog.logWarning("CRAWLER","Shutdown of remaining crawler threads finish.");
- }
- catch (Exception e) {
- serverLog.logWarning("CRAWLER","Unexpected error while trying to shutdown all remaining crawler threads.",e);
- }
-
- super.close();
-
- }
-
-}
diff --git a/source/de/anomic/plasma/crawler/ftp/CrawlWorker.java b/source/de/anomic/plasma/crawler/plasmaFTPLoader.java index fbaaefcbc..004c902aa 100644 --- a/source/de/anomic/plasma/crawler/ftp/CrawlWorker.java +++ b/source/de/anomic/plasma/crawler/plasmaFTPLoader.java @@ -45,68 +45,53 @@ // Contributions and changes to the program code must be marked as such.
-package de.anomic.plasma.crawler.ftp;
+package de.anomic.plasma.crawler;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.Date;
import de.anomic.net.ftpc;
import de.anomic.plasma.plasmaCrawlEURL;
+import de.anomic.plasma.plasmaCrawlEntry;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaSwitchboard;
-import de.anomic.plasma.cache.IResourceInfo;
import de.anomic.plasma.cache.ftp.ResourceInfo;
-import de.anomic.plasma.crawler.AbstractCrawlWorker;
-import de.anomic.plasma.crawler.plasmaCrawlWorker;
-import de.anomic.plasma.crawler.plasmaCrawlerPool;
-import de.anomic.plasma.plasmaHTCache.Entry;
import de.anomic.server.logging.serverLog;
-import de.anomic.yacy.yacyURL;
-public class CrawlWorker extends AbstractCrawlWorker implements plasmaCrawlWorker {
+public class plasmaFTPLoader {
- public CrawlWorker(ThreadGroup theTG, plasmaCrawlerPool thePool, plasmaSwitchboard theSb, serverLog theLog) {
- super(theTG, thePool, theSb, theLog);
-
- // this crawler supports ftp
- this.protocol = "ftp";
- }
-
- public void close() {
- // TODO: abort a currently established connection
- }
-
- public void init() {
- // nothing todo here
+ private plasmaSwitchboard sb;
+ private serverLog log;
+
+ public plasmaFTPLoader(plasmaSwitchboard sb, serverLog log) {
+ this.sb = sb;
+ this.log = log;
}
- protected plasmaHTCache.Entry createCacheEntry(String mimeType, Date fileDate) {
- IResourceInfo resourceInfo = new ResourceInfo(
- this.url,
- this.refererURLString,
- mimeType,
- fileDate
- );
-
+ protected plasmaHTCache.Entry createCacheEntry(plasmaCrawlEntry entry, String mimeType, Date fileDate) {
return plasmaHTCache.newEntry(
new Date(),
- this.depth,
- this.url,
- this.name,
+ entry.depth(),
+ entry.url(),
+ entry.name(),
"OK",
- resourceInfo,
- this.initiator,
- this.profile
+ new ResourceInfo(
+ entry.url(),
+ sb.getURL(entry.referrerhash()),
+ mimeType,
+ fileDate
+ ),
+ entry.initiator(),
+ sb.profilesActiveCrawls.getEntry(entry.profileHandle())
);
- }
-
- public Entry load() throws IOException {
+ }
+
+ public plasmaHTCache.Entry load(plasmaCrawlEntry entry) {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
PrintStream out = new PrintStream(bout);
@@ -118,7 +103,7 @@ public class CrawlWorker extends AbstractCrawlWorker implements plasmaCrawlWorke ftpc ftpClient = new ftpc(System.in, out, err);
// get username and password
- String userInfo = this.url.getUserInfo();
+ String userInfo = entry.url().getUserInfo();
String userName = "anonymous", userPwd = "anonymous";
if (userInfo != null) {
int pos = userInfo.indexOf(":");
@@ -129,9 +114,9 @@ public class CrawlWorker extends AbstractCrawlWorker implements plasmaCrawlWorke }
// get server name, port and file path
- String host = this.url.getHost();
- String fullPath = this.url.getPath();
- int port = this.url.getPort();
+ String host = entry.url().getHost();
+ String fullPath = entry.url().getPath();
+ int port = entry.url().getPort();
plasmaHTCache.Entry htCache = null;
try {
@@ -141,27 +126,12 @@ public class CrawlWorker extends AbstractCrawlWorker implements plasmaCrawlWorke } else {
ftpClient.exec("open " + host + " " + port, false);
}
- if (berr.size() > 0) {
- this.log.logWarning("Unable to connect to ftp server " + this.url.getHost() + " hosting URL " + this.url.toString() + "\nErrorlog: " + berr.toString());
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_CONNECTION_ERROR);
- return null;
- }
// login to the server
- ftpClient.exec("user " + userName + " " + userPwd, false);
- if (berr.size() > 0) {
- this.log.logWarning("Unable to login to ftp server " + this.url.getHost() + " hosting URL " + this.url.toString() + "\nErrorlog: " + berr.toString());
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_SERVER_LOGIN_FAILED);
- return null;
- }
+ ftpClient.exec("user " + userName + " " + userPwd, false);
// change transfer mode to binary
ftpClient.exec("binary", false);
- if (berr.size() > 0) {
- this.log.logWarning("Unable to set the file transfer mode to binary for URL " + this.url.toString() + "\nErrorlog: " + berr.toString());
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_SERVER_TRASFER_MODE_PROBLEM);
- return null;
- }
// determine filename and path
String file, path;
@@ -188,12 +158,11 @@ public class CrawlWorker extends AbstractCrawlWorker implements plasmaCrawlWorke if (isFolder) {
fullPath = fullPath + "/";
file = "";
- this.url = yacyURL.newURL(this.url,fullPath);
}
}
// creating a cache file object
- File cacheFile = plasmaHTCache.getCachePath(this.url);
+ File cacheFile = plasmaHTCache.getCachePath(entry.url());
// TODO: aborting download if content is to long ...
@@ -202,7 +171,7 @@ public class CrawlWorker extends AbstractCrawlWorker implements plasmaCrawlWorke // testing if the file already exists
if (cacheFile.isFile()) {
// delete the file if it already exists
- plasmaHTCache.deleteFile(this.url);
+ plasmaHTCache.deleteURLfromCache(entry.url());
} else {
// create parent directories
cacheFile.getParentFile().mkdirs();
@@ -216,7 +185,7 @@ public class CrawlWorker extends AbstractCrawlWorker implements plasmaCrawlWorke fileDate = new Date();
// create a htcache entry
- htCache = createCacheEntry(mimeType,fileDate);
+ htCache = createCacheEntry(entry, mimeType, fileDate);
// generate the dirlist
StringBuffer dirList = ftpClient.dirhtml(fullPath);
@@ -228,22 +197,22 @@ public class CrawlWorker extends AbstractCrawlWorker implements plasmaCrawlWorke writer.flush();
writer.close();
} catch (Exception e) {
- this.log.logInfo("Unable to write dirlist for URL " + this.url.toString());
+ this.log.logInfo("Unable to write dirlist for URL " + entry.url().toString());
htCache = null;
}
} else {
// determine the mimetype of the resource
- String extension = plasmaParser.getFileExt(this.url);
+ String extension = plasmaParser.getFileExt(entry.url());
mimeType = plasmaParser.getMimeTypeByFileExt(extension);
// if the mimetype and file extension is supported we start to download the file
- if ((this.acceptAllContent) || (plasmaParser.supportedContent(plasmaParser.PARSER_MODE_CRAWLER,this.url,mimeType))) {
+ if (plasmaParser.supportedContent(plasmaParser.PARSER_MODE_CRAWLER, entry.url(), mimeType)) {
// TODO: determine the real file date
fileDate = new Date();
// create a htcache entry
- htCache = createCacheEntry(mimeType,fileDate);
+ htCache = createCacheEntry(entry, mimeType, fileDate);
// change into working directory
ftpClient.exec("cd \"" + fullPath + "\"", false);
@@ -252,8 +221,8 @@ public class CrawlWorker extends AbstractCrawlWorker implements plasmaCrawlWorke ftpClient.exec("get \"" + file + "\" \"" + cacheFile.getAbsolutePath() + "\"", false);
} else {
// if the response has not the right file type then reject file
- this.log.logInfo("REJECTED WRONG MIME/EXT TYPE " + mimeType + " for URL " + this.url.toString());
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT);
+ this.log.logInfo("REJECTED WRONG MIME/EXT TYPE " + mimeType + " for URL " + entry.url().toString());
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT);
return null;
}
}
@@ -261,19 +230,14 @@ public class CrawlWorker extends AbstractCrawlWorker implements plasmaCrawlWorke // pass the downloaded resource to the cache manager
if (berr.size() > 0 || htCache == null) {
// if the response has not the right file type then reject file
- this.log.logWarning("Unable to download URL " + this.url.toString() + "\nErrorlog: " + berr.toString());
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR);
+ this.log.logWarning("Unable to download URL " + entry.url().toString() + "\nErrorlog: " + berr.toString());
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_DOWNLOAD_ERROR);
// an error has occured. cleanup
if (cacheFile.exists()) cacheFile.delete();
} else {
// announce the file
- plasmaHTCache.writeFileAnnouncement(cacheFile);
-
- // enQueue new entry with response header
- if (this.profile != null) {
- plasmaHTCache.push(htCache);
- }
+ plasmaHTCache.writeFileAnnouncement(cacheFile);
}
return htCache;
diff --git a/source/de/anomic/plasma/crawler/http/CrawlWorker.java b/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java index d02d044cd..dd225b421 100644 --- a/source/de/anomic/plasma/crawler/http/CrawlWorker.java +++ b/source/de/anomic/plasma/crawler/plasmaHTTPLoader.java @@ -42,7 +42,7 @@ //the intact and unchanged copyright notice.
//Contributions and changes to the program code must be marked as such.
-package de.anomic.plasma.crawler.http;
+package de.anomic.plasma.crawler;
import java.io.File;
import java.io.FileOutputStream;
@@ -61,20 +61,18 @@ import de.anomic.http.httpdBoundedSizeOutputStream; import de.anomic.http.httpdLimitExceededException;
import de.anomic.http.httpdProxyHandler;
import de.anomic.plasma.plasmaCrawlEURL;
-import de.anomic.plasma.plasmaCrawlLoader;
+import de.anomic.plasma.plasmaCrawlEntry;
import de.anomic.plasma.plasmaHTCache;
import de.anomic.plasma.plasmaParser;
import de.anomic.plasma.plasmaSwitchboard;
import de.anomic.plasma.cache.IResourceInfo;
import de.anomic.plasma.cache.http.ResourceInfo;
-import de.anomic.plasma.crawler.AbstractCrawlWorker;
-import de.anomic.plasma.crawler.plasmaCrawlerPool;
import de.anomic.plasma.urlPattern.plasmaURLPattern;
import de.anomic.server.serverSystem;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyURL;
-public final class CrawlWorker extends AbstractCrawlWorker {
+public final class plasmaHTTPLoader {
public static final int DEFAULT_CRAWLING_RETRY_COUNT = 5;
@@ -96,86 +94,69 @@ public final class CrawlWorker extends AbstractCrawlWorker { private String acceptEncoding;
private String acceptLanguage;
private String acceptCharset;
+ private plasmaSwitchboard sb;
+ private serverLog log;
- /**
- * Constructor of this class
- * @param theTG
- * @param thePool
- * @param theSb
- * @param theLog
- */
- public CrawlWorker(
- ThreadGroup theTG,
- plasmaCrawlerPool thePool,
- plasmaSwitchboard theSb,
- serverLog theLog) {
- super(theTG,thePool,theSb,theLog);
-
- // this crawler supports http
- this.protocol = "http";
- }
-
- public void init() {
+ public plasmaHTTPLoader(plasmaSwitchboard sb, serverLog theLog) {
+ this.sb = sb;
+ this.log = theLog;
+
// refreshing timeout value
- if (this.theMsg.timeout < 0) {
- this.socketTimeout = (int) this.sb.getConfigLong("crawler.clientTimeout", 10000);
- } else {
- this.socketTimeout = this.theMsg.timeout;
- }
+ this.socketTimeout = (int) sb.getConfigLong("crawler.clientTimeout", 10000);
// maximum allowed file size
- this.maxFileSize = this.sb.getConfigLong("crawler.http.maxFileSize", -1);
+ this.maxFileSize = sb.getConfigLong("crawler.http.maxFileSize", -1);
// some http header values
- this.acceptEncoding = this.sb.getConfig("crawler.http.acceptEncoding", "gzip,deflate");
- this.acceptLanguage = this.sb.getConfig("crawler.http.acceptLanguage","en-us,en;q=0.5");
- this.acceptCharset = this.sb.getConfig("crawler.http.acceptCharset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
+ this.acceptEncoding = sb.getConfig("crawler.http.acceptEncoding", "gzip,deflate");
+ this.acceptLanguage = sb.getConfig("crawler.http.acceptLanguage","en-us,en;q=0.5");
+ this.acceptCharset = sb.getConfig("crawler.http.acceptCharset","ISO-8859-1,utf-8;q=0.7,*;q=0.7");
// getting the http proxy config
- this.remoteProxyConfig = this.sb.remoteProxyConfig;
+ this.remoteProxyConfig = sb.remoteProxyConfig;
}
-
- public plasmaHTCache.Entry load() throws IOException {
- return load(DEFAULT_CRAWLING_RETRY_COUNT);
- }
- protected plasmaHTCache.Entry createCacheEntry(yacyURL requestUrl, Date requestDate, httpHeader requestHeader, httpc.response response) {
- IResourceInfo resourceInfo = new ResourceInfo(requestUrl,requestHeader,response.responseHeader);
+ protected plasmaHTCache.Entry createCacheEntry(plasmaCrawlEntry entry, Date requestDate, httpHeader requestHeader, httpc.response response) {
+ IResourceInfo resourceInfo = new ResourceInfo(entry.url(), requestHeader, response.responseHeader);
return plasmaHTCache.newEntry(
requestDate,
- this.depth,
- this.url,
- this.name,
+ entry.depth(),
+ entry.url(),
+ entry.name(),
response.status,
resourceInfo,
- this.initiator,
- this.profile
+ entry.initiator(),
+ sb.profilesActiveCrawls.getEntry(entry.profileHandle())
);
}
+
+ public plasmaHTCache.Entry load(plasmaCrawlEntry entry) {
+ return load(entry, DEFAULT_CRAWLING_RETRY_COUNT);
+ }
- private plasmaHTCache.Entry load(int crawlingRetryCount) throws IOException {
-
- // if the recrawling limit was exceeded we stop crawling now
- if (crawlingRetryCount <= 0) return null;
+ private plasmaHTCache.Entry load(plasmaCrawlEntry entry, int retryCount) {
+ if (retryCount < 0) {
+ this.log.logInfo("Redirection counter exceeded for URL " + entry.url().toString() + ". Processing aborted.");
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_COUNTER_EXCEEDED).store();
+ return null;
+ }
+
Date requestDate = new Date(); // remember the time...
- String host = this.url.getHost();
- String path = this.url.getFile();
- int port = this.url.getPort();
- boolean ssl = this.url.getProtocol().equals("https");
+ String host = entry.url().getHost();
+ String path = entry.url().getFile();
+ int port = entry.url().getPort();
+ boolean ssl = entry.url().getProtocol().equals("https");
if (port < 0) port = (ssl) ? 443 : 80;
// check if url is in blacklist
String hostlow = host.toLowerCase();
if (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER, hostlow, path)) {
- this.log.logInfo("CRAWLER Rejecting URL '" + this.url.toString() + "'. URL is in blacklist.");
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_URL_IN_BLACKLIST);
+ this.log.logInfo("CRAWLER Rejecting URL '" + entry.url().toString() + "'. URL is in blacklist.");
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_URL_IN_BLACKLIST).store();
return null;
}
-
- // TODO: resolve yacy and yacyh domains
- //String yAddress = yacyCore.seedDB.resolveYacyAddress(host);
-
+
// take a file from the net
httpc remote = null;
plasmaHTCache.Entry htCache = null;
@@ -183,8 +164,10 @@ public final class CrawlWorker extends AbstractCrawlWorker { // create a request header
httpHeader requestHeader = new httpHeader();
requestHeader.put(httpHeader.USER_AGENT, httpdProxyHandler.crawlerUserAgent);
- if (this.refererURLString != null && this.refererURLString.length() > 0)
- requestHeader.put(httpHeader.REFERER, this.refererURLString);
+ yacyURL refererURL = null;
+ if (entry.referrerhash() != null) refererURL = sb.getURL(entry.referrerhash());
+ if (refererURL != null)
+ requestHeader.put(httpHeader.REFERER, refererURL.toNormalform(true, true));
if (this.acceptLanguage != null && this.acceptLanguage.length() > 0)
requestHeader.put(httpHeader.ACCEPT_LANGUAGE, this.acceptLanguage);
if (this.acceptCharset != null && this.acceptCharset.length() > 0)
@@ -205,13 +188,13 @@ public final class CrawlWorker extends AbstractCrawlWorker { // the transfer is ok
// create a new cache entry
- htCache = createCacheEntry(this.url,requestDate, requestHeader, res);
+ htCache = createCacheEntry(entry, requestDate, requestHeader, res);
// aborting download if content is to long ...
if (htCache.cacheFile().getAbsolutePath().length() > serverSystem.maxPathLength) {
remote.close();
- this.log.logInfo("REJECTED URL " + this.url.toString() + " because path too long '" + plasmaHTCache.cachePath.getAbsolutePath() + "'");
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_CACHEFILE_PATH_TOO_LONG);
+ this.log.logInfo("REJECTED URL " + entry.url().toString() + " because path too long '" + plasmaHTCache.cachePath.getAbsolutePath() + "'");
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_CACHEFILE_PATH_TOO_LONG);
return (htCache = null);
}
@@ -219,20 +202,20 @@ public final class CrawlWorker extends AbstractCrawlWorker { if (!htCache.cacheFile().getCanonicalPath().startsWith(plasmaHTCache.cachePath.getCanonicalPath())) {
// if the response has not the right file type then reject file
remote.close();
- this.log.logInfo("REJECTED URL " + this.url.toString() + " because of an invalid file path ('" +
+ this.log.logInfo("REJECTED URL " + entry.url().toString() + " because of an invalid file path ('" +
htCache.cacheFile().getCanonicalPath() + "' does not start with '" +
plasmaHTCache.cachePath.getAbsolutePath() + "').");
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_INVALID_CACHEFILE_PATH);
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_INVALID_CACHEFILE_PATH);
return (htCache = null);
}
// request has been placed and result has been returned. work off response
- File cacheFile = plasmaHTCache.getCachePath(this.url);
+ File cacheFile = plasmaHTCache.getCachePath(entry.url());
try {
- if ((this.acceptAllContent) || (plasmaParser.supportedContent(plasmaParser.PARSER_MODE_CRAWLER,this.url,res.responseHeader.mime()))) {
+ if (plasmaParser.supportedContent(plasmaParser.PARSER_MODE_CRAWLER,entry.url(),res.responseHeader.mime())) {
// delete old content
if (cacheFile.isFile()) {
- plasmaHTCache.deleteFile(this.url);
+ plasmaHTCache.deleteURLfromCache(entry.url());
}
// create parent directories
@@ -246,31 +229,21 @@ public final class CrawlWorker extends AbstractCrawlWorker { // getting content length
long contentLength = (res.isGzipped()) ? res.getGzippedLength() : res.responseHeader.contentLength();
- // check if the file is too large to keep it in memory
- if (this.keepInMemory) {
- // if the content length is unknown or larger than 5MB we
- // do not keep resource in memory
- // TODO: make MAX_KEEP_IN_MEMORY_SIZE configureble
- if ((contentLength == -1) || (contentLength > 5 * 1024 * 1024)) {
- this.keepInMemory = false;
- }
- }
-
// check the maximum allowed file size
if (this.maxFileSize > -1) {
if (contentLength == -1) {
fos = new httpdBoundedSizeOutputStream(fos,this.maxFileSize);
} else if (contentLength > this.maxFileSize) {
remote.close();
- this.log.logInfo("REJECTED URL " + this.url + " because file size '" + contentLength + "' exceeds max filesize limit of " + this.maxFileSize + " bytes.");
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_FILESIZE_LIMIT_EXCEEDED);
+ this.log.logInfo("REJECTED URL " + entry.url() + " because file size '" + contentLength + "' exceeds max filesize limit of " + this.maxFileSize + " bytes.");
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_FILESIZE_LIMIT_EXCEEDED);
return null;
}
}
// we write the new cache entry to file system directly
byte[] cacheArray = null;
- cacheArray = res.writeContent(fos,this.keepInMemory);
+ cacheArray = res.writeContent(fos, false);
remote.close();
htCache.setCacheArray(cacheArray);
plasmaHTCache.writeFileAnnouncement(cacheFile);
@@ -279,16 +252,13 @@ public final class CrawlWorker extends AbstractCrawlWorker { remote.close();
}
- // enQueue new entry with response header
- if (this.profile != null) {
- plasmaHTCache.push(htCache);
- }
+ return htCache;
} else {
// if the response has not the right file type then reject file
remote.close();
- this.log.logInfo("REJECTED WRONG MIME/EXT TYPE " + res.responseHeader.mime() + " for URL " + this.url.toString());
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT);
- htCache = null;
+ this.log.logInfo("REJECTED WRONG MIME/EXT TYPE " + res.responseHeader.mime() + " for URL " + entry.url().toString());
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_MIMETYPE_OR_EXT);
+ return null;
}
} catch (SocketException e) {
// this may happen if the client suddenly closes its connection
@@ -297,34 +267,33 @@ public final class CrawlWorker extends AbstractCrawlWorker { // but we clean the cache also, since it may be only partial
// and most possible corrupted
if (cacheFile.exists()) cacheFile.delete();
- this.log.logSevere("CRAWLER LOADER ERROR1: with URL=" + this.url.toString() + ": " + e.toString());
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_CONNECTION_ERROR);
+ this.log.logSevere("CRAWLER LOADER ERROR1: with URL=" + entry.url().toString() + ": " + e.toString());
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_CONNECTION_ERROR);
htCache = null;
}
} else if (res.status.startsWith("30")) {
- if (crawlingRetryCount > 0) {
if (res.responseHeader.containsKey(httpHeader.LOCATION)) {
// getting redirection URL
String redirectionUrlString = (String) res.responseHeader.get(httpHeader.LOCATION);
redirectionUrlString = redirectionUrlString.trim();
if (redirectionUrlString.length() == 0) {
- this.log.logWarning("CRAWLER Redirection of URL=" + this.url.toString() + " aborted. Location header is empty.");
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_REDIRECTION_HEADER_EMPTY);
+ this.log.logWarning("CRAWLER Redirection of URL=" + entry.url().toString() + " aborted. Location header is empty.");
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_HEADER_EMPTY);
return null;
}
// normalizing URL
- yacyURL redirectionUrl = yacyURL.newURL(this.url, redirectionUrlString);
+ yacyURL redirectionUrl = yacyURL.newURL(entry.url(), redirectionUrlString);
// restart crawling with new url
- this.log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + this.url.toString());
+ this.log.logInfo("CRAWLER Redirection detected ('" + res.status + "') for URL " + entry.url().toString());
this.log.logInfo("CRAWLER ..Redirecting request to: " + redirectionUrl);
// if we are already doing a shutdown we don't need to retry crawling
if (Thread.currentThread().isInterrupted()) {
- this.log.logSevere("CRAWLER Retry of URL=" + this.url.toString() + " aborted because of server shutdown.");
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_SERVER_SHUTDOWN);
+ this.log.logSevere("CRAWLER Retry of URL=" + entry.url().toString() + " aborted because of server shutdown.");
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_SERVER_SHUTDOWN);
return null;
}
@@ -332,48 +301,29 @@ public final class CrawlWorker extends AbstractCrawlWorker { String urlhash = redirectionUrl.hash();
// check if the url was already indexed
- String dbname = plasmaCrawlLoader.switchboard.urlExists(urlhash);
+ String dbname = sb.urlExists(urlhash);
if (dbname != null) {
- this.log.logWarning("CRAWLER Redirection of URL=" + this.url.toString() + " ignored. The url appears already in db " + dbname);
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_REDIRECTION_TO_DOUBLE_CONTENT);
+ this.log.logWarning("CRAWLER Redirection of URL=" + entry.url().toString() + " ignored. The url appears already in db " + dbname);
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_REDIRECTION_TO_DOUBLE_CONTENT);
return null;
}
// retry crawling with new url
- this.url = redirectionUrl;
- plasmaHTCache.Entry redirectedEntry = load(crawlingRetryCount-1);
+ entry.redirectURL(redirectionUrl);
+ return load(entry, retryCount - 1);
- if (redirectedEntry != null) {
-// TODO: Here we can store the content of the redirection
-// as content of the original URL if some criterias are met
-//
-// plasmaHTCache.Entry newEntry = (plasmaHTCache.Entry) redirectedEntry.clone();
-// newEntry.url = url;
-// TODO: which http header should we store here?
-//
-// // enQueue new entry with response header
-// if (profile != null) {
-// cacheManager.push(newEntry);
-// }
-// htCache = newEntry;
- }
}
- } else {
- this.log.logInfo("Redirection counter exceeded for URL " + this.url.toString() + ". Processing aborted.");
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_REDIRECTION_COUNTER_EXCEEDED);
- }
- }else {
+ } else {
// if the response has not the right response type then reject file
- this.log.logInfo("REJECTED WRONG STATUS TYPE '" + res.status + "' for URL " + this.url.toString());
+ this.log.logInfo("REJECTED WRONG STATUS TYPE '" + res.status + "' for URL " + entry.url().toString());
// not processed any further
- addURLtoErrorDB(plasmaCrawlEURL.DENIED_WRONG_HTTP_STATUSCODE + res.statusCode + ")");
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, plasmaCrawlEURL.DENIED_WRONG_HTTP_STATUSCODE + res.statusCode + ")");
}
if (remote != null) remote.close();
return htCache;
} catch (Exception e) {
- boolean retryCrawling = false;
String errorMsg = e.getMessage();
String failreason = null;
@@ -385,95 +335,70 @@ public final class CrawlWorker extends AbstractCrawlWorker { this.log.logInfo("CRAWLER Interruption detected because of server shutdown.");
failreason = plasmaCrawlEURL.DENIED_SERVER_SHUTDOWN;
} else if (e instanceof httpdLimitExceededException) {
- this.log.logWarning("CRAWLER Max file size limit '" + this.maxFileSize + "' exceeded while downloading URL " + this.url);
+ this.log.logWarning("CRAWLER Max file size limit '" + this.maxFileSize + "' exceeded while downloading URL " + entry.url());
failreason = plasmaCrawlEURL.DENIED_FILESIZE_LIMIT_EXCEEDED;
} else if (e instanceof MalformedURLException) {
- this.log.logWarning("CRAWLER Malformed URL '" + this.url.toString() + "' detected. ");
+ this.log.logWarning("CRAWLER Malformed URL '" + entry.url().toString() + "' detected. ");
failreason = plasmaCrawlEURL.DENIED_MALFORMED_URL;
} else if (e instanceof NoRouteToHostException) {
- this.log.logWarning("CRAWLER No route to host found while trying to crawl URL '" + this.url.toString() + "'.");
+ this.log.logWarning("CRAWLER No route to host found while trying to crawl URL '" + entry.url().toString() + "'.");
failreason = plasmaCrawlEURL.DENIED_NO_ROUTE_TO_HOST;
} else if ((e instanceof UnknownHostException) ||
((errorMsg != null) && (errorMsg.indexOf("unknown host") >= 0))) {
- this.log.logWarning("CRAWLER Unknown host in URL '" + this.url.toString() + "'. " +
- "Referer URL: " + ((this.refererURLString == null) ?"Unknown":this.refererURLString));
+ this.log.logWarning("CRAWLER Unknown host in URL '" + entry.url().toString() + "'. " +
+ "Referer URL: " + ((entry.referrerhash() == null) ? "Unknown" : sb.getURL(entry.referrerhash()).toNormalform(true, true)));
failreason = plasmaCrawlEURL.DENIED_UNKNOWN_HOST;
} else if (e instanceof java.net.BindException) {
- this.log.logWarning("CRAWLER BindException detected while trying to download content from '" + this.url.toString() +
+ this.log.logWarning("CRAWLER BindException detected while trying to download content from '" + entry.url().toString() +
"'. Retrying request.");
- failreason = plasmaCrawlEURL.DENIED_CONNECTION_BIND_EXCEPTION;
- retryCrawling = true;
+ failreason = plasmaCrawlEURL.DENIED_CONNECTION_BIND_EXCEPTION;
} else if ((errorMsg != null) && (
(errorMsg.indexOf("Corrupt GZIP trailer") >= 0) ||
(errorMsg.indexOf("Not in GZIP format") >= 0) ||
(errorMsg.indexOf("Unexpected end of ZLIB") >= 0)
)) {
- this.log.logWarning("CRAWLER Problems detected while receiving gzip encoded content from '" + this.url.toString() +
+ this.log.logWarning("CRAWLER Problems detected while receiving gzip encoded content from '" + entry.url().toString() +
"'. Retrying request without using gzip content encoding.");
failreason = plasmaCrawlEURL.DENIED_CONTENT_DECODING_ERROR;
this.acceptEncoding = null;
- retryCrawling = true;
} else if ((errorMsg != null) && (errorMsg.indexOf("Read timed out") >= 0)) {
- this.log.logWarning("CRAWLER Read timeout while receiving content from '" + this.url.toString() +
+ this.log.logWarning("CRAWLER Read timeout while receiving content from '" + entry.url().toString() +
"'. Retrying request.");
failreason = plasmaCrawlEURL.DENIED_CONNECTION_TIMEOUT;
- retryCrawling = true;
} else if ((errorMsg != null) && (errorMsg.indexOf("connect timed out") >= 0)) {
- this.log.logWarning("CRAWLER Timeout while trying to connect to '" + this.url.toString() +
+ this.log.logWarning("CRAWLER Timeout while trying to connect to '" + entry.url().toString() +
"'. Retrying request.");
failreason = plasmaCrawlEURL.DENIED_CONNECTION_TIMEOUT;
- retryCrawling = true;
} else if ((errorMsg != null) && (errorMsg.indexOf("Connection timed out") >= 0)) {
- this.log.logWarning("CRAWLER Connection timeout while receiving content from '" + this.url.toString() +
+ this.log.logWarning("CRAWLER Connection timeout while receiving content from '" + entry.url().toString() +
"'. Retrying request.");
failreason = plasmaCrawlEURL.DENIED_CONNECTION_TIMEOUT;
- retryCrawling = true;
} else if ((errorMsg != null) && (errorMsg.indexOf("Connection refused") >= 0)) {
- this.log.logWarning("CRAWLER Connection refused while trying to connect to '" + this.url.toString() + "'.");
+ this.log.logWarning("CRAWLER Connection refused while trying to connect to '" + entry.url().toString() + "'.");
failreason = plasmaCrawlEURL.DENIED_CONNECTION_REFUSED;
} else if ((errorMsg != null) && (errorMsg.indexOf("There is not enough space on the disk") >= 0)) {
- this.log.logSevere("CRAWLER Not enough space on the disk detected while crawling '" + this.url.toString() + "'. " +
+ this.log.logSevere("CRAWLER Not enough space on the disk detected while crawling '" + entry.url().toString() + "'. " +
"Pausing crawlers. ");
- plasmaCrawlLoader.switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
- plasmaCrawlLoader.switchboard.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
+ sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_LOCAL_CRAWL);
+ sb.pauseCrawlJob(plasmaSwitchboard.CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
failreason = plasmaCrawlEURL.DENIED_OUT_OF_DISK_SPACE;
} else if ((errorMsg != null) && (errorMsg.indexOf("Network is unreachable") >=0)) {
- this.log.logSevere("CRAWLER Network is unreachable while trying to crawl URL '" + this.url.toString() + "'. ");
+ this.log.logSevere("CRAWLER Network is unreachable while trying to crawl URL '" + entry.url().toString() + "'. ");
failreason = plasmaCrawlEURL.DENIED_NETWORK_IS_UNREACHABLE;
} else if ((errorMsg != null) && (errorMsg.indexOf("No trusted certificate found")>= 0)) {
- this.log.logSevere("CRAWLER No trusted certificate found for URL '" + this.url.toString() + "'. ");
+ this.log.logSevere("CRAWLER No trusted certificate found for URL '" + entry.url().toString() + "'. ");
failreason = plasmaCrawlEURL.DENIED_SSL_UNTRUSTED_CERT;
} else {
- this.log.logSevere("CRAWLER Unexpected Error with URL '" + this.url.toString() + "': " + e.toString(), e);
+ this.log.logSevere("CRAWLER Unexpected Error with URL '" + entry.url().toString() + "': " + e.toString(), e);
failreason = plasmaCrawlEURL.DENIED_CONNECTION_ERROR;
}
- if (retryCrawling) {
- // if we are already doing a shutdown we don't need to retry crawling
- if (Thread.currentThread().isInterrupted()) {
- this.log.logSevere("CRAWLER Retry of URL=" + this.url.toString() + " aborted because of server shutdown.");
- return null;
- }
-
- // setting the retry counter to 1
- if (crawlingRetryCount > 2) crawlingRetryCount = 2;
-
- // retry crawling
- return load(crawlingRetryCount - 1);
- }
if (failreason != null) {
// add url into error db
- addURLtoErrorDB(failreason);
+ sb.crawlQueues.errorURL.newEntry(entry, null, new Date(), 1, failreason);
}
return null;
}
}
- public void close() {
- if (this.isAlive()) {
- try {
- // TODO: this object should care of all open clien connections within this class and close them here
- } catch (Exception e) {/* ignore this. shutdown in progress */}
- }
- }
}
diff --git a/source/de/anomic/plasma/crawler/plasmaProtocolLoader.java b/source/de/anomic/plasma/crawler/plasmaProtocolLoader.java new file mode 100644 index 000000000..1cbb3645c --- /dev/null +++ b/source/de/anomic/plasma/crawler/plasmaProtocolLoader.java @@ -0,0 +1,97 @@ +// plasmaProtocolLoader.java
+// (C) 2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
+// first published 24.10.2007 on http://yacy.net
+//
+// This is a part of YaCy, a peer-to-peer based web search engine
+//
+// $LastChangedDate: 2006-04-02 22:40:07 +0200 (So, 02 Apr 2006) $
+// $LastChangedRevision: 1986 $
+// $LastChangedBy: orbiter $
+//
+// LICENSE
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation; either version 2 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+package de.anomic.plasma.crawler;
+
+import java.util.Arrays;
+import java.util.HashSet;
+
+import de.anomic.plasma.plasmaCrawlEntry;
+import de.anomic.plasma.plasmaHTCache;
+import de.anomic.plasma.plasmaSwitchboard;
+import de.anomic.server.logging.serverLog;
+
+public final class plasmaProtocolLoader {
+
+ private plasmaSwitchboard sb;
+ private serverLog log;
+ private HashSet supportedProtocols;
+ private plasmaHTTPLoader httpLoader;
+ private plasmaFTPLoader ftpLoader;
+
+ public plasmaProtocolLoader(plasmaSwitchboard sb, serverLog log) {
+ this.sb = sb;
+ this.log = log;
+ this.supportedProtocols = new HashSet(Arrays.asList(new String[]{"http","https"/* ,"ftp" */}));
+
+ // initiate loader objects
+ httpLoader = new plasmaHTTPLoader(sb, log);
+ ftpLoader = new plasmaFTPLoader(sb, log);
+ }
+
+ public boolean isSupportedProtocol(String protocol) {
+ if ((protocol == null) || (protocol.length() == 0)) return false;
+ return this.supportedProtocols.contains(protocol.trim().toLowerCase());
+ }
+
+ public HashSet getSupportedProtocols() {
+ return (HashSet) this.supportedProtocols.clone();
+ }
+
+ public plasmaHTCache.Entry load(plasmaCrawlEntry entry) {
+ // getting the protocol of the next URL
+ String protocol = entry.url().getProtocol();
+
+ if ((protocol.equals("http") || (protocol.equals("https")))) return httpLoader.load(entry);
+ if (protocol.equals("ftp")) return ftpLoader.load(entry);
+
+ this.log.logWarning("Unsupported protocol '" + protocol + "' in url " + entry.url());
+ return null;
+ }
+
+ public String process(plasmaCrawlEntry entry) {
+ // load a resource, store it to htcache and push queue entry to switchboard queue
+ // returns null if everything went fine, a fail reason string if a problem occurred
+ plasmaHTCache.Entry h;
+ try {
+ h = load(entry);
+ entry.setStatus("loaded");
+ if (h == null) return "load failed";
+ boolean stored = sb.htEntryStoreProcess(h);
+ entry.setStatus("stored-" + ((stored) ? "ok" : "fail"));
+ return (stored) ? null : "not stored";
+ } catch (Exception e) {
+ log.logWarning("problem loading " + entry.url().toString(), e);
+ return "load error - " + e.getMessage();
+ }
+ }
+
+}
+
+
+
+
+
diff --git a/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java b/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java index ae8b3fbce..e42487099 100644 --- a/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java +++ b/source/de/anomic/plasma/dbImport/plasmaCrawlNURLImporter.java @@ -187,8 +187,8 @@ public class plasmaCrawlNURLImporter extends AbstractImporter implements dbImpor }
// if the url does not alredy exists in the destination stack we insert it now
- if (!this.sb.noticeURL.existsInStack(nextHash)) {
- this.sb.noticeURL.push((stackTypes[stackType] != -1) ? stackTypes[stackType] : plasmaCrawlNURL.STACK_TYPE_CORE, nextEntry);
+ if (!this.sb.crawlQueues.noticeURL.existsInStack(nextHash)) {
+ this.sb.crawlQueues.noticeURL.push((stackTypes[stackType] != -1) ? stackTypes[stackType] : plasmaCrawlNURL.STACK_TYPE_CORE, nextEntry);
}
// removing hash from the import db
diff --git a/source/de/anomic/plasma/plasmaCrawlEntry.java b/source/de/anomic/plasma/plasmaCrawlEntry.java index 98521dbf3..b86101862 100644 --- a/source/de/anomic/plasma/plasmaCrawlEntry.java +++ b/source/de/anomic/plasma/plasmaCrawlEntry.java @@ -76,7 +76,9 @@ public class plasmaCrawlEntry { private int forkfactor; // sum of anchors of all ancestors private kelondroBitfield flags; private int handle; - + private String status; + private int initialHash; // to provide a object hash that does not change even if the url changes because of redirection + public plasmaCrawlEntry(yacyURL url) { this(yacyCore.seedDB.mySeed().hash, url, null, null, new Date(), null, 0, 0, 0); } @@ -105,6 +107,8 @@ public class plasmaCrawlEntry { ) { // create new entry and store it into database assert appdate != null; + assert url != null; + if ((initiator == null) || (initiator.length() == 0)) initiator = yacyURL.dummyHash; this.initiator = initiator; this.url = url; this.referrer = (referrer == null) ? yacyURL.dummyHash : referrer; @@ -119,6 +123,8 @@ public class plasmaCrawlEntry { this.loaddate = 0; this.serverdate = 0; this.imsdate = 0; + this.status = "loaded(args)"; + this.initialHash = url.hashCode(); } public plasmaCrawlEntry(kelondroRow.Entry entry) throws IOException { @@ -143,9 +149,24 @@ public class plasmaCrawlEntry { this.loaddate = entry.getColLong(12); this.serverdate = entry.getColLong(13); this.imsdate = entry.getColLong(14); + this.status = "loaded(kelondroRow.Entry)"; + this.initialHash = url.hashCode(); return; } + public int hashCode() { + // overloads Object.hashCode() + return this.initialHash; + } + + public void setStatus(String s) { + this.status = s; + } + + public String getStatus() { + return this.status; + } + private static String normalizeHandle(int h) { String d = Integer.toHexString(h); while (d.length() < rowdef.width(11)) d = "0" + d; @@ -187,6 +208,11 @@ public class plasmaCrawlEntry { // the url return url; } + + public void redirectURL(yacyURL redirectedURL) { + // replace old URL by new one. This should only be used in case of url redirection + this.url = redirectedURL; + } public String referrerhash() { // the urlhash of a referer url diff --git a/source/de/anomic/plasma/plasmaCrawlLoader.java b/source/de/anomic/plasma/plasmaCrawlLoader.java deleted file mode 100644 index 328d6ec94..000000000 --- a/source/de/anomic/plasma/plasmaCrawlLoader.java +++ /dev/null @@ -1,322 +0,0 @@ -// plasmaCrawlerLoader.java
-// ------------------------
-// part of YaCy
-// (C) by Michael Peter Christen; mc@anomic.de
-// first published on http://www.anomic.de
-// Frankfurt, Germany, 2004
-//
-// $LastChangedDate$
-// $LastChangedRevision$
-// $LastChangedBy$
-//
-// This program is free software; you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with this program; if not, write to the Free Software
-// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-//
-// Using this software in any meaning (reading, learning, copying, compiling,
-// running) means that you agree that the Author(s) is (are) not responsible
-// for cost, loss of data or any harm that may be caused directly or indirectly
-// by usage of this softare or this documentation. The usage of this software
-// is on your own risk. The installation and usage (starting/running) of this
-// software may allow other people or application to access your computer and
-// any attached devices and is highly dependent on the configuration of the
-// software which must be done by the user of the software; the author(s) is
-// (are) also not responsible for proper configuration and usage of the
-// software, even if provoked by documentation provided together with
-// the software.
-//
-// Any changes to this file according to the GPL as documented in the file
-// gpl.txt aside this file in the shipment you received can be done to the
-// lines that follows this copyright notice here, but changes must not be
-// done inside the copyright notive above. A re-distribution must contain
-// the intact and unchanged copyright notice.
-// Contributions and changes to the program code must be marked as such.
-
-package de.anomic.plasma;
-
-import java.util.Arrays;
-import java.util.HashSet;
-
-import org.apache.commons.pool.impl.GenericKeyedObjectPool;
-import org.apache.commons.pool.impl.GenericObjectPool;
-
-import de.anomic.plasma.crawler.plasmaCrawlWorker;
-import de.anomic.plasma.crawler.plasmaCrawlerException;
-import de.anomic.plasma.crawler.plasmaCrawlerFactory;
-import de.anomic.plasma.crawler.plasmaCrawlerMsgQueue;
-import de.anomic.plasma.crawler.plasmaCrawlerPool;
-import de.anomic.server.logging.serverLog;
-import de.anomic.yacy.yacyURL;
-
-public final class plasmaCrawlLoader extends Thread {
-
- public static plasmaSwitchboard switchboard;
-
- private final serverLog log;
-
- private HashSet supportedProtocols;
-
- private final plasmaCrawlerMsgQueue theQueue;
- private final plasmaCrawlerPool crawlwerPool;
- private GenericKeyedObjectPool.Config crawlerPoolConfig = null;
- private final ThreadGroup theThreadGroup = new ThreadGroup("CrawlerThreads");
- private boolean stopped = false;
-
- public plasmaCrawlLoader(serverLog theLog) {
-
- this.setName("plasmaCrawlLoader");
-
- this.log = theLog;
-
- // supported protocols
- // TODO: change this, e.g. by loading settings from file
- this.supportedProtocols = new HashSet(Arrays.asList(new String[]{"http","https"/* ,"ftp" */}));
-
- // configuring the crawler messagequeue
- this.theQueue = new plasmaCrawlerMsgQueue();
-
- // configuring the crawler thread pool
- // implementation of session thread pool
- this.crawlerPoolConfig = new GenericKeyedObjectPool.Config();
-
- // The maximum number of active connections that can be allocated from pool at the same time,
- // 0 for no limit
- this.crawlerPoolConfig.maxActive = Integer.parseInt(switchboard.getConfig("crawler.MaxActiveThreads","10"));
-
- // The maximum number of idle connections connections in the pool
- // 0 = no limit.
- this.crawlerPoolConfig.maxIdle = Integer.parseInt(switchboard.getConfig("crawler.MaxIdleThreads","7"));
-
- // minIdle configuration not possible for keyedObjectPools
- //this.crawlerPoolConfig.minIdle = Integer.parseInt(switchboard.getConfig("crawler.MinIdleThreads","5"));
-
- // block undefinitely
- this.crawlerPoolConfig.maxWait = -1;
-
- // Action to take in case of an exhausted DBCP statement pool
- // 0 = fail, 1 = block, 2= grow
- this.crawlerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
- this.crawlerPoolConfig.minEvictableIdleTimeMillis = 30000;
- //this.crawlerPoolConfig.timeBetweenEvictionRunsMillis = 30000;
-// config.testOnReturn = true;
-
- plasmaCrawlerFactory theFactory = new plasmaCrawlerFactory(
- this.theThreadGroup,
- switchboard,
- this.log);
-
- this.crawlwerPool = new plasmaCrawlerPool(theFactory,this.crawlerPoolConfig,this.theThreadGroup);
-
- // start the crawl loader
- this.start();
- }
-
- public GenericKeyedObjectPool.Config getPoolConfig() {
- return this.crawlerPoolConfig;
- }
-
- public void setPoolConfig(GenericKeyedObjectPool.Config newConfig) {
- this.crawlwerPool.setConfig(newConfig);
- }
-
- public boolean isSupportedProtocol(String protocol) {
- if ((protocol == null) || (protocol.length() == 0)) return false;
- return this.supportedProtocols.contains(protocol.trim().toLowerCase());
- }
-
- public HashSet getSupportedProtocols() {
- return (HashSet) this.supportedProtocols.clone();
- }
-
- public void close() {
- try {
- // setting the stop flag to true
- this.stopped = true;
-
- // interrupting the plasmaCrawlLoader
- this.interrupt();
-
- // waiting for the thread to finish...
- this.log.logInfo("Waiting for plasmaCrawlLoader shutdown ...");
- this.join(5000);
- } catch (Exception e) {
- // we where interrupted while waiting for the crawlLoader Thread to finish
- }
- }
-
- public ThreadGroup threadStatus() {
- return this.theThreadGroup;
- }
-
- private void execute(plasmaCrawlLoaderMessage theMsg, boolean useThreadPool) throws Exception {
- // getting the protocol of the next URL
- String protocol = theMsg.url.getProtocol();
-
- // TODO: remove this
- if (protocol.equals("https")) protocol = "http";
-
- // get a new worker thread
- plasmaCrawlWorker theWorker = null;
- if (useThreadPool) {
- // getting a new crawler from the crawler pool
- theWorker = (plasmaCrawlWorker) this.crawlwerPool.borrowObject(protocol);
- } else {
- // create a new one
- theWorker = (plasmaCrawlWorker) this.crawlwerPool.getFactory().makeObject(protocol,false);
- }
-
- if (theWorker == null) {
- this.log.logWarning("Unsupported protocol '" + protocol + "' in url " + theMsg.url);
- } else {
- theWorker.execute(theMsg);
- }
- }
-
- public void run() {
-
- while (!this.stopped && !Thread.interrupted()) {
- try {
- // getting a new message from the crawler queue
- plasmaCrawlLoaderMessage theMsg = this.theQueue.waitForMessage();
-
- // start new crawl job
- this.execute(theMsg, true);
-
- } catch (InterruptedException e) {
- Thread.interrupted();
- this.stopped = true;
- }
- catch (Exception e) {
- this.log.logSevere("plasmaCrawlLoader.run/loop", e);
- }
- }
-
- // consuming the "is interrupted"-flag
- this.isInterrupted();
-
- // closing the pool
- try {
- this.crawlwerPool.close();
- }
- catch (Exception e) {
- this.log.logSevere("plasmaCrawlLoader.run/close", e);
- }
-
- }
-
- public plasmaHTCache.Entry loadSync(
- yacyURL url,
- String urlName,
- String referer,
- String initiator,
- int depth,
- plasmaCrawlProfile.entry profile,
- int timeout,
- boolean keepInMemory
- ) throws plasmaCrawlerException {
-
- plasmaHTCache.Entry result = null;
- if (!this.crawlwerPool.isClosed) {
- int crawlingPriority = 5;
-
- // creating a new crawler queue object
- plasmaCrawlLoaderMessage theMsg = new plasmaCrawlLoaderMessage(
- url,
- urlName,
- referer,
- initiator,
- depth,
- profile,
- crawlingPriority,
- true,
- timeout,
- keepInMemory
- );
-
-
- try {
- // start new crawl job
- this.execute(theMsg, false);
-
- // wait for the crawl job result
- result = theMsg.waitForResult();
- } catch (Exception e) {
- this.log.logSevere("plasmaCrawlLoader.loadSync: Unexpected error", e);
- throw new plasmaCrawlerException("Unexpected error: " + e.getMessage());
- }
-
- // check if an error has occured
- if (result == null) {
- String errorMsg = theMsg.getError();
- throw new plasmaCrawlerException(errorMsg);
- }
- }
-
- // return the result
- return result;
- }
-
- public void loadAsync(
- yacyURL url,
- String urlName,
- String referer,
- String initiator,
- int depth,
- plasmaCrawlProfile.entry profile,
- int timeout,
- boolean keepInMemory
- ) {
-
- if (!this.crawlwerPool.isClosed) {
- int crawlingPriority = 5;
-
- // creating a new crawler queue object
- plasmaCrawlLoaderMessage theMsg = new plasmaCrawlLoaderMessage(
- url, // url
- urlName, // url name
- referer, // referer URL
- initiator, // crawling initiator peer
- depth, // crawling depth
- profile, // crawling profile
- crawlingPriority, // crawling priority
- false, // only download documents whose mimetypes are enabled for the crawler
- timeout, // -1 = use default crawler timeout
- keepInMemory // kept in memory ?
- );
-
- // adding the message to the queue
- try {
- this.theQueue.addMessage(theMsg);
- } catch (InterruptedException e) {
- this.log.logSevere("plasmaCrawlLoader.loadAsync", e);
- }
- }
- }
-
- public int getNumIdleWorker() {
- return this.crawlwerPool.getNumIdle();
- }
-
- public int getNumActiveWorker() {
- return size();
- }
-
- public int size() {
- return this.crawlwerPool.getNumActive();
- }
-}
-
-
-
-
-
diff --git a/source/de/anomic/plasma/plasmaCrawlStacker.java b/source/de/anomic/plasma/plasmaCrawlStacker.java index a95b30c9a..4164bce76 100644 --- a/source/de/anomic/plasma/plasmaCrawlStacker.java +++ b/source/de/anomic/plasma/plasmaCrawlStacker.java @@ -6,6 +6,7 @@ // Frankfurt, Germany, 2005
//
// This file was contributed by Martin Thelian
+// ([MC] removed all multithreading and thread pools, this is not necessary here; complete renovation 2007)
//
// $LastChangedDate$
// $LastChangedRevision$
@@ -48,15 +49,12 @@ package de.anomic.plasma; import java.io.File;
import java.io.IOException;
-import java.net.InetAddress;
-import java.net.MalformedURLException;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedList;
-import org.apache.commons.pool.impl.GenericObjectPool;
-
-import de.anomic.data.robotsParser;
import de.anomic.index.indexURLEntry;
import de.anomic.kelondro.kelondroCache;
import de.anomic.kelondro.kelondroException;
@@ -68,126 +66,155 @@ import de.anomic.kelondro.kelondroRowSet; import de.anomic.kelondro.kelondroTree;
import de.anomic.plasma.urlPattern.plasmaURLPattern;
import de.anomic.server.serverDomains;
-import de.anomic.server.serverSemaphore;
import de.anomic.server.logging.serverLog;
import de.anomic.yacy.yacyCore;
import de.anomic.yacy.yacyURL;
-public final class plasmaCrawlStacker {
+public final class plasmaCrawlStacker extends Thread {
// keys for different database types
public static final int QUEUE_DB_TYPE_RAM = 0;
public static final int QUEUE_DB_TYPE_TREE = 1;
public static final int QUEUE_DB_TYPE_FLEX = 2;
- final WorkerPool theWorkerPool;
- private GenericObjectPool.Config theWorkerPoolConfig = null;
- final ThreadGroup theWorkerThreadGroup = new ThreadGroup("stackCrawlThreadGroup");
final serverLog log = new serverLog("STACKCRAWL");
- final plasmaSwitchboard sb;
- //private boolean stopped = false;
- private stackCrawlQueue queue;
+
+ private plasmaSwitchboard sb;
+ private final LinkedList urlEntryHashCache;
+ private kelondroIndex urlEntryCache;
+ private File cacheStacksPath;
+ private long preloadTime;
+ private int dbtype;
+
+ // objects for the prefetch task
+ private ArrayList dnsfetchHosts = new ArrayList();
public plasmaCrawlStacker(plasmaSwitchboard sb, File dbPath, long preloadTime, int dbtype) {
this.sb = sb;
- this.queue = new stackCrawlQueue(dbPath, preloadTime, dbtype);
- this.log.logInfo(this.queue.size() + " entries in the stackCrawl queue.");
- this.log.logInfo("STACKCRAWL thread initialized.");
+ // init the message list
+ this.urlEntryHashCache = new LinkedList();
- // configuring the thread pool
- // implementation of session thread pool
- this.theWorkerPoolConfig = new GenericObjectPool.Config();
+ // create a stack for newly entered entries
+ this.cacheStacksPath = dbPath;
+ this.preloadTime = preloadTime;
+ this.dbtype = dbtype;
- // The maximum number of active connections that can be allocated from pool at the same time,
- // 0 for no limit
- this.theWorkerPoolConfig.maxActive = Integer.parseInt(sb.getConfig("stacker.MaxActiveThreads","50"));
+ openDB();
+ try {
+ // loop through the list and fill the messageList with url hashs
+ Iterator rows = this.urlEntryCache.rows(true, null);
+ kelondroRow.Entry entry;
+ while (rows.hasNext()) {
+ entry = (kelondroRow.Entry) rows.next();
+ if (entry == null) {
+ System.out.println("ERROR! null element found");
+ continue;
+ }
+ this.urlEntryHashCache.add(entry.getColString(0, null));
+ }
+ } catch (kelondroException e) {
+ /* if we have an error, we start with a fresh database */
+ plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, kelondroException:" + e.getMessage() + ". Reseting DB.\n", e);
- // The maximum number of idle connections connections in the pool
- // 0 = no limit.
- this.theWorkerPoolConfig.maxIdle = Integer.parseInt(sb.getConfig("stacker.MaxIdleThreads","10"));
- this.theWorkerPoolConfig.minIdle = Integer.parseInt(sb.getConfig("stacker.MinIdleThreads","5"));
+ // deleting old db and creating a new db
+ try {this.urlEntryCache.close();} catch (Exception ex) {}
+ deleteDB();
+ openDB();
+ } catch (IOException e) {
+ /* if we have an error, we start with a fresh database */
+ plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n", e);
- // block undefinitely
- this.theWorkerPoolConfig.maxWait = -1;
+ // deleting old db and creating a new db
+ try {this.urlEntryCache.close();} catch (Exception ex) {}
+ deleteDB();
+ openDB();
+ }
+ this.log.logInfo(size() + " entries in the stackCrawl queue.");
+ this.start(); // start the prefetcher thread
+ this.log.logInfo("STACKCRAWL thread initialized.");
+ }
- // Action to take in case of an exhausted DBCP statement pool
- // 0 = fail, 1 = block, 2= grow
- this.theWorkerPoolConfig.whenExhaustedAction = GenericObjectPool.WHEN_EXHAUSTED_BLOCK;
- this.theWorkerPoolConfig.minEvictableIdleTimeMillis = 30000;
- //this.theWorkerPoolConfig.timeBetweenEvictionRunsMillis = 30000;
-
- // creating worker pool
- this.theWorkerPool = new WorkerPool(new WorkterFactory(this.theWorkerThreadGroup),this.theWorkerPoolConfig);
-
+ public void run() {
+ String nextHost;
+ try {
+ while (!Thread.currentThread().isInterrupted()) { // action loop
+ if (dnsfetchHosts.size() == 0) synchronized (this) { wait(); }
+ synchronized (dnsfetchHosts) {
+ nextHost = (String) dnsfetchHosts.remove(dnsfetchHosts.size() - 1);
+ }
+ try {
+ serverDomains.dnsResolve(nextHost);
+ } catch (Exception e) {}
+ }
+ } catch (InterruptedException e) {}
+ }
+
+ public void prefetchHost(String host) {
+ try {
+ serverDomains.dnsResolveFromCache(host);
+ } catch (UnknownHostException e) {
+ synchronized (this) {
+ dnsfetchHosts.add(host);
+ notifyAll();
+ }
+ }
}
- public GenericObjectPool.Config getPoolConfig() {
- return this.theWorkerPoolConfig;
- }
-
- public int getDBType() {
- return this.queue.getDBType();
+ public void terminateDNSPrefetcher() {
+ synchronized (this) {
+ interrupt();
+ }
}
- public void setPoolConfig(GenericObjectPool.Config newConfig) {
- this.theWorkerPool.setConfig(newConfig);
- }
-
public void close() {
try {
- this.log.logFine("Shutdown. Terminating worker threads.");
- if (this.theWorkerPool != null) this.theWorkerPool.close();
+ this.log.logFine("Shutdown. Flushing remaining " + size() + " crawl stacker job entries. please wait.");
+ while (size() > 0) {
+ if (!job()) break;
+ }
} catch (Exception e1) {
this.log.logSevere("Unable to shutdown all remaining stackCrawl threads", e1);
}
+ terminateDNSPrefetcher();
this.log.logFine("Shutdown. Closing stackCrawl queue.");
- if (this.queue != null) this.queue.close();
- this.queue = null;
- }
-
- public int getNumActiveWorker() {
- return this.theWorkerPool.getNumActive();
- }
-
- public int getNumIdleWorker() {
- return this.theWorkerPool.getNumIdle();
- }
-
- public int size() {
- return this.queue.size();
+
+ // closing the db
+ this.urlEntryCache.close();
+
+ // clearing the hash list
+ this.urlEntryHashCache.clear();
}
- public void job() {
+ public boolean job() {
+ plasmaCrawlEntry entry;
try {
- // getting a new message from the crawler queue
- checkInterruption();
- plasmaCrawlEntry theMsg = this.queue.waitForMessage();
-
- if (theMsg != null) {
- // getting a free session thread from the pool
- checkInterruption();
- Worker worker = (Worker) this.theWorkerPool.borrowObject();
-
- // processing the new request
- worker.execute(theMsg);
+ entry = dequeueEntry();
+ } catch (IOException e) {
+ e.printStackTrace();
+ return false;
+ }
+ if (entry == null) return false;
+
+ try {
+
+ String rejectReason = sb.crawlStacker.stackCrawl(entry);
+
+ // if the url was rejected we store it into the error URL db
+ if (rejectReason != null) {
+ plasmaCrawlZURL.Entry ee = sb.crawlQueues.errorURL.newEntry(entry, yacyCore.seedDB.mySeed().hash, null, 0, rejectReason);
+ ee.store();
+ sb.crawlQueues.errorURL.push(ee);
}
} catch (Exception e) {
- if (e instanceof InterruptedException) {
- this.log.logFine("Interruption detected.");
- } else if ((e instanceof IllegalStateException) &&
- (e.getMessage() != null) &&
- (e.getMessage().indexOf("Pool not open") >= -1)) {
- this.log.logFine("Pool was closed.");
-
- } else {
- this.log.logSevere("plasmaStackCrawlThread.run/loop", e);
- }
+ plasmaCrawlStacker.this.log.logWarning("Error while processing stackCrawl entry.\n" + "Entry: " + entry.toString() + "Error: " + e.toString(), e);
+ return false;
}
+ return true;
}
- public void enqueue(
+ public void enqueueEntry(
yacyURL nexturl,
String referrerhash,
String initiatorHash,
@@ -196,8 +223,8 @@ public final class plasmaCrawlStacker { int currentdepth,
plasmaCrawlProfile.entry profile,
boolean first) {
- if (profile != null) try {
- this.queue.addMessage(new plasmaCrawlEntry(
+ if (profile == null) return;
+ plasmaCrawlEntry newEntry = new plasmaCrawlEntry(
initiatorHash,
nexturl,
referrerhash,
@@ -207,38 +234,116 @@ public final class plasmaCrawlStacker { currentdepth,
0,
0
- ),
- first);
- } catch (Exception e) {
- e.printStackTrace();
+ );
+
+ if (newEntry == null) return;
+
+ synchronized(this.urlEntryHashCache) {
+ kelondroRow.Entry oldValue;
+ prefetchHost(nexturl.getHost());
+ try {
+ oldValue = this.urlEntryCache.put(newEntry.toRow());
+ } catch (IOException e) {
+ oldValue = null;
+ }
+ if (oldValue == null) {
+ if (first) {
+ this.urlEntryHashCache.addFirst(newEntry.url().hash());
+ } else {
+ this.urlEntryHashCache.addLast(newEntry.url().hash());
+ }
+ }
}
}
- public String dequeue(plasmaCrawlEntry theMsg) throws InterruptedException {
-
- plasmaCrawlProfile.entry profile = this.sb.profilesActiveCrawls.getEntry(theMsg.profileHandle());
- if (profile == null) {
- String errorMsg = "LOST PROFILE HANDLE '" + theMsg.profileHandle() + "' for URL " + theMsg.url();
- this.log.logSevere(errorMsg);
- throw new IllegalStateException(errorMsg);
+ private void deleteDB() {
+ if (this.dbtype == QUEUE_DB_TYPE_RAM) {
+ // do nothing..
+ }
+ if (this.dbtype == QUEUE_DB_TYPE_FLEX) {
+ kelondroFlexWidthArray.delete(cacheStacksPath, "urlNoticeStacker8.db");
+ }
+ if (this.dbtype == QUEUE_DB_TYPE_TREE) {
+ File cacheFile = new File(cacheStacksPath, "urlNoticeStacker8.db");
+ cacheFile.delete();
}
-
- return stackCrawl(
- theMsg.url().toNormalform(true, true),
- theMsg.referrerhash(),
- theMsg.initiator(),
- theMsg.name(),
- theMsg.loaddate(),
- theMsg.depth(),
- profile);
+ }
+
+ private void openDB() {
+ if (!(cacheStacksPath.exists())) cacheStacksPath.mkdir(); // make the path
+
+ if (this.dbtype == QUEUE_DB_TYPE_RAM) {
+ this.urlEntryCache = new kelondroRowSet(plasmaCrawlEntry.rowdef, 0);
+ }
+ if (this.dbtype == QUEUE_DB_TYPE_FLEX) {
+ String newCacheName = "urlNoticeStacker8.db";
+ cacheStacksPath.mkdirs();
+ try {
+ this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, true), true, false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ // kill DB and try again
+ kelondroFlexTable.delete(cacheStacksPath, newCacheName);
+ try {
+ this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, true), true, false);
+ } catch (Exception ee) {
+ ee.printStackTrace();
+ System.exit(-1);
+ }
+ }
+ }
+ if (this.dbtype == QUEUE_DB_TYPE_TREE) {
+ File cacheFile = new File(cacheStacksPath, "urlNoticeStacker8.db");
+ cacheFile.getParentFile().mkdirs();
+ this.urlEntryCache = new kelondroCache(kelondroTree.open(cacheFile, true, preloadTime, plasmaCrawlEntry.rowdef), true, true);
+ }
+ }
+
+ public int size() {
+ synchronized (this.urlEntryHashCache) {
+ return this.urlEntryHashCache.size();
+ }
+ }
+
+ public int getDBType() {
+ return this.dbtype;
+ }
+
+ public plasmaCrawlEntry dequeueEntry() throws IOException {
+ if (this.urlEntryHashCache.size() == 0) return null;
+ String urlHash = null;
+ kelondroRow.Entry entry = null;
+ synchronized (this.urlEntryHashCache) {
+ urlHash = (String) this.urlEntryHashCache.removeFirst();
+ if (urlHash == null) throw new IOException("urlHash is null");
+ entry = this.urlEntryCache.remove(urlHash.getBytes(), false);
+ }
+
+ if ((urlHash == null) || (entry == null)) return null;
+ return new plasmaCrawlEntry(entry);
}
- public void checkInterruption() throws InterruptedException {
- Thread curThread = Thread.currentThread();
- if (curThread.isInterrupted()) throw new InterruptedException("Shutdown in progress ...");
+ public String stackCrawl(yacyURL url, String referrerhash, String initiatorHash, String name, Date loadDate, int currentdepth, plasmaCrawlProfile.entry profile) throws InterruptedException {
+ // stacks a crawl item. The position can also be remote
+ // returns null if successful, a reason string if not successful
+ //this.log.logFinest("stackCrawl: nexturlString='" + nexturlString + "'");
+
+ // add the url into the crawling queue
+ plasmaCrawlEntry entry = new plasmaCrawlEntry(
+ initiatorHash, // initiator, needed for p2p-feedback
+ url, // url clear text string
+ referrerhash, // last url in crawling queue
+ name, // load date
+ loadDate, // the anchor name
+ (profile == null) ? null : profile.handle(), // profile must not be null!
+ currentdepth, // depth so far
+ 0, // anchors, default value
+ 0 // forkfactor, default value
+ );
+ return stackCrawl(entry);
}
- public String stackCrawl(String nexturlString, String referrerString, String initiatorHash, String name, Date loadDate, int currentdepth, plasmaCrawlProfile.entry profile) throws InterruptedException {
+ public String stackCrawl(plasmaCrawlEntry entry) throws InterruptedException {
// stacks a crawl item. The position can also be remote
// returns null if successful, a reason string if not successful
//this.log.logFinest("stackCrawl: nexturlString='" + nexturlString + "'");
@@ -246,157 +351,111 @@ public final class plasmaCrawlStacker { long startTime = System.currentTimeMillis();
String reason = null; // failure reason
- // getting the initiator peer hash
- if ((initiatorHash == null) || (initiatorHash.length() == 0)) initiatorHash = yacyURL.dummyHash;
-
- // strange errors
- if (nexturlString == null) {
- reason = plasmaCrawlEURL.DENIED_URL_NULL;
- this.log.logSevere("Wrong URL in stackCrawl: url=null");
- return reason;
- }
-
- // getting the referer url and url hash
- yacyURL referrerURL = null;
- if (referrerString != null) {
- try {
- referrerURL = new yacyURL(referrerString, null);
- } catch (MalformedURLException e) {
- referrerURL = null;
- referrerString = null;
- }
- }
-
- // check for malformed urls
- yacyURL nexturl = null;
- try {
- nexturl = new yacyURL(nexturlString, null);
- } catch (MalformedURLException e) {
- reason = plasmaCrawlEURL.DENIED_MALFORMED_URL;
- this.log.logSevere("Wrong URL in stackCrawl: " + nexturlString +
- ". Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
- return reason;
- }
-
// check if the protocol is supported
- String urlProtocol = nexturl.getProtocol();
- if (!this.sb.cacheLoader.isSupportedProtocol(urlProtocol)) {
+ String urlProtocol = entry.url().getProtocol();
+ if (!sb.crawlQueues.isSupportedProtocol(urlProtocol)) {
reason = plasmaCrawlEURL.DENIED_UNSUPPORTED_PROTOCOL;
- this.log.logSevere("Unsupported protocol in URL '" + nexturlString + "'. " +
+ this.log.logSevere("Unsupported protocol in URL '" + entry.url().toString() + "'. " +
"Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
return reason;
}
-
+
// check if ip is local ip address
- checkInterruption(); // TODO: this is protocol specific
- InetAddress hostAddress = serverDomains.dnsResolve(nexturl.getHost());
- if (hostAddress == null) {
- // if a http proxy is configured name resolution may not work
- if (this.sb.remoteProxyConfig == null || !this.sb.remoteProxyConfig.useProxy()) {
- reason = plasmaCrawlEURL.DENIED_UNKNOWN_HOST;
- this.log.logFine("Unknown host in URL '" + nexturlString + "'. " +
- "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
- return reason;
- }
- } else if (!sb.acceptURL(hostAddress)) {
+ if (!sb.acceptURL(entry.url())) {
reason = plasmaCrawlEURL.DENIED_IP_ADDRESS_NOT_IN_DECLARED_DOMAIN + "[" + sb.getConfig("network.unit.domain", "unknown") + "]";
- this.log.logFine("Host in URL '" + nexturlString + "' has IP address outside of declared range (" + sb.getConfig("network.unit.domain", "unknown") + "). " +
+ this.log.logFine("Host in URL '" + entry.url().toString() + "' has IP address outside of declared range (" + sb.getConfig("network.unit.domain", "unknown") + "). " +
"Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
return reason;
}
// check blacklist
- checkInterruption();
- if (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER,nexturl)) {
+ if (plasmaSwitchboard.urlBlacklist.isListed(plasmaURLPattern.BLACKLIST_CRAWLER, entry.url())) {
reason = plasmaCrawlEURL.DENIED_URL_IN_BLACKLIST;
- this.log.logFine("URL '" + nexturlString + "' is in blacklist. " +
+ this.log.logFine("URL '" + entry.url().toString() + "' is in blacklist. " +
"Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
return reason;
- }
+ }
+
+ plasmaCrawlProfile.entry profile = sb.profilesActiveCrawls.getEntry(entry.profileHandle());
+ if (profile == null) {
+ String errorMsg = "LOST PROFILE HANDLE '" + entry.profileHandle() + "' for URL " + entry.url();
+ log.logWarning(errorMsg);
+ return errorMsg;
+ }
// filter deny
- if ((currentdepth > 0) && (profile != null) && (!(nexturlString.matches(profile.generalFilter())))) {
+ if ((entry.depth() > 0) && (profile != null) && (!(entry.url().toString().matches(profile.generalFilter())))) {
reason = plasmaCrawlEURL.DENIED_URL_DOES_NOT_MATCH_FILTER;
- this.log.logFine("URL '" + nexturlString + "' does not match crawling filter '" + profile.generalFilter() + "'. " +
+ this.log.logFine("URL '" + entry.url().toString() + "' does not match crawling filter '" + profile.generalFilter() + "'. " +
"Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
return reason;
}
// deny cgi
- if (plasmaHTCache.isCGI(nexturlString)) {
+ if (entry.url().isCGI()) {
reason = plasmaCrawlEURL.DENIED_CGI_URL;
- this.log.logFine("URL '" + nexturlString + "' is CGI URL. " +
+ this.log.logFine("URL '" + entry.url().toString() + "' is CGI URL. " +
"Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
return reason;
}
// deny post properties
- if ((plasmaHTCache.isPOST(nexturlString)) && (profile != null) && (!(profile.crawlingQ()))) {
+ if ((entry.url().isPOST()) && (profile != null) && (!(profile.crawlingQ()))) {
reason = plasmaCrawlEURL.DENIED_POST_URL;
- this.log.logFine("URL '" + nexturlString + "' is post URL. " +
+ this.log.logFine("URL '" + entry.url().toString() + "' is post URL. " +
"Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
return reason;
}
+ yacyURL referrerURL = (entry.referrerhash() == null) ? null : sb.crawlQueues.getURL(entry.referrerhash());
+
// add domain to profile domain list
if ((profile.domFilterDepth() != Integer.MAX_VALUE) || (profile.domMaxPages() != Integer.MAX_VALUE)) {
- profile.domInc(nexturl.getHost(), (referrerURL == null) ? null : referrerURL.getHost().toLowerCase(), currentdepth);
+ profile.domInc(entry.url().getHost(), (referrerURL == null) ? null : referrerURL.getHost().toLowerCase(), entry.depth());
}
// deny urls that do not match with the profile domain list
- if (!(profile.grantedDomAppearance(nexturl.getHost()))) {
+ if (!(profile.grantedDomAppearance(entry.url().getHost()))) {
reason = plasmaCrawlEURL.DENIED_NO_MATCH_WITH_DOMAIN_FILTER;
- this.log.logFine("URL '" + nexturlString + "' is not listed in granted domains. " +
+ this.log.logFine("URL '" + entry.url().toString() + "' is not listed in granted domains. " +
"Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
return reason;
}
// deny urls that exceed allowed number of occurrences
- if (!(profile.grantedDomCount(nexturl.getHost()))) {
+ if (!(profile.grantedDomCount(entry.url().getHost()))) {
reason = plasmaCrawlEURL.DENIED_DOMAIN_COUNT_EXCEEDED;
- this.log.logFine("URL '" + nexturlString + "' appeared too often, a maximum of " + profile.domMaxPages() + " is allowed. "+
+ this.log.logFine("URL '" + entry.url().toString() + "' appeared too often, a maximum of " + profile.domMaxPages() + " is allowed. "+
"Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
return reason;
}
// check if the url is double registered
- checkInterruption();
- String dbocc = this.sb.urlExists(nexturl.hash());
- indexURLEntry oldEntry = null;
- oldEntry = this.sb.wordIndex.loadedURL.load(nexturl.hash(), null);
+ String dbocc = sb.crawlQueues.urlExists(entry.url().hash());
+ indexURLEntry oldEntry = this.sb.wordIndex.loadedURL.load(entry.url().hash(), null);
boolean recrawl = (oldEntry != null) && ((System.currentTimeMillis() - oldEntry.loaddate().getTime()) > profile.recrawlIfOlder());
// apply recrawl rule
if ((dbocc != null) && (!(recrawl))) {
reason = plasmaCrawlEURL.DOUBLE_REGISTERED + dbocc + ")";
- this.log.logFine("URL '" + nexturlString + "' is double registered in '" + dbocc + "'. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
+ this.log.logFine("URL '" + entry.url().toString() + "' is double registered in '" + dbocc + "'. " + "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
return reason;
}
- // checking robots.txt for http(s) resources
- checkInterruption();
- if ((urlProtocol.equals("http") || urlProtocol.equals("https")) && robotsParser.isDisallowed(nexturl)) {
- reason = plasmaCrawlEURL.DENIED_ROBOTS_TXT;
-
- this.log.logFine("Crawling of URL '" + nexturlString + "' disallowed by robots.txt. " +
- "Stack processing time: " + (System.currentTimeMillis()-startTime) + "ms");
- return reason;
- }
-
// show potential re-crawl
if (recrawl) {
- this.log.logFine("RE-CRAWL of URL '" + nexturlString + "': this url was crawled " +
+ this.log.logFine("RE-CRAWL of URL '" + entry.url().toString() + "': this url was crawled " +
((System.currentTimeMillis() - oldEntry.loaddate().getTime()) / 60000 / 60 / 24) + " days ago.");
}
// store information
- boolean local = ((initiatorHash.equals(yacyURL.dummyHash)) || (initiatorHash.equals(yacyCore.seedDB.mySeed().hash)));
+ boolean local = ((entry.initiator().equals(yacyURL.dummyHash)) || (entry.initiator().equals(yacyCore.seedDB.mySeed().hash)));
boolean global =
(profile != null) &&
(profile.remoteIndexing()) /* granted */ &&
- (currentdepth == profile.generalDepth()) /* leaf node */ &&
+ (entry.depth() == profile.generalDepth()) /* leaf node */ &&
//(initiatorHash.equals(yacyCore.seedDB.mySeed.hash)) /* not proxy */ &&
(
(yacyCore.seedDB.mySeed().isSenior()) ||
@@ -404,474 +463,15 @@ public final class plasmaCrawlStacker { ) /* qualified */;
if ((!local)&&(!global)&&(!profile.handle().equals(this.sb.defaultRemoteProfile.handle()))) {
- this.log.logSevere("URL '" + nexturlString + "' can neither be crawled local nor global.");
+ this.log.logSevere("URL '" + entry.url().toString() + "' can neither be crawled local nor global.");
}
// add the url into the crawling queue
- checkInterruption();
- plasmaCrawlEntry ne = new plasmaCrawlEntry(initiatorHash, /* initiator, needed for p2p-feedback */
- nexturl, /* url clear text string */
- (referrerURL == null) ? null : referrerURL.hash(), /* last url in crawling queue */
- name, /* load date */
- loadDate, /* the anchor name */
- (profile == null) ? null : profile.handle(), // profile must not be null!
- currentdepth, /*depth so far*/
- 0, /*anchors, default value */
- 0 /*forkfactor, default value */
- );
- this.sb.noticeURL.push(
+ sb.crawlQueues.noticeURL.push(
((global) ? plasmaCrawlNURL.STACK_TYPE_LIMIT :
((local) ? plasmaCrawlNURL.STACK_TYPE_CORE : plasmaCrawlNURL.STACK_TYPE_REMOTE)) /*local/remote stack*/,
- ne);
+ entry);
return null;
}
- final class stackCrawlQueue {
-
- private final serverSemaphore readSync;
- private final serverSemaphore writeSync;
- private final LinkedList urlEntryHashCache;
- private kelondroIndex urlEntryCache;
- private File cacheStacksPath;
- private long preloadTime;
- private int dbtype;
-
- public stackCrawlQueue(File cacheStacksPath, long preloadTime, int dbtype) {
- // init the read semaphore
- this.readSync = new serverSemaphore (0);
-
- // init the write semaphore
- this.writeSync = new serverSemaphore (1);
-
- // init the message list
- this.urlEntryHashCache = new LinkedList();
-
- // create a stack for newly entered entries
- this.cacheStacksPath = cacheStacksPath;
- this.preloadTime = preloadTime;
- this.dbtype = dbtype;
-
- openDB();
- try {
- // loop through the list and fill the messageList with url hashs
- Iterator rows = this.urlEntryCache.rows(true, null);
- kelondroRow.Entry entry;
- while (rows.hasNext()) {
- entry = (kelondroRow.Entry) rows.next();
- if (entry == null) {
- System.out.println("ERROR! null element found");
- continue;
- }
- this.urlEntryHashCache.add(entry.getColString(0, null));
- this.readSync.V();
- }
- } catch (kelondroException e) {
- /* if we have an error, we start with a fresh database */
- plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, kelondroException:" + e.getMessage() + ". Reseting DB.\n", e);
-
- // deleting old db and creating a new db
- try {this.urlEntryCache.close();} catch (Exception ex) {}
- deleteDB();
- openDB();
- } catch (IOException e) {
- /* if we have an error, we start with a fresh database */
- plasmaCrawlStacker.this.log.logSevere("Unable to initialize crawl stacker queue, IOException:" + e.getMessage() + ". Reseting DB.\n", e);
-
- // deleting old db and creating a new db
- try {this.urlEntryCache.close();} catch (Exception ex) {}
- deleteDB();
- openDB();
- }
-
- }
-
- private void deleteDB() {
- if (this.dbtype == QUEUE_DB_TYPE_RAM) {
- // do nothing..
- }
- if (this.dbtype == QUEUE_DB_TYPE_FLEX) {
- kelondroFlexWidthArray.delete(cacheStacksPath, "urlNoticeStacker8.db");
- }
- if (this.dbtype == QUEUE_DB_TYPE_TREE) {
- File cacheFile = new File(cacheStacksPath, "urlNoticeStacker8.db");
- cacheFile.delete();
- }
- }
-
- private void openDB() {
- if (!(cacheStacksPath.exists())) cacheStacksPath.mkdir(); // make the path
-
- if (this.dbtype == QUEUE_DB_TYPE_RAM) {
- this.urlEntryCache = new kelondroRowSet(plasmaCrawlEntry.rowdef, 0);
- }
- if (this.dbtype == QUEUE_DB_TYPE_FLEX) {
- String newCacheName = "urlNoticeStacker8.db";
- cacheStacksPath.mkdirs();
- try {
- this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, true), true, false);
- } catch (Exception e) {
- e.printStackTrace();
- // kill DB and try again
- kelondroFlexTable.delete(cacheStacksPath, newCacheName);
- try {
- this.urlEntryCache = new kelondroCache(new kelondroFlexTable(cacheStacksPath, newCacheName, preloadTime, plasmaCrawlEntry.rowdef, true), true, false);
- } catch (Exception ee) {
- ee.printStackTrace();
- System.exit(-1);
- }
- }
- }
- if (this.dbtype == QUEUE_DB_TYPE_TREE) {
- File cacheFile = new File(cacheStacksPath, "urlNoticeStacker8.db");
- cacheFile.getParentFile().mkdirs();
- this.urlEntryCache = new kelondroCache(kelondroTree.open(cacheFile, true, preloadTime, plasmaCrawlEntry.rowdef), true, true);
- }
- }
-
- public void close() {
- // closing the db
- this.urlEntryCache.close();
-
- // clearing the hash list
- this.urlEntryHashCache.clear();
- }
-
- public void addMessage(plasmaCrawlEntry newMessage, boolean first) throws InterruptedException, IOException {
- if (newMessage == null) throw new NullPointerException();
-
- this.writeSync.P();
- try {
-
- boolean insertionDone = false;
- synchronized(this.urlEntryHashCache) {
- kelondroRow.Entry oldValue = this.urlEntryCache.put(newMessage.toRow());
- if (oldValue == null) {
- if (first) {
- this.urlEntryHashCache.addFirst(newMessage.url().hash());
- } else {
- this.urlEntryHashCache.addLast(newMessage.url().hash());
- }
- insertionDone = true;
- }
- }
-
- if (insertionDone) {
- this.readSync.V();
- }
- } finally {
- this.writeSync.V();
- }
- }
-
- public int size() {
- synchronized(this.urlEntryHashCache) {
- return this.urlEntryHashCache.size();
- }
- }
-
- public int getDBType() {
- return this.dbtype;
- }
-
- public plasmaCrawlEntry waitForMessage() throws InterruptedException, IOException {
- this.readSync.P();
- this.writeSync.P();
-
- if (this.urlEntryHashCache.size() == 0) return null;
- String urlHash = null;
- kelondroRow.Entry entry = null;
- try {
- synchronized(this.urlEntryHashCache) {
- urlHash = (String) this.urlEntryHashCache.removeFirst();
- if (urlHash == null) throw new IOException("urlHash is null");
- entry = this.urlEntryCache.remove(urlHash.getBytes(), false);
- }
- } finally {
- this.writeSync.V();
- }
-
- if ((urlHash == null) || (entry == null)) return null;
- return new plasmaCrawlEntry(entry);
- }
- }
-
- public final class WorkterFactory implements org.apache.commons.pool.PoolableObjectFactory {
-
- final ThreadGroup workerThreadGroup;
- public WorkterFactory(ThreadGroup theWorkerThreadGroup) {
- super();
-
- if (theWorkerThreadGroup == null)
- throw new IllegalArgumentException("The threadgroup object must not be null.");
-
- this.workerThreadGroup = theWorkerThreadGroup;
-}
-
- public Object makeObject() {
- Worker newWorker = new Worker(this.workerThreadGroup);
- newWorker.setPriority(Thread.MAX_PRIORITY);
- return newWorker;
- }
-
- /**
- * @see org.apache.commons.pool.PoolableObjectFactory#destroyObject(java.lang.Object)
- */
- public void destroyObject(Object obj) {
- if (obj instanceof Worker) {
- Worker theWorker = (Worker) obj;
- synchronized(theWorker) {
- theWorker.setName("stackCrawlThread_destroyed");
- theWorker.destroyed = true;
- theWorker.setStopped(true);
- theWorker.interrupt();
- }
- }
- }
-
- /**
- * @see org.apache.commons.pool.PoolableObjectFactory#validateObject(java.lang.Object)
- */
- public boolean validateObject(Object obj) {
- return true;
- }
-
- /**
- * @param obj
- *
- */
- public void activateObject(Object obj) {
- //log.debug(" activateObject...");
- }
-
- /**
- * @param obj
- *
- */
- public void passivateObject(Object obj) {
- //log.debug(" passivateObject..." + obj);
-// if (obj instanceof Session) {
-// Session theSession = (Session) obj;
-// }
- }
- }
-
- public final class WorkerPool extends GenericObjectPool {
- public boolean isClosed = false;
-
- /**
- * First constructor.
- * @param objFactory
- */
- public WorkerPool(WorkterFactory objFactory) {
- super(objFactory);
- this.setMaxIdle(10); // Maximum idle threads.
- this.setMaxActive(50); // Maximum active threads.
- this.setMinEvictableIdleTimeMillis(30000); //Evictor runs every 30 secs.
- //this.setMaxWait(1000); // Wait 1 second till a thread is available
- }
-
- public WorkerPool(plasmaCrawlStacker.WorkterFactory objFactory,
- GenericObjectPool.Config config) {
- super(objFactory, config);
- }
-
- public Object borrowObject() throws Exception {
- return super.borrowObject();
- }
-
- public void returnObject(Object obj) {
- if (obj == null) return;
- if (obj instanceof Worker) {
- try {
- ((Worker)obj).setName("stackCrawlThread_inPool");
- super.returnObject(obj);
- } catch (Exception e) {
- ((Worker)obj).setStopped(true);
- serverLog.logSevere("STACKCRAWL-POOL","Unable to return stackcrawl thread to pool.",e);
- }
- } else {
- serverLog.logSevere("STACKCRAWL-POOL","Object of wront type '" + obj.getClass().getName() +
- "' returned to pool.");
- }
- }
-
- public void invalidateObject(Object obj) {
- if (obj == null) return;
- if (this.isClosed) return;
- if (obj instanceof Worker) {
- try {
- ((Worker)obj).setName("stackCrawlThread_invalidated");
- ((Worker)obj).setStopped(true);
- super.invalidateObject(obj);
- } catch (Exception e) {
- serverLog.logSevere("STACKCRAWL-POOL","Unable to invalidate stackcrawl thread.",e);
- }
- }
- }
-
- public synchronized void close() throws Exception {
-
- /*
- * shutdown all still running session threads ...
- */
- this.isClosed = true;
-
- /* waiting for all threads to finish */
- int threadCount = theWorkerThreadGroup.activeCount();
- Thread[] threadList = new Thread[threadCount];
- threadCount = theWorkerThreadGroup.enumerate(threadList);
-
- try {
- // trying to gracefull stop all still running sessions ...
- log.logInfo("Signaling shutdown to " + threadCount + " remaining stackCrawl threads ...");
- for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) {
- Thread currentThread = threadList[currentThreadIdx];
- if (currentThread.isAlive()) {
- ((Worker)currentThread).setStopped(true);
- }
- }
-
- // waiting a frew ms for the session objects to continue processing
- try { Thread.sleep(500); } catch (InterruptedException ex) {}
-
- // interrupting all still running or pooled threads ...
- log.logInfo("Sending interruption signal to " + theWorkerThreadGroup.activeCount() + " remaining stackCrawl threads ...");
- theWorkerThreadGroup.interrupt();
-
- // if there are some sessions that are blocking in IO, we simply close the socket
- log.logFine("Trying to abort " + theWorkerThreadGroup.activeCount() + " remaining stackCrawl threads ...");
- for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) {
- Thread currentThread = threadList[currentThreadIdx];
- if (currentThread.isAlive()) {
- log.logInfo("Trying to shutdown stackCrawl thread '" + currentThread.getName() + "' [" + currentThreadIdx + "].");
- ((Worker)currentThread).close();
- }
- }
-
- // we need to use a timeout here because of missing interruptable session threads ...
- log.logFine("Waiting for " + theWorkerThreadGroup.activeCount() + " remaining stackCrawl threads to finish shutdown ...");
- for ( int currentThreadIdx = 0; currentThreadIdx < threadCount; currentThreadIdx++ ) {
- Thread currentThread = threadList[currentThreadIdx];
- if (currentThread.isAlive()) {
- log.logFine("Waiting for stackCrawl thread '" + currentThread.getName() + "' [" + currentThreadIdx + "] to finish shutdown.");
- try { currentThread.join(500); } catch (InterruptedException ex) {}
- }
- }
-
- log.logInfo("Shutdown of remaining stackCrawl threads finished.");
- } catch (Exception e) {
- log.logSevere("Unexpected error while trying to shutdown all remaining stackCrawl threads.",e);
- }
-
- super.close();
- }
-
- }
-
- public final class Worker extends Thread {
- boolean destroyed = false;
- private boolean running = false;
- private boolean stopped = false;
- private boolean done = false;
- private plasmaCrawlEntry theMsg;
-
- public Worker(ThreadGroup theThreadGroup) {
- super(theThreadGroup,"stackCrawlThread_created");
- }
-
- public void setStopped(boolean stopped) {
- this.stopped = stopped;
- }
-
- public void close() {
- if (this.isAlive()) {
- try {
- // TODO: this object should care of all open clien connections within this class and close them here
- } catch (Exception e) {}
- }
- }
-
- public synchronized void execute(plasmaCrawlEntry newMsg) {
- this.theMsg = newMsg;
- this.done = false;
-
- if (!this.running) {
- // this.setDaemon(true);
- this.start();
- } else {
- this.notifyAll();
- }
- }
-
- public void reset() {
- this.done = true;
- this.theMsg = null;
- }
-
- public boolean isRunning() {
- return this.running;
- }
-
- public void run() {
- this.running = true;
-
- try {
- // The thread keeps running.
- while (!this.stopped && !this.isInterrupted() && !plasmaCrawlStacker.this.theWorkerPool.isClosed) {
- if (this.done) {
- synchronized (this) {
- // return thread back into pool
- plasmaCrawlStacker.this.theWorkerPool.returnObject(this);
-
- // We are waiting for a new task now.
- if (!this.stopped && !this.destroyed && !this.isInterrupted()) {
- this.wait();
- }
- }
- } else {
- try {
- // executing the new task
- execute();
- } finally {
- // reset thread
- reset();
- }
- }
- }
- } catch (InterruptedException ex) {
- serverLog.logFiner("STACKCRAWL-POOL","Interruption of thread '" + this.getName() + "' detected.");
- } finally {
- if (plasmaCrawlStacker.this.theWorkerPool != null && !this.destroyed)
- plasmaCrawlStacker.this.theWorkerPool.invalidateObject(this);
- }
- }
-
- private void execute() throws InterruptedException {
- try {
- this.setName("stackCrawlThread_" + this.theMsg.url());
- String rejectReason = dequeue(this.theMsg);
-
- // check for interruption
- checkInterruption();
-
- // if the url was rejected we store it into the error URL db
- if (rejectReason != null) {
- plasmaCrawlZURL.Entry ee = sb.errorURL.newEntry(
- this.theMsg, yacyCore.seedDB.mySeed().hash, null,
- 0, rejectReason);
- ee.store();
- sb.errorURL.stackPushEntry(ee);
- }
- } catch (Exception e) {
- if (e instanceof InterruptedException) throw (InterruptedException) e;
- plasmaCrawlStacker.this.log.logWarning("Error while processing stackCrawl entry.\n" +
- "Entry: " + this.theMsg.toString() +
- "Error: " + e.toString(),e);
- } finally {
- this.done = true;
- }
-
- }
- }
-
}
diff --git a/source/de/anomic/plasma/plasmaCrawlZURL.java b/source/de/anomic/plasma/plasmaCrawlZURL.java index 63a28ae87..f98d20119 100644 --- a/source/de/anomic/plasma/plasmaCrawlZURL.java +++ b/source/de/anomic/plasma/plasmaCrawlZURL.java @@ -54,29 +54,29 @@ public class plasmaCrawlZURL { 0); // the class object - private kelondroIndex urlIndexFile = null; - private LinkedList rejectedStack = new LinkedList(); // strings: url + private kelondroIndex urlIndex = null; + private LinkedList stack = new LinkedList(); // strings: url public plasmaCrawlZURL(File cachePath, String tablename, boolean startWithEmptyFile) { // creates a new ZURL in a file cachePath.mkdirs(); if (startWithEmptyFile) kelondroFlexTable.delete(cachePath, tablename); - urlIndexFile = new kelondroFlexTable(cachePath, tablename, -1, rowdef, true); + urlIndex = new kelondroFlexTable(cachePath, tablename, -1, rowdef, true); } public plasmaCrawlZURL() { // creates a new ZUR in RAM - urlIndexFile = new kelondroRowSet(rowdef, 0); + urlIndex = new kelondroRowSet(rowdef, 0); } public int size() { - return urlIndexFile.size() ; + return urlIndex.size() ; } public void close() { - if (urlIndexFile != null) { - urlIndexFile.close(); - urlIndexFile = null; + if (urlIndex != null) { + urlIndex.close(); + urlIndex = null; } } @@ -95,45 +95,52 @@ public class plasmaCrawlZURL { public boolean remove(String hash) { if (hash == null) return false; try { - urlIndexFile.remove(hash.getBytes(), false); + urlIndex.remove(hash.getBytes(), false); return true; } catch (IOException e) { return false; } } - public synchronized void stackPushEntry(Entry e) { - rejectedStack.add(e.hash()); + public synchronized void push(Entry e) { + stack.add(e.hash()); } - public Entry stackPopEntry(int pos) throws IOException { - String urlhash = (String) rejectedStack.get(pos); + public Entry top(int pos) throws IOException { + String urlhash = (String) stack.get(pos); if (urlhash == null) return null; - return new Entry(urlhash); + return getEntry(urlhash); } - public synchronized Entry getEntry(String hash) throws IOException { - return new Entry(hash); + public synchronized Entry getEntry(String urlhash) { + try { + kelondroRow.Entry entry = urlIndex.get(urlhash.getBytes()); + if (entry == null) return null; + return new Entry(entry); + } catch (IOException e) { + e.printStackTrace(); + return null; + } } public boolean getUseNewDB() { - return (urlIndexFile instanceof kelondroFlexTable); + return (urlIndex instanceof kelondroFlexTable); } public boolean exists(String urlHash) { try { - return urlIndexFile.has(urlHash.getBytes()); + return urlIndex.has(urlHash.getBytes()); } catch (IOException e) { return false; } } public void clearStack() { - rejectedStack.clear(); + stack.clear(); } public int stackSize() { - return rejectedStack.size(); + return stack.size(); } public class Entry { @@ -153,6 +160,7 @@ public class plasmaCrawlZURL { plasmaCrawlEntry bentry, String executor, Date workdate, int workcount, String anycause) { // create new entry + assert bentry != null; this.bentry = bentry; this.executor = (executor == null) ? yacyCore.seedDB.mySeed().hash : executor; this.workdate = (workdate == null) ? new Date() : workdate; @@ -161,17 +169,9 @@ public class plasmaCrawlZURL { stored = false; } - public Entry(String hash) throws IOException { - kelondroRow.Entry entry = urlIndexFile.get(hash.getBytes()); - if (entry != null) { - insertEntry(entry); - } - this.stored = true; - } - public Entry(kelondroRow.Entry entry) throws IOException { insertEntry(entry); - this.stored = false; + this.stored = true; } private void insertEntry(kelondroRow.Entry entry) throws IOException { @@ -197,7 +197,7 @@ public class plasmaCrawlZURL { newrow.setCol(4, this.anycause.getBytes()); newrow.setCol(5, this.bentry.toRow().bytes()); try { - urlIndexFile.put(newrow); + urlIndex.put(newrow); this.stored = true; } catch (IOException e) { System.out.println("INTERNAL ERROR AT plasmaEURL:url2hash:" + e.toString()); @@ -241,7 +241,7 @@ public class plasmaCrawlZURL { boolean error = false; public kiter(boolean up, String firstHash) throws IOException { - i = urlIndexFile.rows(up, (firstHash == null) ? null : firstHash.getBytes()); + i = urlIndex.rows(up, (firstHash == null) ? null : firstHash.getBytes()); error = false; } diff --git a/source/de/anomic/plasma/plasmaHTCache.java b/source/de/anomic/plasma/plasmaHTCache.java index 4b62afd8d..f98b505cf 100644 --- a/source/de/anomic/plasma/plasmaHTCache.java +++ b/source/de/anomic/plasma/plasmaHTCache.java @@ -401,13 +401,9 @@ public final class plasmaHTCache { }
}
}
-
- public static boolean deleteFile(yacyURL url) {
- return deleteURLfromCache("", url, "FROM");
- }
-
- private static boolean deleteURLfromCache(String key, yacyURL url, String msg) {
- if (deleteFileandDirs(key, getCachePath(url), msg)) {
+
+ public static boolean deleteURLfromCache(yacyURL url) {
+ if (deleteFileandDirs(getCachePath(url), "FROM")) {
try {
// As the file is gone, the entry in responseHeader.db is not needed anymore
log.logFinest("Trying to remove responseHeader from URL: " + url.toNormalform(false, true));
@@ -432,9 +428,9 @@ public final class plasmaHTCache { return false;
}
- private static boolean deleteFileandDirs(String key, File obj, String msg) {
+ private static boolean deleteFileandDirs(File obj, String msg) {
if (deleteFile(obj)) {
- log.logInfo("DELETED " + msg + " CACHE [" + key + "]: " + obj.toString());
+ log.logInfo("DELETED " + msg + " CACHE: " + obj.toString());
obj = obj.getParentFile();
// If the has been emptied, remove it
// Loop as long as we produce empty driectoriers, but stop at HTCACHE
@@ -462,7 +458,7 @@ public final class plasmaHTCache { if (file != null) {
if (filesInUse.contains(file)) continue;
log.logFinest("Trying to delete [" + key + "] = old file: " + file.toString());
- if (deleteFileandDirs(key, file, "OLD")) {
+ if (deleteFileandDirs(file, "OLD")) {
try {
// As the file is gone, the entry in responseHeader.db is not needed anymore
String urlHash = getHash(file);
@@ -647,9 +643,9 @@ public final class plasmaHTCache { return plasmaParser.supportedMimeTypesContains(mimeType);
}
- public static boolean noIndexingURL(String urlString) {
- if (urlString == null) return false;
- urlString = urlString.toLowerCase();
+ public static boolean noIndexingURL(yacyURL url) {
+ if (url == null) return false;
+ String urlString = url.toString().toLowerCase();
//http://www.yacy.net/getimage.php?image.png
@@ -978,30 +974,12 @@ public final class plasmaHTCache { return 0;
}
- public static boolean isPOST(String urlString) {
- return (urlString.indexOf("?") >= 0 ||
- urlString.indexOf("&") >= 0);
- }
-
- public static boolean isCGI(String urlString) {
- String ls = urlString.toLowerCase();
- return ((ls.indexOf(".cgi") >= 0) ||
- (ls.indexOf(".exe") >= 0) ||
- (ls.indexOf(";jsessionid=") >= 0) ||
- (ls.indexOf("sessionid/") >= 0) ||
- (ls.indexOf("phpsessid=") >= 0) ||
- (ls.indexOf("search.php?sid=") >= 0) ||
- (ls.indexOf("memberlist.php?sid=") >= 0));
- }
-
public static Entry newEntry(
Date initDate,
int depth,
yacyURL url,
String name,
- //httpHeader requestHeader,
- String responseStatus,
- //httpHeader responseHeader,
+ String responseStatus,
IResourceInfo docInfo,
String initiator,
plasmaCrawlProfile.entry profile
@@ -1010,10 +988,8 @@ public final class plasmaHTCache { initDate,
depth,
url,
- name,
- //requestHeader,
- responseStatus,
- //responseHeader,
+ name,
+ responseStatus,
docInfo,
initiator,
profile
@@ -1025,14 +1001,11 @@ public final class plasmaHTCache { // the class objects
private Date initDate; // the date when the request happened; will be used as a key
private int depth; // the depth of prefetching
-// private httpHeader requestHeader; // we carry also the header to prevent too many file system access
-// private httpHeader responseHeader; // we carry also the header to prevent too many file system access
private String responseStatus;
private File cacheFile; // the cache file
private byte[] cacheArray; // or the cache as byte-array
private yacyURL url;
private String name; // the name of the link, read as anchor from an <a>-tag
- //private int status; // cache load/hit/stale etc status
private Date lastModified;
private char doctype;
private String language;
@@ -1050,9 +1023,7 @@ public final class plasmaHTCache { this.depth,
this.url,
this.name,
- //this.requestHeader,
this.responseStatus,
- //this.responseHeader,
this.resInfo,
this.initiator,
this.profile
@@ -1063,9 +1034,7 @@ public final class plasmaHTCache { int depth,
yacyURL url,
String name,
- //httpHeader requestHeader,
String responseStatus,
- //httpHeader responseHeader,
IResourceInfo resourceInfo,
String initiator,
plasmaCrawlProfile.entry profile
@@ -1082,9 +1051,7 @@ public final class plasmaHTCache { // assigned:
this.initDate = initDate;
this.depth = depth;
- //this.requestHeader = requestHeader;
this.responseStatus = responseStatus;
- //this.responseHeader = responseHeader;
this.profile = profile;
this.initiator = (initiator == null) ? null : ((initiator.length() == 0) ? null : initiator);
@@ -1101,6 +1068,7 @@ public final class plasmaHTCache { }
public String name() {
+ // the anchor name; can be either the text inside the anchor tag or the page description after loading of the page
return this.name;
}
@@ -1155,14 +1123,6 @@ public final class plasmaHTCache { return this.cacheArray;
}
-// public httpHeader requestHeader() {
-// return this.requestHeader;
-// }
-
-// public httpHeader responseHeader() {
-// return this.responseHeader;
-// }
-
public IResourceInfo getDocumentInfo() {
return this.resInfo;
}
@@ -1217,10 +1177,8 @@ public final class plasmaHTCache { // check status code
if ((this.resInfo != null) && (!this.resInfo.validResponseStatus(this.responseStatus))) {
return "bad_status_" + this.responseStatus.substring(0,3);
- }
-// if (!(this.responseStatus.startsWith("200") ||
-// this.responseStatus.startsWith("203"))) { return "bad_status_" + this.responseStatus.substring(0,3); }
-
+ }
+
// check storage location
// sometimes a file name is equal to a path name in the same directory;
// or sometimes a file name is equal a directory name created earlier;
@@ -1231,8 +1189,8 @@ public final class plasmaHTCache { // -CGI access in request
// CGI access makes the page very individual, and therefore not usable in caches
- if (isPOST(this.url.toNormalform(true, true)) && !this.profile.crawlingQ()) { return "dynamic_post"; }
- if (isCGI(this.url.toNormalform(true, true))) { return "dynamic_cgi"; }
+ if (this.url.isPOST() && !this.profile.crawlingQ()) { return "dynamic_post"; }
+ if (this.url.isCGI()) { return "dynamic_cgi"; }
if (this.resInfo != null) {
return this.resInfo.shallStoreCacheForProxy();
@@ -1246,12 +1204,11 @@ public final class plasmaHTCache { * @return whether the file should be taken from the cache
*/
public boolean shallUseCacheForProxy() {
-// System.out.println("SHALL READ CACHE: requestHeader = " + requestHeader.toString() + ", responseHeader = " + responseHeader.toString());
// -CGI access in request
// CGI access makes the page very individual, and therefore not usable in caches
- if (isPOST(this.url.toNormalform(true, true))) { return false; }
- if (isCGI(this.url.toNormalform(true, true))) { return false; }
+ if (this.url.isPOST()) { return false; }
+ if (this.url.isCGI()) { return false; }
if (this.resInfo != null) {
return this.resInfo.shallUseCacheForProxy();
diff --git a/source/de/anomic/plasma/plasmaSnippetCache.java b/source/de/anomic/plasma/plasmaSnippetCache.java index fcd91a775..bed84ce1a 100644 --- a/source/de/anomic/plasma/plasmaSnippetCache.java +++ b/source/de/anomic/plasma/plasmaSnippetCache.java @@ -45,7 +45,6 @@ package de.anomic.plasma; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Enumeration; @@ -63,7 +62,6 @@ import de.anomic.http.httpc; import de.anomic.kelondro.kelondroMScoreCluster; import de.anomic.kelondro.kelondroMSetTools; import de.anomic.plasma.cache.IResourceInfo; -import de.anomic.plasma.crawler.plasmaCrawlerException; import de.anomic.plasma.parser.ParserException; import de.anomic.server.logging.serverLog; import de.anomic.yacy.yacySearch; @@ -284,7 +282,7 @@ public class plasmaSnippetCache { // if not found try to download it // download resource using the crawler and keep resource in memory if possible - plasmaHTCache.Entry entry = loadResourceFromWeb(url, timeout, true, true); + plasmaHTCache.Entry entry = plasmaSwitchboard.getSwitchboard().crawlQueues.loadResourceFromWeb(url, timeout, true, true); // getting resource metadata (e.g. the http headers for http resources) if (entry != null) { @@ -309,7 +307,7 @@ public class plasmaSnippetCache { return new TextSnippet(url, null, ERROR_SOURCE_LOADING, queryhashes, "no resource available"); } } catch (Exception e) { - if (!(e instanceof plasmaCrawlerException)) e.printStackTrace(); + e.printStackTrace(); return new TextSnippet(url, null, ERROR_SOURCE_LOADING, queryhashes, "error loading resource: " + e.getMessage()); } @@ -390,7 +388,7 @@ public class plasmaSnippetCache { // if not found try to download it // download resource using the crawler and keep resource in memory if possible - plasmaHTCache.Entry entry = loadResourceFromWeb(url, timeout, true, forText); + plasmaHTCache.Entry entry = plasmaSwitchboard.getSwitchboard().crawlQueues.loadResourceFromWeb(url, timeout, true, forText); // getting resource metadata (e.g. the http headers for http resources) if (entry != null) { @@ -820,7 +818,6 @@ public class plasmaSnippetCache { */ public static Object[] getResource(yacyURL url, boolean fetchOnline, int socketTimeout, boolean forText) { // load the url as resource from the web - try { long contentLength = -1; // trying to load the resource body from cache @@ -831,7 +828,7 @@ public class plasmaSnippetCache { // if the content is not available in cache try to download it from web // try to download the resource using a crawler - plasmaHTCache.Entry entry = loadResourceFromWeb(url, (socketTimeout < 0) ? -1 : socketTimeout, true, forText); + plasmaHTCache.Entry entry = plasmaSwitchboard.getSwitchboard().crawlQueues.loadResourceFromWeb(url, (socketTimeout < 0) ? -1 : socketTimeout, true, forText); // read resource body (if it is there) byte[] resourceArray = entry.cacheArray(); @@ -848,30 +845,6 @@ public class plasmaSnippetCache { return null; } return new Object[]{resource,new Long(contentLength)}; - } catch (IOException e) { - return null; - } - } - - public static plasmaHTCache.Entry loadResourceFromWeb( - yacyURL url, - int socketTimeout, - boolean keepInMemory, - boolean forText - ) throws plasmaCrawlerException { - - plasmaHTCache.Entry result = plasmaSwitchboard.getSwitchboard().cacheLoader.loadSync( - url, // the url - "", // name of the url, from anchor tag <a>name</a> - null, // referer - yacyCore.seedDB.mySeed().hash, // initiator - 0, // depth - (forText) ? plasmaSwitchboard.getSwitchboard().defaultTextSnippetProfile : plasmaSwitchboard.getSwitchboard().defaultMediaSnippetProfile, // crawl profile - socketTimeout, - keepInMemory - ); - - return result; } public static String failConsequences(TextSnippet snippet, String eventID) { diff --git a/source/de/anomic/plasma/plasmaSwitchboard.java b/source/de/anomic/plasma/plasmaSwitchboard.java index dc3c99e9e..d35c79bec 100644 --- a/source/de/anomic/plasma/plasmaSwitchboard.java +++ b/source/de/anomic/plasma/plasmaSwitchboard.java @@ -149,6 +149,8 @@ import de.anomic.kelondro.kelondroException; import de.anomic.kelondro.kelondroMSetTools;
import de.anomic.kelondro.kelondroMapTable;
import de.anomic.kelondro.kelondroNaturalOrder;
+import de.anomic.plasma.crawler.plasmaCrawlQueues;
+import de.anomic.plasma.crawler.plasmaProtocolLoader;
import de.anomic.plasma.dbImport.dbImportManager;
import de.anomic.plasma.parser.ParserException;
import de.anomic.plasma.urlPattern.defaultURLPattern;
@@ -162,7 +164,6 @@ import de.anomic.server.serverSemaphore; import de.anomic.server.serverSwitch;
import de.anomic.server.serverThread;
import de.anomic.server.logging.serverLog;
-import de.anomic.tools.crypt;
import de.anomic.yacy.yacyURL;
import de.anomic.yacy.yacyVersion;
import de.anomic.yacy.yacyClient;
@@ -176,7 +177,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // load slots
public static int crawlSlots = 10;
public static int indexingSlots = 30;
- public static int stackCrawlSlots = 1000000;
+ public static int stackCrawlSlots = 2000;
private int dhtTransferIndexCount = 100;
@@ -213,12 +214,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public File workPath;
public File releasePath;
public HashMap rankingPermissions;
- public plasmaCrawlNURL noticeURL;
- public plasmaCrawlZURL errorURL, delegatedURL;
public plasmaWordIndex wordIndex;
- public plasmaCrawlLoader cacheLoader;
+ public plasmaCrawlQueues crawlQueues;
public plasmaSwitchboardQueue sbQueue;
- public plasmaCrawlStacker sbStackCrawlThread;
+ public plasmaCrawlStacker crawlStacker;
public messageBoard messageDB;
public wikiBoard wikiDB;
public blogBoard blogDB;
@@ -274,7 +273,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser /*
* Some constants
*/
- private static final String STR_REMOTECRAWLTRIGGER = "REMOTECRAWLTRIGGER: REMOTE CRAWL TO PEER ";
+ public static final String STR_REMOTECRAWLTRIGGER = "REMOTECRAWLTRIGGER: REMOTE CRAWL TO PEER ";
private serverSemaphore shutdownSync = new serverSemaphore(0);
private boolean terminate = false;
@@ -282,8 +281,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser //private Object crawlingPausedSync = new Object();
//private boolean crawlingIsPaused = false;
- private static final int CRAWLJOB_SYNC = 0;
- private static final int CRAWLJOB_STATUS = 1;
+ public static final int CRAWLJOB_SYNC = 0;
+ public static final int CRAWLJOB_STATUS = 1;
//////////////////////////////////////////////////////////////////////////////////////////////
// Thread settings
@@ -883,7 +882,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public static final String DBFILE_USER = "DATA/SETTINGS/user.db";
- private Hashtable crawlJobsStatus = new Hashtable();
+ public Hashtable crawlJobsStatus = new Hashtable();
private static plasmaSwitchboard sb;
@@ -1145,10 +1144,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // start indexing management
log.logConfig("Starting Indexing Management");
- noticeURL = new plasmaCrawlNURL(plasmaPath);
- //errorURL = new plasmaCrawlZURL(); // fresh error DB each startup; can be hold in RAM and reduces IO;
- errorURL = new plasmaCrawlZURL(plasmaPath, "urlError1.db", true);
- delegatedURL = new plasmaCrawlZURL(plasmaPath, "urlDelegated1.db", false);
wordIndex = new plasmaWordIndex(indexPrimaryPath, indexSecondaryPath, ramRWI_time, log);
// set a high maximum cache size to current size; this is adopted later automatically
@@ -1210,8 +1205,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // start a loader
log.logConfig("Starting Crawl Loader");
crawlSlots = Integer.parseInt(getConfig(CRAWLER_THREADS_ACTIVE_MAX, "10"));
- plasmaCrawlLoader.switchboard = this;
- this.cacheLoader = new plasmaCrawlLoader(this.log);
+ this.crawlQueues = new plasmaCrawlQueues(this, plasmaPath);
/*
* Creating sync objects and loading status for the crawl jobs
@@ -1313,7 +1307,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser }
// initializing the stackCrawlThread
- this.sbStackCrawlThread = new plasmaCrawlStacker(this, this.plasmaPath, ramPreNURL_time, (int) getConfigLong("tableTypeForPreNURL", 0));
+ this.crawlStacker = new plasmaCrawlStacker(this, this.plasmaPath, ramPreNURL_time, (int) getConfigLong("tableTypeForPreNURL", 0));
//this.sbStackCrawlThread = new plasmaStackCrawlThread(this,this.plasmaPath,ramPreNURL);
//this.sbStackCrawlThread.start();
@@ -1335,7 +1329,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser deployThread(CLEANUP, "Cleanup", "simple cleaning process for monitoring information", null,
new serverInstantThread(this, CLEANUP_METHOD_START, CLEANUP_METHOD_JOBCOUNT, CLEANUP_METHOD_FREEMEM), 10000); // all 5 Minutes
deployThread(CRAWLSTACK, "Crawl URL Stacker", "process that checks url for double-occurrences and for allowance/disallowance by robots.txt", null,
- new serverInstantThread(sbStackCrawlThread, CRAWLSTACK_METHOD_START, CRAWLSTACK_METHOD_JOBCOUNT, CRAWLSTACK_METHOD_FREEMEM), 8000);
+ new serverInstantThread(crawlStacker, CRAWLSTACK_METHOD_START, CRAWLSTACK_METHOD_JOBCOUNT, CRAWLSTACK_METHOD_FREEMEM), 8000);
deployThread(INDEXER, "Parsing/Indexing", "thread that performes document parsing and indexing", "/IndexCreateIndexingQueue_p.html",
new serverInstantThread(this, INDEXER_METHOD_START, INDEXER_METHOD_JOBCOUNT, INDEXER_METHOD_FREEMEM), 10000);
@@ -1352,11 +1346,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser deployThread(PROXY_CACHE_ENQUEUE, "Proxy Cache Enqueue", "job takes new input files from RAM stack, stores them, and hands over to the Indexing Stack", null,
new serverInstantThread(this, PROXY_CACHE_ENQUEUE_METHOD_START, PROXY_CACHE_ENQUEUE_METHOD_JOBCOUNT, PROXY_CACHE_ENQUEUE_METHOD_FREEMEM), 10000);
deployThread(CRAWLJOB_REMOTE_TRIGGERED_CRAWL, "Remote Crawl Job", "thread that performes a single crawl/indexing step triggered by a remote peer", null,
- new serverInstantThread(this, CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM), 30000);
+ new serverInstantThread(crawlQueues, CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_START, CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_JOBCOUNT, CRAWLJOB_REMOTE_TRIGGERED_CRAWL_METHOD_FREEMEM), 30000);
deployThread(CRAWLJOB_GLOBAL_CRAWL_TRIGGER, "Global Crawl Trigger", "thread that triggeres remote peers for crawling", "/IndexCreateWWWGlobalQueue_p.html",
- new serverInstantThread(this, CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_START, CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_JOBCOUNT, CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_FREEMEM), 30000); // error here?
+ new serverInstantThread(crawlQueues, CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_START, CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_JOBCOUNT, CRAWLJOB_GLOBAL_CRAWL_TRIGGER_METHOD_FREEMEM), 30000); // error here?
deployThread(CRAWLJOB_LOCAL_CRAWL, "Local Crawl", "thread that performes a single crawl step from the local crawl queue", "/IndexCreateWWWLocalQueue_p.html",
- new serverInstantThread(this, CRAWLJOB_LOCAL_CRAWL_METHOD_START, CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM), 10000);
+ new serverInstantThread(crawlQueues, CRAWLJOB_LOCAL_CRAWL_METHOD_START, CRAWLJOB_LOCAL_CRAWL_METHOD_JOBCOUNT, CRAWLJOB_LOCAL_CRAWL_METHOD_FREEMEM), 10000);
deployThread(SEED_UPLOAD, "Seed-List Upload", "task that a principal peer performes to generate and upload a seed-list to a ftp account", null,
new serverInstantThread(yc, SEED_UPLOAD_METHOD_START, SEED_UPLOAD_METHOD_JOBCOUNT, SEED_UPLOAD_METHOD_FREEMEM), 180000);
serverInstantThread peerPing = null;
@@ -1487,13 +1481,12 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser if (url == null) return false;
String host = url.getHost();
if (host == null) return false;
- return acceptURL(serverDomains.dnsResolve(host));
- }
-
- public boolean acceptURL(InetAddress hostAddress) {
- // returns true if the url can be accepted accoring to network.unit.domain
- if (hostAddress == null) return false; // if we don't know the host, we cannot load that resource anyway
- if (this.acceptGlobalURLs && this.acceptLocalURLs) return true; // fast shortcut
+ if (this.acceptGlobalURLs && this.acceptLocalURLs) return true; // fast shortcut to avoid dnsResolve
+ InetAddress hostAddress = serverDomains.dnsResolve(host);
+ // if we don't know the host, we cannot load that resource anyway.
+ // But in case we use a proxy, it is possible that we dont have a DNS service.
+ if (hostAddress == null) return ((this.remoteProxyConfig != null) && (this.remoteProxyConfig.useProxy()));
+ // check if this is a local address and we are allowed to index local pages:
boolean local = hostAddress.isSiteLocalAddress() || hostAddress.isLoopbackAddress();
return (this.acceptGlobalURLs && !local) || (this.acceptLocalURLs && local);
}
@@ -1503,29 +1496,20 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // if it exists, the name of the database is returned,
// if it not exists, null is returned
if (wordIndex.loadedURL.exists(hash)) return "loaded";
- if (noticeURL.existsInStack(hash)) return "crawler";
- if (delegatedURL.exists(hash)) return "delegated";
- if (errorURL.exists(hash)) return "errors";
- return null;
+ return this.crawlQueues.urlExists(hash);
}
public void urlRemove(String hash) {
wordIndex.loadedURL.remove(hash);
- noticeURL.removeByURLHash(hash);
- delegatedURL.remove(hash);
- errorURL.remove(hash);
+ crawlQueues.urlRemove(hash);
}
- public yacyURL getURL(String urlhash) throws IOException {
+ public yacyURL getURL(String urlhash) {
if (urlhash.equals(yacyURL.dummyHash)) return null;
- plasmaCrawlEntry ne = noticeURL.get(urlhash);
- if (ne != null) return ne.url();
+ yacyURL ne = crawlQueues.getURL(urlhash);
+ if (ne != null) return ne;
indexURLEntry le = wordIndex.loadedURL.load(urlhash, null);
if (le != null) return le.comp().url();
- plasmaCrawlZURL.Entry ee = delegatedURL.getEntry(urlhash);
- if (ee != null) return ee.url();
- ee = errorURL.getEntry(urlhash);
- if (ee != null) return ee.url();
return null;
}
@@ -1610,7 +1594,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser * {@link plasmaCrawlProfile Crawl Profiles} are saved independantly from the queues themselves
* and therefore have to be cleaned up from time to time. This method only performs the clean-up
* if - and only if - the {@link plasmaSwitchboardQueue switchboard},
- * {@link plasmaCrawlLoader loader} and {@link plasmaCrawlNURL local crawl} queues are all empty.
+ * {@link plasmaProtocolLoader loader} and {@link plasmaCrawlNURL local crawl} queues are all empty.
* <p>
* Then it iterates through all existing {@link plasmaCrawlProfile crawl profiles} and removes
* all profiles which are not hardcoded.
@@ -1629,9 +1613,9 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser * shutdown procedure
*/
public boolean cleanProfiles() throws InterruptedException {
- if ((sbQueue.size() > 0) || (cacheLoader.size() > 0) ||
- (sbStackCrawlThread != null && sbStackCrawlThread.size() > 0) ||
- (noticeURL.notEmpty()))
+ if ((sbQueue.size() > 0) || (crawlQueues.size() > 0) ||
+ (crawlStacker != null && crawlStacker.size() > 0) ||
+ (crawlQueues.noticeURL.notEmpty()))
return false;
final Iterator iter = profilesActiveCrawls.profiles(true);
plasmaCrawlProfile.entry entry;
@@ -1658,14 +1642,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser }
return hasDoneSomething;
}
- /*
- synchronized public void htEntryStoreEnqueued(plasmaHTCache.Entry entry) {
- if (plasmaHTCache.full())
- htEntryStoreProcess(entry);
- else
- plasmaHTCache.push(entry);
- }
- */
+
synchronized public boolean htEntryStoreProcess(plasmaHTCache.Entry entry) {
if (entry == null) return false;
@@ -1695,14 +1672,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser *
* check if ip is local ip address // TODO: remove this procotol specific code here
* ========================================================================= */
- InetAddress hostAddress = serverDomains.dnsResolve(entry.url().getHost());
- if (hostAddress == null) {
- if (this.remoteProxyConfig == null || !this.remoteProxyConfig.useProxy()) {
- this.log.logFine("Unknown host in URL '" + entry.url() + "'. Will not be indexed.");
- doIndexing = false;
- }
- } else if (!acceptURL(hostAddress)) {
- this.log.logFine("Host in URL '" + entry.url() + "' has private ip address. Will not be indexed.");
+ if (!acceptURL(entry.url())) {
+ this.log.logFine("Host in URL '" + entry.url() + "' is not in defined indexing domain.");
doIndexing = false;
}
@@ -1759,7 +1730,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser ));
} else {
if (!entry.profile().storeHTCache() && entry.cacheFile().exists()) {
- plasmaHTCache.deleteFile(entry.url());
+ plasmaHTCache.deleteURLfromCache(entry.url());
}
}
@@ -1782,7 +1753,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser log.logConfig("SWITCHBOARD SHUTDOWN STEP 2: sending termination signal to threaded indexing");
// closing all still running db importer jobs
this.dbImportManager.close();
- cacheLoader.close();
+ crawlQueues.close();
wikiDB.close();
blogDB.close();
blogCommentDB.close();
@@ -1790,7 +1761,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser bookmarksDB.close();
messageDB.close();
if (facilityDB != null) facilityDB.close();
- sbStackCrawlThread.close();
+ crawlStacker.close();
profilesActiveCrawls.close();
robots.close();
parser.close();
@@ -1800,9 +1771,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser webStructure.flushCitationReference("crg");
webStructure.close();
log.logConfig("SWITCHBOARD SHUTDOWN STEP 3: sending termination signal to database manager (stand by...)");
- noticeURL.close();
- delegatedURL.close();
- errorURL.close();
wordIndex.close();
yc.close();
log.logConfig("SWITCHBOARD SHUTDOWN TERMINATED");
@@ -1849,8 +1817,7 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser boolean doneSomething = false;
// possibly delete entries from last chunk
- if ((this.dhtTransferChunk != null) &&
- (this.dhtTransferChunk.getStatus() == plasmaDHTChunk.chunkStatus_COMPLETE)) {
+ if ((this.dhtTransferChunk != null) && (this.dhtTransferChunk.getStatus() == plasmaDHTChunk.chunkStatus_COMPLETE)) {
String deletedURLs = this.dhtTransferChunk.deleteTransferIndexes();
this.log.logFine("Deleted from " + this.dhtTransferChunk.containers().length + " transferred RWIs locally, removed " + deletedURLs + " URL references");
this.dhtTransferChunk = null;
@@ -1883,16 +1850,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser return doneSomething; // nothing to do
}
- /*
- if (wordIndex.wordCacheRAMSize() + 1000 > (int) getConfigLong("wordCacheMaxLow", 8000)) {
- log.logFine("deQueue: word index ram cache too full (" + ((int) getConfigLong("wordCacheMaxLow", 8000) - wordIndex.wordCacheRAMSize()) + " slots left); dismissed to omit ram flush lock");
- return false;
- }
- */
-
- int stackCrawlQueueSize;
- if ((stackCrawlQueueSize = sbStackCrawlThread.size()) >= stackCrawlSlots) {
- log.logFine("deQueue: too many processes in stack crawl thread queue, dismissed to protect emergency case (" + "stackCrawlQueue=" + stackCrawlQueueSize + ")");
+ if (crawlStacker.size() >= stackCrawlSlots) {
+ log.logFine("deQueue: too many processes in stack crawl thread queue (" + "stackCrawlQueue=" + crawlStacker.size() + ")");
return doneSomething;
}
@@ -1906,10 +1865,10 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // do one processing step
log.logFine("DEQUEUE: sbQueueSize=" + sbQueue.size() +
- ", coreStackSize=" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) +
- ", limitStackSize=" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) +
- ", overhangStackSize=" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_OVERHANG) +
- ", remoteStackSize=" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE));
+ ", coreStackSize=" + crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) +
+ ", limitStackSize=" + crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) +
+ ", overhangStackSize=" + crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_OVERHANG) +
+ ", remoteStackSize=" + crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE));
try {
int sizeBefore = sbQueue.size();
nextentry = sbQueue.pop();
@@ -1946,8 +1905,8 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser public int cleanupJobSize() {
int c = 0;
- if ((delegatedURL.stackSize() > 1000)) c++;
- if ((errorURL.stackSize() > 1000)) c++;
+ if ((crawlQueues.delegatedURL.stackSize() > 1000)) c++;
+ if ((crawlQueues.errorURL.stackSize() > 1000)) c++;
for (int i = 1; i <= 6; i++) {
if (wordIndex.loadedURL.getStackSize(i) > 1000) c++;
}
@@ -1970,17 +1929,17 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // clean up delegated stack
checkInterruption();
- if ((delegatedURL.stackSize() > 1000)) {
- log.logFine("Cleaning Delegated-URLs report stack, " + delegatedURL.stackSize() + " entries on stack");
- delegatedURL.clearStack();
+ if ((crawlQueues.delegatedURL.stackSize() > 1000)) {
+ log.logFine("Cleaning Delegated-URLs report stack, " + crawlQueues.delegatedURL.stackSize() + " entries on stack");
+ crawlQueues.delegatedURL.clearStack();
hasDoneSomething = true;
}
// clean up error stack
checkInterruption();
- if ((errorURL.stackSize() > 1000)) {
- log.logFine("Cleaning Error-URLs report stack, " + errorURL.stackSize() + " entries on stack");
- errorURL.clearStack();
+ if ((crawlQueues.errorURL.stackSize() > 1000)) {
+ log.logFine("Cleaning Error-URLs report stack, " + crawlQueues.errorURL.stackSize() + " entries on stack");
+ crawlQueues.errorURL.clearStack();
hasDoneSomething = true;
}
@@ -2147,228 +2106,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser }
}
- public int coreCrawlJobSize() {
- return noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE);
- }
-
- public boolean coreCrawlJob() {
- if (noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) == 0) {
- //log.logDebug("CoreCrawl: queue is empty");
- return false;
- }
- if (sbQueue.size() >= indexingSlots) {
- log.logFine("CoreCrawl: too many processes in indexing queue, dismissed (" +
- "sbQueueSize=" + sbQueue.size() + ")");
- return false;
- }
- if (cacheLoader.size() >= crawlSlots) {
- log.logFine("CoreCrawl: too many processes in loader queue, dismissed (" +
- "cacheLoader=" + cacheLoader.size() + ")");
- return false;
- }
- if (onlineCaution()) {
- log.logFine("CoreCrawl: online caution, omitting processing");
- return false;
- }
- // if the server is busy, we do crawling more slowly
- //if (!(cacheManager.idle())) try {Thread.currentThread().sleep(2000);} catch (InterruptedException e) {}
-
- // if crawling was paused we have to wait until we wer notified to continue
- Object[] status = (Object[])this.crawlJobsStatus.get(CRAWLJOB_LOCAL_CRAWL);
- synchronized(status[CRAWLJOB_SYNC]) {
- if (((Boolean)status[CRAWLJOB_STATUS]).booleanValue()) {
- try {
- status[CRAWLJOB_SYNC].wait();
- }
- catch (InterruptedException e){ return false;}
- }
- }
-
- // do a local crawl
- plasmaCrawlEntry urlEntry = null;
- while (urlEntry == null && noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) > 0) {
- String stats = "LOCALCRAWL[" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_OVERHANG) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE) + "]";
- try {
- urlEntry = noticeURL.pop(plasmaCrawlNURL.STACK_TYPE_CORE, true);
- String profileHandle = urlEntry.profileHandle();
- // System.out.println("DEBUG plasmaSwitchboard.processCrawling:
- // profileHandle = " + profileHandle + ", urlEntry.url = " + urlEntry.url());
- if (profileHandle == null) {
- log.logSevere(stats + ": NULL PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url());
- return true;
- }
- plasmaCrawlProfile.entry profile = profilesActiveCrawls.getEntry(profileHandle);
- if (profile == null) {
- log.logWarning(stats + ": LOST PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url());
- return true;
- }
- log.logFine("LOCALCRAWL: URL=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter=" + profile.generalFilter()
- + ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false")));
-
- processLocalCrawling(urlEntry, profile, stats);
- return true;
- } catch (IOException e) {
- log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e);
- if (e.getMessage().indexOf("hash is null") > 0) noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_CORE);
- }
- }
- return true;
- }
-
- public int limitCrawlTriggerJobSize() {
- return noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT);
- }
-
- public boolean limitCrawlTriggerJob() {
- if (noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) == 0) {
- //log.logDebug("LimitCrawl: queue is empty");
- return false;
- }
- boolean robinsonPrivateCase = ((isRobinsonMode()) &&
- (!getConfig(CLUSTER_MODE, "").equals(CLUSTER_MODE_PUBLIC_CLUSTER)) &&
- (!getConfig(CLUSTER_MODE, "").equals(CLUSTER_MODE_PRIVATE_CLUSTER)));
-
- if ((robinsonPrivateCase) || ((coreCrawlJobSize() <= 20) && (limitCrawlTriggerJobSize() > 10))) {
- // it is not efficient if the core crawl job is empty and we have too much to do
- // move some tasks to the core crawl job
- int toshift = 10; // this cannot be a big number because the balancer makes a forced waiting if it cannot balance
- if (toshift > limitCrawlTriggerJobSize()) toshift = limitCrawlTriggerJobSize();
- for (int i = 0; i < toshift; i++) {
- noticeURL.shift(plasmaCrawlNURL.STACK_TYPE_LIMIT, plasmaCrawlNURL.STACK_TYPE_CORE);
- }
- log.logInfo("shifted " + toshift + " jobs from global crawl to local crawl (coreCrawlJobSize()=" + coreCrawlJobSize() + ", limitCrawlTriggerJobSize()=" + limitCrawlTriggerJobSize() + ", cluster.mode=" + getConfig(CLUSTER_MODE, "") + ", robinsonMode=" + ((isRobinsonMode()) ? "on" : "off"));
- if (robinsonPrivateCase) return false;
- }
-
- // check local indexing queues
- // in case the placing of remote crawl fails, there must be space in the local queue to work off the remote crawl
- if (sbQueue.size() >= indexingSlots * 2) {
- log.logFine("LimitCrawl: too many processes in indexing queue, dismissed (" +
- "sbQueueSize=" + sbQueue.size() + ")");
- return false;
- }
- if (cacheLoader.size() >= crawlSlots) {
- log.logFine("LimitCrawl: too many processes in loader queue, dismissed (" +
- "cacheLoader=" + cacheLoader.size() + ")");
- return false;
- }
- if (onlineCaution()) {
- log.logFine("LimitCrawl: online caution, omitting processing");
- return false;
- }
-
- // if crawling was paused we have to wait until we were notified to continue
- Object[] status = (Object[])this.crawlJobsStatus.get(CRAWLJOB_GLOBAL_CRAWL_TRIGGER);
- synchronized(status[CRAWLJOB_SYNC]) {
- if (((Boolean)status[CRAWLJOB_STATUS]).booleanValue()) {
- try {
- status[CRAWLJOB_SYNC].wait();
- }
- catch (InterruptedException e){ return false;}
- }
- }
-
- // start a global crawl, if possible
- String stats = "REMOTECRAWLTRIGGER[" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_OVERHANG) + ", "
- + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE) + "]";
- try {
- plasmaCrawlEntry urlEntry = noticeURL.pop(plasmaCrawlNURL.STACK_TYPE_LIMIT, true);
- String profileHandle = urlEntry.profileHandle();
- // System.out.println("DEBUG plasmaSwitchboard.processCrawling:
- // profileHandle = " + profileHandle + ", urlEntry.url = " + urlEntry.url());
- plasmaCrawlProfile.entry profile = profilesActiveCrawls.getEntry(profileHandle);
- if (profile == null) {
- log.logWarning(stats + ": LOST PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url());
- return true;
- }
- log.logFine("plasmaSwitchboard.limitCrawlTriggerJob: url=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter="
- + profile.generalFilter() + ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false")));
-
- boolean tryRemote = ((noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) != 0) || (sbQueue.size() != 0)) &&
- (profile.remoteIndexing()) &&
- (urlEntry.initiator() != null) &&
- // (!(urlEntry.initiator().equals(indexURL.dummyHash))) &&
- ((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal()));
- if (tryRemote) {
- boolean success = processRemoteCrawlTrigger(urlEntry);
- if (success) return true;
- }
-
- processLocalCrawling(urlEntry, profile, stats); // emergency case, work off the crawl locally
- return true;
- } catch (IOException e) {
- log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e);
- if (e.getMessage().indexOf("hash is null") > 0) noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_LIMIT);
- return true; // if we return a false here we will block everything
- }
- }
-
- public int remoteTriggeredCrawlJobSize() {
- return noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE);
- }
-
- public boolean remoteTriggeredCrawlJob() {
- // work off crawl requests that had been placed by other peers to our crawl stack
-
- // do nothing if either there are private processes to be done
- // or there is no global crawl on the stack
- if (noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE) == 0) {
- //log.logDebug("GlobalCrawl: queue is empty");
- return false;
- }
- if (sbQueue.size() >= indexingSlots) {
- log.logFine("GlobalCrawl: too many processes in indexing queue, dismissed (" +
- "sbQueueSize=" + sbQueue.size() + ")");
- return false;
- }
- if (cacheLoader.size() >= crawlSlots) {
- log.logFine("GlobalCrawl: too many processes in loader queue, dismissed (" +
- "cacheLoader=" + cacheLoader.size() + ")");
- return false;
- }
- if (onlineCaution()) {
- log.logFine("GlobalCrawl: online caution, omitting processing");
- return false;
- }
-
- // if crawling was paused we have to wait until we wer notified to continue
- Object[] status = (Object[])this.crawlJobsStatus.get(CRAWLJOB_REMOTE_TRIGGERED_CRAWL);
- synchronized(status[CRAWLJOB_SYNC]) {
- if (((Boolean)status[CRAWLJOB_STATUS]).booleanValue()) {
- try {
- status[CRAWLJOB_SYNC].wait();
- }
- catch (InterruptedException e){ return false;}
- }
- }
-
- // we don't want to crawl a global URL globally, since WE are the global part. (from this point of view)
- String stats = "REMOTETRIGGEREDCRAWL[" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_OVERHANG) + ", "
- + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE) + "]";
- try {
- plasmaCrawlEntry urlEntry = noticeURL.pop(plasmaCrawlNURL.STACK_TYPE_REMOTE, true);
- String profileHandle = urlEntry.profileHandle();
- // System.out.println("DEBUG plasmaSwitchboard.processCrawling:
- // profileHandle = " + profileHandle + ", urlEntry.url = " +
- // urlEntry.url());
- plasmaCrawlProfile.entry profile = profilesActiveCrawls.getEntry(profileHandle);
-
- if (profile == null) {
- log.logWarning(stats + ": LOST PROFILE HANDLE '" + urlEntry.profileHandle() + "' for URL " + urlEntry.url());
- return false;
- }
- log.logFine("plasmaSwitchboard.remoteTriggeredCrawlJob: url=" + urlEntry.url() + ", initiator=" + urlEntry.initiator() + ", crawlOrder=" + ((profile.remoteIndexing()) ? "true" : "false") + ", depth=" + urlEntry.depth() + ", crawlDepth=" + profile.generalDepth() + ", filter="
- + profile.generalFilter() + ", permission=" + ((yacyCore.seedDB == null) ? "undefined" : (((yacyCore.seedDB.mySeed().isSenior()) || (yacyCore.seedDB.mySeed().isPrincipal())) ? "true" : "false")));
-
- processLocalCrawling(urlEntry, profile, stats);
- return true;
- } catch (IOException e) {
- log.logSevere(stats + ": CANNOT FETCH ENTRY: " + e.getMessage(), e);
- if (e.getMessage().indexOf("hash is null") > 0) noticeURL.clear(plasmaCrawlNURL.STACK_TYPE_REMOTE);
- return true;
- }
- }
-
private plasmaParserDocument parseResource(plasmaSwitchboardQueue.Entry entry, String initiatorHash) throws InterruptedException, ParserException {
// the mimetype of this entry
@@ -2474,11 +2211,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser nextUrl = new yacyURL(nextUrlString, null);
// enqueue the hyperlink into the pre-notice-url db
- sbStackCrawlThread.enqueue(nextUrl, entry.urlHash(), initiatorPeerHash, (String) nextEntry.getValue(), docDate, entry.depth() + 1, entry.profile(), entry.depth() <= 1);
+ crawlStacker.enqueueEntry(nextUrl, entry.urlHash(), initiatorPeerHash, (String) nextEntry.getValue(), docDate, entry.depth() + 1, entry.profile(), ((entry.depth() <= 1) || (entry.depth() + 1 >= entry.profile().generalDepth())));
} catch (MalformedURLException e1) {}
}
log.logInfo("CRAWL: ADDED " + hl.size() + " LINKS FROM " + entry.url().toNormalform(false, true) +
- ", NEW CRAWL STACK SIZE IS " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE));
+ ", NEW CRAWL STACK SIZE IS " + crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE));
}
stackEndTime = System.currentTimeMillis();
@@ -2763,137 +2500,13 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser // explicit delete/free resources
if ((entry != null) && (entry.profile() != null) && (!(entry.profile().storeHTCache()))) {
plasmaHTCache.filesInUse.remove(entry.cacheFile());
- plasmaHTCache.deleteFile(entry.url());
+ //plasmaHTCache.deleteURLfromCache(entry.url());
}
entry = null;
if (document != null) try { document.close(); } catch (Exception e) { /* ignore this */ }
}
}
-
- private void processLocalCrawling(plasmaCrawlEntry urlEntry, plasmaCrawlProfile.entry profile, String stats) {
- // work off one Crawl stack entry
- if ((urlEntry == null) || (urlEntry.url() == null)) {
- log.logInfo(stats + ": urlEntry=null");
- return;
- }
-
- // convert the referrer hash into the corresponding URL
- yacyURL refererURL = null;
- String refererHash = urlEntry.referrerhash();
- if ((refererHash != null) && (!refererHash.equals(yacyURL.dummyHash))) try {
- refererURL = this.getURL(refererHash);
- } catch (IOException e) {
- refererURL = null;
- }
- cacheLoader.loadAsync(urlEntry.url(), urlEntry.name(), (refererURL!=null)?refererURL.toString():null, urlEntry.initiator(), urlEntry.depth(), profile, -1, false);
- log.logInfo(stats + ": enqueued for load " + urlEntry.url() + " [" + urlEntry.url().hash() + "]");
- return;
- }
-
- private boolean processRemoteCrawlTrigger(plasmaCrawlEntry urlEntry) {
- // if this returns true, then the urlEntry is considered as stored somewhere and the case is finished
- // if this returns false, the urlEntry will be enqueued to the local crawl again
-
- // wrong access
- if (urlEntry == null) {
- log.logInfo("REMOTECRAWLTRIGGER[" + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_CORE) + ", " + noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_REMOTE) + "]: urlEntry=null");
- return true; // superfluous request; true correct in this context because the urlEntry shall not be tracked any more
- }
-
- // check url
- if (urlEntry.url() == null) {
- log.logFine("ERROR: plasmaSwitchboard.processRemoteCrawlTrigger - url is null. name=" + urlEntry.name());
- return true; // same case as above: no more consideration
- }
-
- // are we qualified for a remote crawl?
- if ((yacyCore.seedDB.mySeed() == null) || (yacyCore.seedDB.mySeed().isJunior())) {
- log.logFine("plasmaSwitchboard.processRemoteCrawlTrigger: no permission");
- return false; // no, we must crawl this page ourselves
- }
-
- // check if peer for remote crawl is available
- yacySeed remoteSeed = ((this.isPublicRobinson()) && (getConfig("cluster.mode", "").equals("publiccluster"))) ?
- yacyCore.dhtAgent.getPublicClusterCrawlSeed(urlEntry.url().hash(), this.clusterhashes) :
- yacyCore.dhtAgent.getGlobalCrawlSeed(urlEntry.url().hash());
- if (remoteSeed == null) {
- log.logFine("plasmaSwitchboard.processRemoteCrawlTrigger: no remote crawl seed available");
- return false;
- }
-
- // do the request
- HashMap page = null;
- try {
- page = yacyClient.crawlOrder(remoteSeed, urlEntry.url(), getURL(urlEntry.referrerhash()), 6000);
- } catch (IOException e1) {
- log.logSevere(STR_REMOTECRAWLTRIGGER + remoteSeed.getName() + " FAILED. URL CANNOT BE RETRIEVED from referrer hash: " + urlEntry.referrerhash(), e1);
- return false;
- }
-
- // check if we got contact to peer and the peer respondet
- if ((page == null) || (page.get("delay") == null)) {
- log.logInfo("CRAWL: REMOTE CRAWL TO PEER " + remoteSeed.getName() + " FAILED. CAUSE: unknown (URL=" + urlEntry.url().toString() + "). Removed peer.");
- yacyCore.peerActions.peerDeparture(remoteSeed, "remote crawl to peer failed; peer answered unappropriate");
- return false; // no response from peer, we will crawl this ourself
- }
-
- String response = (String) page.get("response");
- log.logFine("plasmaSwitchboard.processRemoteCrawlTrigger: remoteSeed="
- + remoteSeed.getName() + ", url=" + urlEntry.url().toString()
- + ", response=" + page.toString()); // DEBUG
-
- // we received an answer and we are told to wait a specific time until we shall ask again for another crawl
- int newdelay = Integer.parseInt((String) page.get("delay"));
- yacyCore.dhtAgent.setCrawlDelay(remoteSeed.hash, newdelay);
- if (response.equals("stacked")) {
- // success, the remote peer accepted the crawl
- log.logInfo(STR_REMOTECRAWLTRIGGER + remoteSeed.getName()
- + " PLACED URL=" + urlEntry.url().toString()
- + "; NEW DELAY=" + newdelay);
- // track this remote crawl
- this.delegatedURL.newEntry(urlEntry, remoteSeed.hash, new Date(), 0, response).store();
- return true;
- }
-
- // check other cases: the remote peer may respond that it already knows that url
- if (response.equals("double")) {
- // in case the peer answers double, it transmits the complete lurl data
- String lurl = (String) page.get("lurl");
- if ((lurl != null) && (lurl.length() != 0)) {
- String propStr = crypt.simpleDecode(lurl, (String) page.get("key"));
- indexURLEntry entry = wordIndex.loadedURL.newEntry(propStr);
- try {
- wordIndex.loadedURL.store(entry);
- wordIndex.loadedURL.stack(entry, yacyCore.seedDB.mySeed().hash, remoteSeed.hash, 1); // *** ueberfluessig/doppelt?
- // noticeURL.remove(entry.hash());
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
-
- log.logInfo(STR_REMOTECRAWLTRIGGER + remoteSeed.getName()
- + " SUPERFLUOUS. CAUSE: " + page.get("reason")
- + " (URL=" + urlEntry.url().toString()
- + "). URL IS CONSIDERED AS 'LOADED!'");
- return true;
- } else {
- log.logInfo(STR_REMOTECRAWLTRIGGER + remoteSeed.getName()
- + " REJECTED. CAUSE: bad lurl response / " + page.get("reason") + " (URL="
- + urlEntry.url().toString() + ")");
- remoteSeed.setFlagAcceptRemoteCrawl(false);
- yacyCore.seedDB.update(remoteSeed.hash, remoteSeed);
- return false;
- }
- }
-
- log.logInfo(STR_REMOTECRAWLTRIGGER + remoteSeed.getName()
- + " DENIED. RESPONSE=" + response + ", CAUSE="
- + page.get("reason") + ", URL=" + urlEntry.url().toString());
- remoteSeed.setFlagAcceptRemoteCrawl(false);
- yacyCore.seedDB.update(remoteSeed.hash, remoteSeed);
- return false;
- }
private static SimpleDateFormat DateFormatter = new SimpleDateFormat("EEE, dd MMM yyyy");
public static String dateString(Date date) {
@@ -3048,12 +2661,6 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser thread.setIdleSleep(1000);
}
- thread = getThread(CRAWLSTACK);
- if (thread != null) {
- setConfig(CRAWLSTACK_BUSYSLEEP , thread.setBusySleep(0));
- thread.setIdleSleep(5000);
- }
-
}
public static int accessFrequency(HashMap tracker, String host) {
@@ -3113,11 +2720,11 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser if (wordIndex.size() < 100) {
return "no DHT distribution: not enough words - wordIndex.size() = " + wordIndex.size();
}
- if ((getConfig(INDEX_DIST_ALLOW_WHILE_CRAWLING, "false").equalsIgnoreCase("false")) && (noticeURL.notEmpty())) {
- return "no DHT distribution: crawl in progress: noticeURL.stackSize() = " + noticeURL.size() + ", sbQueue.size() = " + sbQueue.size();
+ if ((getConfig(INDEX_DIST_ALLOW_WHILE_CRAWLING, "false").equalsIgnoreCase("false")) && (crawlQueues.noticeURL.notEmpty())) {
+ return "no DHT distribution: crawl in progress: noticeURL.stackSize() = " + crawlQueues.noticeURL.size() + ", sbQueue.size() = " + sbQueue.size();
}
if ((getConfig(INDEX_DIST_ALLOW_WHILE_INDEXING, "false").equalsIgnoreCase("false")) && (sbQueue.size() > 1)) {
- return "no DHT distribution: indexing in progress: noticeURL.stackSize() = " + noticeURL.size() + ", sbQueue.size() = " + sbQueue.size();
+ return "no DHT distribution: indexing in progress: noticeURL.stackSize() = " + crawlQueues.noticeURL.size() + ", sbQueue.size() = " + sbQueue.size();
}
return null;
}
@@ -3271,13 +2878,13 @@ public final class plasmaSwitchboard extends serverAbstractSwitch implements ser 0,
0,
0);
- plasmaCrawlZURL.Entry ee = this.errorURL.newEntry(
+ plasmaCrawlZURL.Entry ee = crawlQueues.errorURL.newEntry(
bentry, initiator, new Date(),
0, failreason);
// store the entry
ee.store();
// push it onto the stack
- this.errorURL.stackPushEntry(ee);
+ crawlQueues.errorURL.push(ee);
}
public void checkInterruption() throws InterruptedException {
diff --git a/source/de/anomic/plasma/plasmaSwitchboardQueue.java b/source/de/anomic/plasma/plasmaSwitchboardQueue.java index 802644143..3d6ee6ff4 100644 --- a/source/de/anomic/plasma/plasmaSwitchboardQueue.java +++ b/source/de/anomic/plasma/plasmaSwitchboardQueue.java @@ -103,6 +103,7 @@ public class plasmaSwitchboardQueue { }
public synchronized void push(Entry entry) throws IOException {
+ if (entry == null) return;
sbQueueStack.push(sbQueueStack.row().newEntry(new byte[][]{
entry.url.toString().getBytes(),
(entry.referrerHash == null) ? yacyURL.dummyHash.getBytes() : entry.referrerHash.getBytes(),
@@ -190,7 +191,7 @@ public class plasmaSwitchboardQueue { }
public class Entry {
- private yacyURL url; // plasmaURL.urlStringLength
+ private yacyURL url; // plasmaURL.urlStringLength
private String referrerHash; // plasmaURL.urlHashLength
private Date ifModifiedSince; // 6
private byte flags; // 1
@@ -359,14 +360,13 @@ public class plasmaSwitchboardQueue { return "Indexing_Not_Allowed";
}
- String nURL = url.toNormalform(true, true);
// -CGI access in request
// CGI access makes the page very individual, and therefore not usable in caches
if (!profile().crawlingQ()) {
- if (plasmaHTCache.isPOST(nURL)) {
+ if (url.isPOST()) {
return "Dynamic_(POST)";
}
- if (plasmaHTCache.isCGI(nURL)) {
+ if (url.isCGI()) {
return "Dynamic_(CGI)";
}
}
@@ -378,7 +378,7 @@ public class plasmaSwitchboardQueue { // we checked that in shallStoreCache
// a picture cannot be indexed
- if (plasmaHTCache.noIndexingURL(nURL)) {
+ if (plasmaHTCache.noIndexingURL(url)) {
return "Media_Content_(forbidden)";
}
@@ -414,12 +414,11 @@ public class plasmaSwitchboardQueue { return "Indexing_Not_Allowed";
}
- final String nURL = url().toNormalform(true, true);
// -CGI access in request
// CGI access makes the page very individual, and therefore not usable in caches
if (!profile().crawlingQ()) {
- if (plasmaHTCache.isPOST(nURL)) { return "Dynamic_(POST)"; }
- if (plasmaHTCache.isCGI(nURL)) { return "Dynamic_(CGI)"; }
+ if (url().isPOST()) { return "Dynamic_(POST)"; }
+ if (url().isCGI()) { return "Dynamic_(CGI)"; }
}
// -authorization cases in request
@@ -433,7 +432,7 @@ public class plasmaSwitchboardQueue { String status = this.getCachedObjectInfo().shallIndexCacheForCrawler();
if (status != null) return status;
}
- if (plasmaHTCache.noIndexingURL(nURL)) { return "Media_Content_(forbidden)"; }
+ if (plasmaHTCache.noIndexingURL(url())) { return "Media_Content_(forbidden)"; }
// -if-modified-since in request
// if the page is fresh at the very moment we can index it
diff --git a/source/de/anomic/server/serverDomains.java b/source/de/anomic/server/serverDomains.java index f00beceff..f040163c6 100644 --- a/source/de/anomic/server/serverDomains.java +++ b/source/de/anomic/server/serverDomains.java @@ -70,6 +70,18 @@ public class serverDomains { * @param host Hostname of the host in demand. * @return String with the ip. null, if the host could not be resolved. */ + public static InetAddress dnsResolveFromCache(String host) throws UnknownHostException { + if ((host == null) || (host.length() == 0)) return null; + host = host.toLowerCase().trim(); + + // trying to resolve host by doing a name cache lookup + InetAddress ip = (InetAddress) nameCacheHit.get(host); + if (ip != null) return ip; + + if (nameCacheMiss.contains(host)) return null; + throw new UnknownHostException("host not in cache"); + } + public static InetAddress dnsResolve(String host) { if ((host == null) || (host.length() == 0)) return null; host = host.toLowerCase().trim(); @@ -79,6 +91,7 @@ public class serverDomains { if (ip != null) return ip; if (nameCacheMiss.contains(host)) return null; + //System.out.println("***DEBUG dnsResolve(" + host + ")"); try { boolean doCaching = true; ip = InetAddress.getByName(host); diff --git a/source/de/anomic/server/serverSemaphore.java b/source/de/anomic/server/serverSemaphore.java index 40f195ae8..26c14c7b9 100644 --- a/source/de/anomic/server/serverSemaphore.java +++ b/source/de/anomic/server/serverSemaphore.java @@ -47,11 +47,7 @@ package de.anomic.server; public final class serverSemaphore {
private long currentValue = 0;
private long maximumValue = Long.MAX_VALUE;
-
- public serverSemaphore() {
- this(0,Long.MAX_VALUE);
- }
-
+
public serverSemaphore(long initialValue) {
this(initialValue,Long.MAX_VALUE);
}
diff --git a/source/de/anomic/urlRedirector/urlRedirectord.java b/source/de/anomic/urlRedirector/urlRedirectord.java index 9d3d199c1..ee66aee43 100644 --- a/source/de/anomic/urlRedirector/urlRedirectord.java +++ b/source/de/anomic/urlRedirector/urlRedirectord.java @@ -192,12 +192,12 @@ public class urlRedirectord implements serverHandler { // first delete old entry, if exists
String urlhash = reqURL.hash();
switchboard.wordIndex.loadedURL.remove(urlhash);
- switchboard.noticeURL.removeByURLHash(urlhash);
- switchboard.errorURL.remove(urlhash);
+ switchboard.crawlQueues.noticeURL.removeByURLHash(urlhash);
+ switchboard.crawlQueues.errorURL.remove(urlhash);
// enqueuing URL for crawling
- reasonString = switchboard.sbStackCrawlThread.stackCrawl(
- this.nextURL,
+ reasonString = switchboard.crawlStacker.stackCrawl(
+ reqURL,
null,
yacyCore.seedDB.mySeed().hash,
"URL Redirector",
diff --git a/source/de/anomic/yacy/yacyPeerActions.java b/source/de/anomic/yacy/yacyPeerActions.java index 224f9d313..a40c03f96 100644 --- a/source/de/anomic/yacy/yacyPeerActions.java +++ b/source/de/anomic/yacy/yacyPeerActions.java @@ -115,8 +115,8 @@ public class yacyPeerActions { seedDB.mySeed().put(yacySeed.UPTIME, Long.toString(uptime/60)); // the number of minutes that the peer is up in minutes/day (moving average MA30)
seedDB.mySeed().put(yacySeed.LCOUNT, Integer.toString(sb.wordIndex.loadedURL.size())); // the number of links that the peer has stored (LURL's)
- seedDB.mySeed().put(yacySeed.NCOUNT, Integer.toString(sb.noticeURL.size())); // the number of links that the peer has noticed, but not loaded (NURL's)
- seedDB.mySeed().put(yacySeed.RCOUNT, Integer.toString(sb.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT))); // the number of links that the peer provides for remote crawling (ZURL's)
+ seedDB.mySeed().put(yacySeed.NCOUNT, Integer.toString(sb.crawlQueues.noticeURL.size())); // the number of links that the peer has noticed, but not loaded (NURL's)
+ seedDB.mySeed().put(yacySeed.RCOUNT, Integer.toString(sb.crawlQueues.noticeURL.stackSize(plasmaCrawlNURL.STACK_TYPE_LIMIT))); // the number of links that the peer provides for remote crawling (ZURL's)
seedDB.mySeed().put(yacySeed.ICOUNT, Integer.toString(sb.wordIndex.size())); // the minimum number of words that the peer has indexed (as it says)
seedDB.mySeed().put(yacySeed.SCOUNT, Integer.toString(seedDB.sizeConnected())); // the number of seeds that the peer has stored
seedDB.mySeed().put(yacySeed.CCOUNT, Double.toString(((int) ((seedDB.sizeConnected() + seedDB.sizeDisconnected() + seedDB.sizePotential()) * 60.0 / (uptime + 1.01)) * 100) / 100.0)); // the number of clients that the peer connects (as connects/hour)
diff --git a/source/de/anomic/yacy/yacyURL.java b/source/de/anomic/yacy/yacyURL.java index ec1e2a302..e8581b008 100644 --- a/source/de/anomic/yacy/yacyURL.java +++ b/source/de/anomic/yacy/yacyURL.java @@ -28,7 +28,6 @@ package de.anomic.yacy; // and to prevent that java.net.URL usage causes DNS queries which are used in java.net. import java.io.File; -import java.net.InetAddress; import java.net.MalformedURLException; import java.util.HashMap; import java.util.Iterator; @@ -900,6 +899,23 @@ public class yacyURL { return this.toString().compareTo(((yacyURL) h).toString()); } + + public boolean isPOST() { + return (path.indexOf("?") >= 0 || + path.indexOf("&") >= 0); + } + + public boolean isCGI() { + String ls = path.toLowerCase(); + return ((ls.indexOf(".cgi") >= 0) || + (ls.indexOf(".exe") >= 0) || + (ls.indexOf(";jsessionid=") >= 0) || + (ls.indexOf("sessionid/") >= 0) || + (ls.indexOf("phpsessid=") >= 0) || + (ls.indexOf("search.php?sid=") >= 0) || + (ls.indexOf("memberlist.php?sid=") >= 0)); + } + // static methods from plasmaURL public static final int flagTypeID(String hash) { @@ -1053,9 +1069,7 @@ public class yacyURL { // checks for local/global IP range and local IP public boolean isLocal() { - InetAddress hostAddress = serverDomains.dnsResolve(this.host); // TODO: use a check with the hash first - if (hostAddress == null) /* we are offline */ return false; // it is rare to be offline in intranets - return hostAddress.isSiteLocalAddress() || hostAddress.isLoopbackAddress(); + return serverDomains.isLocal(this.host); } // language calculation diff --git a/source/yacy.java b/source/yacy.java index 40c5d611f..2d5d6ffbc 100644 --- a/source/yacy.java +++ b/source/yacy.java @@ -806,7 +806,7 @@ public final class yacy { }
}
if (source.equals("eurl")) {
- Iterator eiter = sb.errorURL.entries(true, null);
+ Iterator eiter = sb.crawlQueues.errorURL.entries(true, null);
plasmaCrawlZURL.Entry entry;
while (eiter.hasNext()) {
try {
@@ -825,7 +825,7 @@ public final class yacy { }
}
if (source.equals("nurl")) {
- Iterator eiter = sb.noticeURL.iterator(plasmaCrawlNURL.STACK_TYPE_CORE);
+ Iterator eiter = sb.crawlQueues.noticeURL.iterator(plasmaCrawlNURL.STACK_TYPE_CORE);
plasmaCrawlEntry entry;
while (eiter.hasNext()) {
try {
@@ -915,7 +915,7 @@ public final class yacy { }
}
if (source.equals("eurl")) {
- Iterator eiter = sb.errorURL.entries(true, null);
+ Iterator eiter = sb.crawlQueues.errorURL.entries(true, null);
plasmaCrawlZURL.Entry entry;
while (eiter.hasNext()) {
entry = (plasmaCrawlZURL.Entry) eiter.next();
@@ -931,7 +931,7 @@ public final class yacy { }
}
if (source.equals("nurl")) {
- Iterator eiter = sb.noticeURL.iterator(plasmaCrawlNURL.STACK_TYPE_CORE);
+ Iterator eiter = sb.crawlQueues.noticeURL.iterator(plasmaCrawlNURL.STACK_TYPE_CORE);
plasmaCrawlEntry entry;
while (eiter.hasNext()) {
entry = (plasmaCrawlEntry) eiter.next();
@@ -570,8 +570,7 @@ filterOutStopwordsFromTopwords=true 80_indexing_busysleep__pro=10
80_indexing_memprereq=6291456
82_crawlstack_idlesleep=5000
-82_crawlstack_busysleep=50
-82_crawlstack_busysleep__pro=10
+82_crawlstack_busysleep=0
82_crawlstack_memprereq=1048576
90_cleanup_idlesleep=300000
90_cleanup_busysleep=300000
@@ -730,12 +729,6 @@ crawler.ftp.maxFileSize__pro=1048576 # maximum number of crawler threads
crawler.MaxActiveThreads = 30
-crawler.MaxIdleThreads = 5
-
-# maximum number of crawl-stacker threads
-stacker.MaxActiveThreads = 50
-stacker.MaxIdleThreads = 10
-stacker.MinIdleThreads = 5
# maximum size of indexing queue
indexer.slots = 40
|