diff options
Diffstat (limited to 'htroot/Crawler_p.java')
| -rw-r--r-- | htroot/Crawler_p.java | 166 |
1 files changed, 111 insertions, 55 deletions
diff --git a/htroot/Crawler_p.java b/htroot/Crawler_p.java index 3a2c4f377..707d67bcd 100644 --- a/htroot/Crawler_p.java +++ b/htroot/Crawler_p.java @@ -35,12 +35,17 @@ import java.util.Set; import java.util.regex.Pattern; import java.util.regex.PatternSyntaxException; +import org.apache.solr.common.SolrException; +import org.apache.solr.core.SolrCore; +import org.apache.solr.search.SyntaxError; + import net.yacy.cora.date.AbstractFormatter; import net.yacy.cora.document.encoding.ASCII; import net.yacy.cora.document.id.AnchorURL; import net.yacy.cora.document.id.DigestURL; import net.yacy.cora.document.id.MultiProtocolURL; import net.yacy.cora.federate.solr.FailCategory; +import net.yacy.cora.federate.solr.instance.EmbeddedInstance; import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.RequestHeader; @@ -70,6 +75,7 @@ import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; import net.yacy.search.index.Fulltext; import net.yacy.search.index.Segment; +import net.yacy.search.index.SingleDocumentMatcher; import net.yacy.search.query.SearchEventCache; import net.yacy.search.schema.CollectionSchema; import net.yacy.server.serverObjects; @@ -464,7 +470,12 @@ public class Crawler_p { boolean hasCrawlstartDataOK = !crawlName.isEmpty(); if (hasCrawlstartDataOK) { // check crawlurl was given in sitecrawl - if ("url".equals(crawlingMode) && rootURLs.size() == 0) hasCrawlstartDataOK = false; + if ("url".equals(crawlingMode) && rootURLs.size() == 0) { + prop.put("info", "5"); //Crawling failed + prop.putHTML("info_crawlingURL", "(no url given)"); + prop.putHTML("info_reasonString", "you must submit at least one crawl url"); + hasCrawlstartDataOK = false; + } } String snapshotsMaxDepthString = post.get("snapshotsMaxDepth", "-1"); @@ -533,6 +544,52 @@ public class Crawler_p { sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); } + /* If a solr query filter is defined, verify now its syntax and that the embedded Solr schema is available */ + final String solrQueryMustMatch = post.get(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, CrawlProfile.SOLR_MATCH_ALL_QUERY).trim(); + final String solrQueryMustNotMatch = post.get(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTNOTMATCH.key, CrawlProfile.SOLR_EMPTY_QUERY).trim(); + if(!(solrQueryMustMatch.isEmpty() || CrawlProfile.SOLR_MATCH_ALL_QUERY.equals(solrQueryMustMatch)) || !CrawlProfile.SOLR_EMPTY_QUERY.equals(solrQueryMustNotMatch)) { + + final EmbeddedInstance embeddedSolr = sb.index.fulltext().getEmbeddedInstance(); + final SolrCore embeddedCore = embeddedSolr != null ? embeddedSolr.getDefaultCore() : null; + final boolean embeddedSolrConnected = embeddedSolr != null && embeddedCore != null; + prop.put("noEmbeddedSolr", !embeddedSolrConnected); + if (embeddedSolrConnected) { + if(!(solrQueryMustMatch.isEmpty() || CrawlProfile.SOLR_MATCH_ALL_QUERY.equals(solrQueryMustMatch))) { + try { + SingleDocumentMatcher.toLuceneQuery(solrQueryMustMatch, embeddedCore); + } catch(final SyntaxError | SolrException e) { + hasCrawlstartDataOK = false; + prop.put("info", "10"); + prop.put("info_solrQuery", solrQueryMustMatch); + } catch(final RuntimeException e) { + hasCrawlstartDataOK = false; + prop.put("info", "11"); + prop.put("info_solrQuery", solrQueryMustMatch); + } + } + + if(!CrawlProfile.SOLR_EMPTY_QUERY.equals(solrQueryMustNotMatch)) { + try { + SingleDocumentMatcher.toLuceneQuery(solrQueryMustNotMatch, embeddedCore); + } catch(final SyntaxError | SolrException e) { + hasCrawlstartDataOK = false; + prop.put("info", "10"); + prop.put("info_solrQuery", solrQueryMustNotMatch); + } catch(final RuntimeException e) { + hasCrawlstartDataOK = false; + prop.put("info", "11"); + prop.put("info_solrQuery", solrQueryMustNotMatch); + } + } + } else { + hasCrawlstartDataOK = false; + prop.put("info", "9"); + } + + + + } + // prepare a new crawling profile final CrawlProfile profile; byte[] handle; @@ -574,6 +631,9 @@ public class Crawler_p { post.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key, CrawlProfile.MATCH_ALL_STRING)); profile.put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, post .get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, CrawlProfile.MATCH_NEVER_STRING)); + profile.put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, solrQueryMustMatch); + profile.put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTNOTMATCH.key, solrQueryMustNotMatch); + handle = ASCII.getBytes(profile.handle()); @@ -587,15 +647,11 @@ public class Crawler_p { profile = null; handle = null; } + // start the crawl - if ("url".equals(crawlingMode)) { - if (rootURLs.size() == 0) { - prop.put("info", "5"); //Crawling failed - prop.putHTML("info_crawlingURL", "(no url given)"); - prop.putHTML("info_reasonString", "you must submit at least one crawl url"); - } else { - + if(hasCrawlstartDataOK) { + if ("url".equals(crawlingMode)) { // stack requests sb.crawler.putActive(handle, profile); final Set<DigestURL> successurls = new HashSet<DigestURL>(); @@ -639,53 +695,53 @@ public class Crawler_p { prop.putHTML("info_reasonString", fr.toString()); } if (successurls.size() > 0) sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); - } - } else if ("sitemap".equals(crawlingMode)) { - try { - final DigestURL sitemapURL = sitemapURLStr.indexOf("//") > 0 ? new DigestURL(sitemapURLStr) : new DigestURL(rootURLs.iterator().next(), sitemapURLStr); // fix for relative paths which should not exist but are used anyway - sb.crawler.putActive(handle, profile); - final SitemapImporter importer = new SitemapImporter(sb, sitemapURL, profile); - importer.start(); - sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); - } catch (final Exception e) { - // mist - prop.put("info", "6");//Error with url - prop.putHTML("info_crawlingStart", sitemapURLStr); - prop.putHTML("info_error", e.getMessage()); - ConcurrentLog.logException(e); - } - } else if ("file".equals(crawlingMode)) { - if (post.containsKey("crawlingFile") && crawlingFile != null) { - try { - if(newcrawlingdepth > 0 && (fullDomain || subPath)) { - /* All links must have already been loaded because they are the part of the newcrawlingMustMatch filter */ - if(hyperlinks_from_file != null) { - sb.crawler.putActive(handle, profile); - sb.crawlStacker.enqueueEntriesAsynchronous(sb.peers.mySeed().hash.getBytes(), profile.handle(), hyperlinks_from_file, profile.timezoneOffset()); - } - } else { - /* No restriction on domains or subpath : we scrape now links and asynchronously push them to the crawlStacker */ - final String crawlingFileContent = post.get("crawlingFile$file", ""); - final ContentScraper scraper = new ContentScraper(new DigestURL(crawlingFile), 10000000, - new HashSet<String>(), new VocabularyScraper(), profile.timezoneOffset()); - FileCrawlStarterTask crawlStarterTask = new FileCrawlStarterTask(crawlingFile, crawlingFileContent, scraper, profile, - sb.crawlStacker, sb.peers.mySeed().hash.getBytes()); - sb.crawler.putActive(handle, profile); - crawlStarterTask.start(); - } - } catch (final PatternSyntaxException e) { - prop.put("info", "4"); // crawlfilter does not match url - prop.putHTML("info_newcrawlingfilter", newcrawlingMustMatch); - prop.putHTML("info_error", e.getMessage()); - } catch (final Exception e) { - // mist - prop.put("info", "7"); // Error with file - prop.putHTML("info_crawlingStart", crawlingFileName); - prop.putHTML("info_error", e.getMessage()); - ConcurrentLog.logException(e); - } - sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); - } + } else if ("sitemap".equals(crawlingMode)) { + try { + final DigestURL sitemapURL = sitemapURLStr.indexOf("//") > 0 ? new DigestURL(sitemapURLStr) : new DigestURL(rootURLs.iterator().next(), sitemapURLStr); // fix for relative paths which should not exist but are used anyway + sb.crawler.putActive(handle, profile); + final SitemapImporter importer = new SitemapImporter(sb, sitemapURL, profile); + importer.start(); + sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); + } catch (final Exception e) { + // mist + prop.put("info", "6");//Error with url + prop.putHTML("info_crawlingStart", sitemapURLStr); + prop.putHTML("info_error", e.getMessage()); + ConcurrentLog.logException(e); + } + } else if ("file".equals(crawlingMode)) { + if (post.containsKey("crawlingFile") && crawlingFile != null) { + try { + if(newcrawlingdepth > 0 && (fullDomain || subPath)) { + /* All links must have already been loaded because they are the part of the newcrawlingMustMatch filter */ + if(hyperlinks_from_file != null) { + sb.crawler.putActive(handle, profile); + sb.crawlStacker.enqueueEntriesAsynchronous(sb.peers.mySeed().hash.getBytes(), profile.handle(), hyperlinks_from_file, profile.timezoneOffset()); + } + } else { + /* No restriction on domains or subpath : we scrape now links and asynchronously push them to the crawlStacker */ + final String crawlingFileContent = post.get("crawlingFile$file", ""); + final ContentScraper scraper = new ContentScraper(new DigestURL(crawlingFile), 10000000, + new HashSet<String>(), new VocabularyScraper(), profile.timezoneOffset()); + FileCrawlStarterTask crawlStarterTask = new FileCrawlStarterTask(crawlingFile, crawlingFileContent, scraper, profile, + sb.crawlStacker, sb.peers.mySeed().hash.getBytes()); + sb.crawler.putActive(handle, profile); + crawlStarterTask.start(); + } + } catch (final PatternSyntaxException e) { + prop.put("info", "4"); // crawlfilter does not match url + prop.putHTML("info_newcrawlingfilter", newcrawlingMustMatch); + prop.putHTML("info_error", e.getMessage()); + } catch (final Exception e) { + // mist + prop.put("info", "7"); // Error with file + prop.putHTML("info_crawlingStart", crawlingFileName); + prop.putHTML("info_error", e.getMessage()); + ConcurrentLog.logException(e); + } + sb.continueCrawlJob(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL); + } + } } } } |