summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--htroot/IndexImportWarc_p.html4
-rw-r--r--htroot/IndexImportZim_p.html2
-rw-r--r--source/net/yacy/crawler/data/CrawlProfile.java259
-rw-r--r--source/net/yacy/document/importer/WarcImporter.java32
-rw-r--r--source/net/yacy/document/importer/ZimImporter.java168
-rw-r--r--source/net/yacy/htroot/IndexImportWarc_p.java10
-rw-r--r--source/net/yacy/htroot/IndexImportZim_p.java19
-rw-r--r--source/net/yacy/http/servlets/YaCyDefaultServlet.java72
-rw-r--r--source/net/yacy/search/Switchboard.java42
9 files changed, 340 insertions, 268 deletions
diff --git a/htroot/IndexImportWarc_p.html b/htroot/IndexImportWarc_p.html
index 2a79d1fc4..e70b19ba6 100644
--- a/htroot/IndexImportWarc_p.html
+++ b/htroot/IndexImportWarc_p.html
@@ -14,7 +14,7 @@
#(import)#
<p>No import thread is running, you can start a new thread here</p>
- <form action="IndexImportWarc_p.html" method="get" accept-charset="UTF-8">
+ <form action="IndexImportWarc_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<!-- no post method here, we don't want to transmit the whole file, only the path-->
<fieldset>
<legend>Warc File Selection: select an warc file (which may be gz compressed)</legend>
@@ -30,6 +30,8 @@
<dt class="TableCellDark"><label for="url">Url:</label></dt>
<dd><input name="url" id="url" value="" size="75"/></dd>
<dt></dt>
+ <dt class="TableCellDark"><label for="collection">Collection:</label></dt>
+ <dd><input name="collection" id="collection" type="text" value="user" size="75" /></dd>
<dd><input name="submit" class="btn btn-primary" type="submit" value="Import Warc File" /></dd>
</dl>
</fieldset>
diff --git a/htroot/IndexImportZim_p.html b/htroot/IndexImportZim_p.html
index 68d74762e..b4914cea8 100644
--- a/htroot/IndexImportZim_p.html
+++ b/htroot/IndexImportZim_p.html
@@ -25,6 +25,8 @@
<dl>
<dt class="TableCellDark"><label for="file">File:</label></dt>
<dd><input name="file" id="file" type="file" value="" size="75" /></dd>
+ <dt class="TableCellDark"><label for="collection">Collection:</label></dt>
+ <dd><input name="collection" id="collection" type="text" value="user" size="75" /></dd>
<dt></dt>
<dd><input name="submit" class="btn btn-primary" type="submit" value="Import ZIM File" /></dd>
</dl>
diff --git a/source/net/yacy/crawler/data/CrawlProfile.java b/source/net/yacy/crawler/data/CrawlProfile.java
index 64af0c489..574a164fb 100644
--- a/source/net/yacy/crawler/data/CrawlProfile.java
+++ b/source/net/yacy/crawler/data/CrawlProfile.java
@@ -64,7 +64,7 @@ import net.yacy.server.serverObjects;
/**
* this is a simple record structure that hold all properties of a single crawl start
*/
-public class CrawlProfile extends ConcurrentHashMap<String, String> implements Map<String, String> {
+public class CrawlProfile extends ConcurrentHashMap<String, String> implements Map<String, String>, Cloneable {
private static final long serialVersionUID = 5527325718810703504L;
@@ -151,7 +151,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public String toString() {
return this.key;
}
- }
+ }
private Pattern crawlerurlmustmatch = null, crawlerurlmustnotmatch = null;
@@ -166,7 +166,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
private Pattern indexurlmustmatch = null, indexurlmustnotmatch = null;
private Pattern indexcontentmustmatch = null, indexcontentmustnotmatch = null;
- /** Pattern on the media type documents must match before being indexed
+ /** Pattern on the media type documents must match before being indexed
* @see CollectionSchema#content_type */
private Pattern indexMediaTypeMustMatch = null;
@@ -251,58 +251,58 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
throw new NullPointerException("name must not be null or empty");
}
if (name.length() > 256) name = name.substring(256);
- this.doms = new ConcurrentHashMap<String, AtomicInteger>();
+ this.doms = new ConcurrentHashMap<>();
final String handle = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(name + crawlerUrlMustMatch + depth + crawlerUrlMustNotMatch + domMaxPages + collections)).substring(0, Word.commonHashLength);
- put(CrawlAttribute.HANDLE.key, handle);
- put(CrawlAttribute.NAME.key, name);
- put(CrawlAttribute.AGENT_NAME.key, userAgentName);
- put(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key, true);
- put(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key, (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
- put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key, (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
- put(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key, (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
- put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key, (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
- put(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key, (crawlerIpMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerIpMustMatch);
- put(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key, (crawlerIpMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerIpMustNotMatch);
- put(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key, (crawlerCountryMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerCountryMustMatch);
- put(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key, (crawlerNoDepthLimitMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerNoDepthLimitMatch);
- put(CrawlAttribute.INDEXING_URL_MUSTMATCH.key, (indexUrlMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustMatch);
- put(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key, (indexUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustNotMatch);
- put(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key, (indexContentMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustMatch);
- put(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key, (indexContentMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustNotMatch);
- put(CrawlAttribute.DEPTH.key, depth);
- put(CrawlAttribute.DIRECT_DOC_BY_URL.key, directDocByURL);
- put(CrawlAttribute.RECRAWL_IF_OLDER.key, recrawlIfOlder == null ? Long.MAX_VALUE : recrawlIfOlder.getTime());
- put(CrawlAttribute.DOM_MAX_PAGES.key, domMaxPages);
- put(CrawlAttribute.CRAWLING_Q.key, crawlingQ); // crawling of urls with '?'
- put(CrawlAttribute.FOLLOW_FRAMES.key, followFrames); // load pages contained in frames or ifames
- put(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key, obeyHtmlRobotsNoindex); // if false, then a meta robots tag containing 'noindex' is ignored
- put(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key, obeyHtmlRobotsNofollow);
- put(CrawlAttribute.INDEX_TEXT.key, indexText);
- put(CrawlAttribute.INDEX_MEDIA.key, indexMedia);
- put(CrawlAttribute.STORE_HTCACHE.key, storeHTCache);
- put(CrawlAttribute.REMOTE_INDEXING.key, remoteIndexing);
- put(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key, snapshotsMaxDepth);
- put(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key, snapshotsLoadImage);
- put(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key, snapshotsReplaceOld);
- put(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key, snapshotsMustnotmatch);
- put(CrawlAttribute.CACHE_STRAGEGY.key, cacheStrategy.toString());
- put(CrawlAttribute.COLLECTIONS.key, CommonPattern.SPACE.matcher(collections.trim()).replaceAll(""));
+ this.put(CrawlAttribute.HANDLE.key, handle);
+ this.put(CrawlAttribute.NAME.key, name);
+ this.put(CrawlAttribute.AGENT_NAME.key, userAgentName);
+ this.put(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key, true);
+ this.put(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key, (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
+ this.put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key, (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
+ this.put(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key, (crawlerUrlMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerUrlMustMatch);
+ this.put(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key, (crawlerUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerUrlMustNotMatch);
+ this.put(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key, (crawlerIpMustMatch == null) ? CrawlProfile.MATCH_ALL_STRING : crawlerIpMustMatch);
+ this.put(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key, (crawlerIpMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerIpMustNotMatch);
+ this.put(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key, (crawlerCountryMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerCountryMustMatch);
+ this.put(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key, (crawlerNoDepthLimitMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : crawlerNoDepthLimitMatch);
+ this.put(CrawlAttribute.INDEXING_URL_MUSTMATCH.key, (indexUrlMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustMatch);
+ this.put(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key, (indexUrlMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexUrlMustNotMatch);
+ this.put(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key, (indexContentMustMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustMatch);
+ this.put(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key, (indexContentMustNotMatch == null) ? CrawlProfile.MATCH_NEVER_STRING : indexContentMustNotMatch);
+ this.put(CrawlAttribute.DEPTH.key, depth);
+ this.put(CrawlAttribute.DIRECT_DOC_BY_URL.key, directDocByURL);
+ this.put(CrawlAttribute.RECRAWL_IF_OLDER.key, recrawlIfOlder == null ? Long.MAX_VALUE : recrawlIfOlder.getTime());
+ this.put(CrawlAttribute.DOM_MAX_PAGES.key, domMaxPages);
+ this.put(CrawlAttribute.CRAWLING_Q.key, crawlingQ); // crawling of urls with '?'
+ this.put(CrawlAttribute.FOLLOW_FRAMES.key, followFrames); // load pages contained in frames or ifames
+ this.put(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key, obeyHtmlRobotsNoindex); // if false, then a meta robots tag containing 'noindex' is ignored
+ this.put(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key, obeyHtmlRobotsNofollow);
+ this.put(CrawlAttribute.INDEX_TEXT.key, indexText);
+ this.put(CrawlAttribute.INDEX_MEDIA.key, indexMedia);
+ this.put(CrawlAttribute.STORE_HTCACHE.key, storeHTCache);
+ this.put(CrawlAttribute.REMOTE_INDEXING.key, remoteIndexing);
+ this.put(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key, snapshotsMaxDepth);
+ this.put(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key, snapshotsLoadImage);
+ this.put(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key, snapshotsReplaceOld);
+ this.put(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key, snapshotsMustnotmatch);
+ this.put(CrawlAttribute.CACHE_STRAGEGY.key, cacheStrategy.toString());
+ this.put(CrawlAttribute.COLLECTIONS.key, CommonPattern.SPACE.matcher(collections.trim()).replaceAll(""));
// we transform the ignore_class_name and scraper information into a JSON Array
this.defaultValency = defaultValency;
- this.valencySwitchTagNames = valencySwitchTagNames == null ? new HashSet<String>() : valencySwitchTagNames;
+ this.valencySwitchTagNames = valencySwitchTagNames == null ? new HashSet<>() : valencySwitchTagNames;
String jsonString = new JSONArray(valencySwitchTagNames).toString();
- put(CrawlAttribute.DEFAULT_VALENCY.key, defaultValency.name());
- put(CrawlAttribute.VALENCY_SWITCH_TAG_NAMES.key, jsonString);
+ this.put(CrawlAttribute.DEFAULT_VALENCY.key, defaultValency.name());
+ this.put(CrawlAttribute.VALENCY_SWITCH_TAG_NAMES.key, jsonString);
this.scraper = scraper == null ? new VocabularyScraper() : scraper;
jsonString = this.scraper.toString();
assert jsonString != null && jsonString.length() > 0 && jsonString.charAt(0) == '{' : "jsonString = " + jsonString;
- put(CrawlAttribute.SCRAPER.key, jsonString);
- put(CrawlAttribute.TIMEZONEOFFSET.key, timezoneOffset);
- put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key, CrawlProfile.MATCH_ALL_STRING);
- put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, CrawlProfile.MATCH_NEVER_STRING);
- put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, CrawlProfile.SOLR_MATCH_ALL_QUERY);
- put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTNOTMATCH.key, CrawlProfile.SOLR_EMPTY_QUERY);
- put(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key, noindexWhenCanonicalUnequalURL);
+ this.put(CrawlAttribute.SCRAPER.key, jsonString);
+ this.put(CrawlAttribute.TIMEZONEOFFSET.key, timezoneOffset);
+ this.put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key, CrawlProfile.MATCH_ALL_STRING);
+ this.put(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, CrawlProfile.MATCH_NEVER_STRING);
+ this.put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, CrawlProfile.SOLR_MATCH_ALL_QUERY);
+ this.put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTNOTMATCH.key, CrawlProfile.SOLR_EMPTY_QUERY);
+ this.put(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key, noindexWhenCanonicalUnequalURL);
}
/**
@@ -311,9 +311,9 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public CrawlProfile(final Map<String, String> ext) {
super(ext == null ? 1 : ext.size());
- if (ext != null) putAll(ext);
- this.doms = new ConcurrentHashMap<String, AtomicInteger>();
- String defaultValency = ext.get(CrawlAttribute.DEFAULT_VALENCY.key);
+ if (ext != null) this.putAll(ext);
+ this.doms = new ConcurrentHashMap<>();
+ final String defaultValency = ext.get(CrawlAttribute.DEFAULT_VALENCY.key);
this.defaultValency = defaultValency == null || defaultValency.length() == 0 ? TagValency.EVAL : TagValency.valueOf(defaultValency);
String jsonString = ext.get(CrawlAttribute.VALENCY_SWITCH_TAG_NAMES.key);
JSONArray a;
@@ -327,10 +327,10 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
a = new JSONArray();
}
}
- this.valencySwitchTagNames = new HashSet<String>();
+ this.valencySwitchTagNames = new HashSet<>();
for (int i = 0; i < a.length(); i++) try {
this.valencySwitchTagNames.add(a.getString(i));
- } catch (JSONException e) {}
+ } catch (final JSONException e) {}
jsonString = ext.get(CrawlAttribute.SCRAPER.key);
if (jsonString == null || jsonString.length() == 0) {
this.scraper = new VocabularyScraper();
@@ -340,12 +340,19 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
loadedScraper = new VocabularyScraper(jsonString);
} catch(final JSONException e) {
ConcurrentLog.logException(e);
- loadedScraper = new VocabularyScraper();
+ loadedScraper = new VocabularyScraper();
}
this.scraper = loadedScraper;
}
}
+ @Override
+ public Object clone() {
+ final ConcurrentHashMap<String, String> m = new ConcurrentHashMap<>(this);
+ final CrawlProfile cp = new CrawlProfile(m);
+ return cp;
+ }
+
public TagValency defaultValency() {
return this.defaultValency;
}
@@ -389,10 +396,10 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public ClientIdentification.Agent getAgent() {
- String agentName = this.get(CrawlAttribute.AGENT_NAME.key);
+ final String agentName = this.get(CrawlAttribute.AGENT_NAME.key);
return ClientIdentification.getAgent(agentName);
}
-
+
public AtomicInteger getCount(final String domain) {
if (domain == null) return new AtomicInteger(0); // in case of file indexing this is required
AtomicInteger dp = this.doms.get(domain);
@@ -436,12 +443,24 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* @return handle of the profile
*/
public String handle() {
- final String r = get(CrawlAttribute.HANDLE.key);
+ final String r = this.get(CrawlAttribute.HANDLE.key);
assert r != null;
//if (r == null) return null;
return r;
}
+ public void setHandle() {
+ final String handle = Base64Order.enhancedCoder.encode(Digest.encodeMD5Raw(
+ this.get(CrawlAttribute.NAME.key) +
+ this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key) +
+ this.get(CrawlAttribute.DEPTH.key) +
+ this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTNOTMATCH.key) +
+ this.domMaxPages() +
+ this.get(CrawlAttribute.COLLECTIONS.key))).substring(0, Word.commonHashLength);
+ assert handle != null && handle.length() == Word.commonHashLength : handle + " != " + Word.commonHashLength;
+ this.put(CrawlAttribute.HANDLE.key, handle);
+ }
+
private Map<String, Pattern> cmap = null;
/**
@@ -449,29 +468,33 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* @return a list of collection names
*/
public Map<String, Pattern> collections() {
- if (cmap != null) return cmap;
- final String r = get(CrawlAttribute.COLLECTIONS.key);
+ if (this.cmap != null) return this.cmap;
+ final String r = this.get(CrawlAttribute.COLLECTIONS.key);
this.cmap = collectionParser(r);
return this.cmap;
}
public static Map<String, Pattern> collectionParser(String collectionString) {
- if (collectionString == null || collectionString.length() == 0) return new HashMap<String, Pattern>();
- String[] cs = CommonPattern.COMMA.split(collectionString);
- final Map<String, Pattern> cm = new LinkedHashMap<String, Pattern>();
- for (String c: cs) {
- int p = c.indexOf(':');
+ if (collectionString == null || collectionString.length() == 0) return new HashMap<>();
+ final String[] cs = CommonPattern.COMMA.split(collectionString);
+ final Map<String, Pattern> cm = new LinkedHashMap<>();
+ for (final String c: cs) {
+ final int p = c.indexOf(':');
if (p < 0) cm.put(c, QueryParams.catchall_pattern); else cm.put(c.substring(0, p), Pattern.compile(c.substring(p + 1)));
}
return cm;
}
+ public void setCollections(String collectionsList) {
+ this.put(CrawlAttribute.COLLECTIONS.key, CommonPattern.SPACE.matcher(collectionsList.trim()).replaceAll(""));
+ }
+
/**
* Gets the name of the CrawlProfile.
* @return name of the profile
*/
public String name() {
- final String r = get(CrawlAttribute.NAME.key);
+ final String r = this.get(CrawlAttribute.NAME.key);
if (r == null) return "";
return r;
}
@@ -481,8 +504,8 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* @return the name of the collection if that is not "user" or the name() otherwise;
*/
public String collectionName() {
- final String r = get(CrawlAttribute.COLLECTIONS.key);
- return r == null || r.length() == 0 || "user".equals(r) ? name() : r;
+ final String r = this.get(CrawlAttribute.COLLECTIONS.key);
+ return r == null || r.length() == 0 || "user".equals(r) ? this.name() : r;
}
/**
@@ -491,7 +514,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern urlMustMatchPattern() {
if (this.crawlerurlmustmatch == null) {
- final String r = get(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key);
+ final String r = this.get(CrawlAttribute.CRAWLER_URL_MUSTMATCH.key);
try {
this.crawlerurlmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.crawlerurlmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -505,11 +528,11 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* and to prevent exceeding the field size limit for
* CollectionSchema.failreason_s (32k) when the pattern is present in a fail doc
* added to the Solr index.
- *
+ *
* @return the urlMustMatchPattern formatted as a String of limited size
*/
public String formattedUrlMustMatchPattern() {
- String patternStr = urlMustMatchPattern().toString();
+ String patternStr = this.urlMustMatchPattern().toString();
if(patternStr.length() > 1000) {
/* The pattern may be quite large when using the 'From Link-List of URL' crawl start point. */
patternStr = patternStr.substring(0, Math.min(patternStr.length(), 1000)) + "...";
@@ -523,7 +546,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern urlMustNotMatchPattern() {
if (this.crawlerurlmustnotmatch == null) {
- final String r = get(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key);
+ final String r = this.get(CrawlAttribute.CRAWLER_URL_MUSTNOTMATCH.key);
try {
this.crawlerurlmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.crawlerurlmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -533,7 +556,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
/**
* Get the pattern on the URL a document must match to allow adding its embedded links to the crawl stack
- *
+ *
* @return a {@link Pattern} instance, defaulting to
* {@link CrawlProfile#MATCH_ALL_PATTERN} when the regular expression
* string is not set or its syntax is incorrect
@@ -541,7 +564,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public Pattern getCrawlerOriginUrlMustMatchPattern() {
if (this.crawlerOriginUrlMustMatch == null) {
/* Cache the compiled pattern for faster next calls */
- final String patternStr = get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key);
+ final String patternStr = this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTMATCH.key);
try {
this.crawlerOriginUrlMustMatch = (patternStr == null
|| patternStr.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN
@@ -555,7 +578,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
/**
* Get the pattern on the URL a document must not match to allow adding its embedded links to the crawl stack
- *
+ *
* @return a {@link Pattern} instance, defaulting to
* {@link CrawlProfile#MATCH_NEVER_PATTERN} when the regular expression
* string is not set or its syntax is incorrect
@@ -563,7 +586,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public Pattern getCrawlerOriginUrlMustNotMatchPattern() {
if (this.crawlerOriginUrlMustNotMatch == null) {
/* Cache the compiled pattern for faster next calls */
- final String patternStr = get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTNOTMATCH.key);
+ final String patternStr = this.get(CrawlAttribute.CRAWLER_ORIGIN_URL_MUSTNOTMATCH.key);
try {
this.crawlerOriginUrlMustNotMatch = (patternStr == null
|| patternStr.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN
@@ -581,7 +604,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern ipMustMatchPattern() {
if (this.crawleripmustmatch == null) {
- final String r = get(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key);
+ final String r = this.get(CrawlAttribute.CRAWLER_IP_MUSTMATCH.key);
try {
this.crawleripmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.crawleripmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -595,7 +618,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern ipMustNotMatchPattern() {
if (this.crawleripmustnotmatch == null) {
- final String r = get(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key);
+ final String r = this.get(CrawlAttribute.CRAWLER_IP_MUSTNOTMATCH.key);
try {
this.crawleripmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.crawleripmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -608,7 +631,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* @return a list of country codes
*/
public String[] countryMustMatchList() {
- String countryMustMatch = get(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key);
+ String countryMustMatch = this.get(CrawlAttribute.CRAWLER_COUNTRY_MUSTMATCH.key);
if (countryMustMatch == null) countryMustMatch = CrawlProfile.MATCH_NEVER_STRING;
if (countryMustMatch.isEmpty()) return new String[0];
String[] list = CommonPattern.COMMA.split(countryMustMatch);
@@ -622,7 +645,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern crawlerNoDepthLimitMatchPattern() {
if (this.crawlernodepthlimitmatch == null) {
- final String r = get(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key);
+ final String r = this.get(CrawlAttribute.CRAWLER_URL_NODEPTHLIMITMATCH.key);
try {
this.crawlernodepthlimitmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.crawlernodepthlimitmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -636,7 +659,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern indexUrlMustMatchPattern() {
if (this.indexurlmustmatch == null) {
- final String r = get(CrawlAttribute.INDEXING_URL_MUSTMATCH.key);
+ final String r = this.get(CrawlAttribute.INDEXING_URL_MUSTMATCH.key);
try {
this.indexurlmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.indexurlmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -650,7 +673,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern indexUrlMustNotMatchPattern() {
if (this.indexurlmustnotmatch == null) {
- final String r = get(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key);
+ final String r = this.get(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key);
try {
this.indexurlmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.indexurlmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -664,7 +687,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern indexContentMustMatchPattern() {
if (this.indexcontentmustmatch == null) {
- final String r = get(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key);
+ final String r = this.get(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key);
try {
this.indexcontentmustmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.indexcontentmustmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -678,7 +701,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
*/
public Pattern indexContentMustNotMatchPattern() {
if (this.indexcontentmustnotmatch == null) {
- final String r = get(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key);
+ final String r = this.get(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key);
try {
this.indexcontentmustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.indexcontentmustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -688,7 +711,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
/**
* Get the Pattern on media type that documents must match in order to be indexed
- *
+ *
* @return a {@link Pattern} instance, defaulting to
* {@link CrawlProfile#MATCH_ALL_PATTERN} when the regular expression
* string is not set or its syntax is incorrect
@@ -696,7 +719,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public Pattern getIndexMediaTypeMustMatchPattern() {
if (this.indexMediaTypeMustMatch == null) {
/* Cache the compiled pattern for faster next calls */
- final String patternStr = get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key);
+ final String patternStr = this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key);
try {
this.indexMediaTypeMustMatch = (patternStr == null
|| patternStr.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN
@@ -710,7 +733,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
/**
* Get the Pattern on media type that documents must not match in order to be indexed
- *
+ *
* @return a {@link Pattern} instance, defaulting to
* {@link CrawlProfile#MATCH_NEVER_PATTERN} when the regular expression
* string is not set or its syntax is incorrect
@@ -718,7 +741,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public Pattern getIndexMediaTypeMustNotMatchPattern() {
if (this.indexMediaTypeMustNotMatch == null) {
/* Cache the compiled pattern for faster next calls */
- final String patternStr = get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key);
+ final String patternStr = this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key);
try {
this.indexMediaTypeMustNotMatch = (patternStr == null
|| patternStr.equals(CrawlProfile.MATCH_NEVER_STRING)) ? CrawlProfile.MATCH_NEVER_PATTERN
@@ -736,7 +759,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* @return depth of crawl job
*/
public int depth() {
- final String r = get(CrawlAttribute.DEPTH.key);
+ final String r = this.get(CrawlAttribute.DEPTH.key);
if (r == null) return 0;
try {
return Integer.parseInt(r);
@@ -751,7 +774,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* be indexed as links (with metadata only on URL and not on content).
*/
public boolean isIndexNonParseableUrls() {
- final String r = get(CrawlAttribute.DIRECT_DOC_BY_URL.key);
+ final String r = this.get(CrawlAttribute.DIRECT_DOC_BY_URL.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
@@ -763,7 +786,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
* with an unknown or unsupported file extension.
*/
public boolean isCrawlerAlwaysCheckMediaType() {
- final String r = get(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key);
+ final String r = this.get(CrawlAttribute.CRAWLER_ALWAYS_CHECK_MEDIA_TYPE.key);
if (r == null) {
return false;
}
@@ -771,7 +794,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public CacheStrategy cacheStrategy() {
- final String r = get(CrawlAttribute.CACHE_STRAGEGY.key);
+ final String r = this.get(CrawlAttribute.CACHE_STRAGEGY.key);
if (r == null) return CacheStrategy.IFEXIST;
try {
return CacheStrategy.decode(Integer.parseInt(r));
@@ -782,7 +805,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public void setCacheStrategy(final CacheStrategy newStrategy) {
- put(CrawlAttribute.CACHE_STRAGEGY.key, newStrategy.toString());
+ this.put(CrawlAttribute.CACHE_STRAGEGY.key, newStrategy.toString());
}
/**
@@ -792,7 +815,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public long recrawlIfOlder() {
// returns a long (millis) that is the minimum age that
// an entry must have to be re-crawled
- final String r = get(CrawlAttribute.RECRAWL_IF_OLDER.key);
+ final String r = this.get(CrawlAttribute.RECRAWL_IF_OLDER.key);
if (r == null) return 0L;
try {
final long l = Long.parseLong(r);
@@ -806,7 +829,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public int domMaxPages() {
// this is the maximum number of pages that are crawled for a single domain
// if -1, this means no limit
- final String r = get(CrawlAttribute.DOM_MAX_PAGES.key);
+ final String r = this.get(CrawlAttribute.DOM_MAX_PAGES.key);
if (r == null) return Integer.MAX_VALUE;
try {
final int i = Integer.parseInt(r);
@@ -819,61 +842,61 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public boolean crawlingQ() {
- final String r = get(CrawlAttribute.CRAWLING_Q.key);
+ final String r = this.get(CrawlAttribute.CRAWLING_Q.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean followFrames() {
- final String r = get(CrawlAttribute.FOLLOW_FRAMES.key);
+ final String r = this.get(CrawlAttribute.FOLLOW_FRAMES.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean obeyHtmlRobotsNoindex() {
- final String r = get(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key);
+ final String r = this.get(CrawlAttribute.OBEY_HTML_ROBOTS_NOINDEX.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean obeyHtmlRobotsNofollow() {
- final String r = get(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key);
+ final String r = this.get(CrawlAttribute.OBEY_HTML_ROBOTS_NOFOLLOW.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean indexText() {
- final String r = get(CrawlAttribute.INDEX_TEXT.key);
+ final String r = this.get(CrawlAttribute.INDEX_TEXT.key);
if (r == null) return true;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean indexMedia() {
- final String r = get(CrawlAttribute.INDEX_MEDIA.key);
+ final String r = this.get(CrawlAttribute.INDEX_MEDIA.key);
if (r == null) return true;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean noindexWhenCanonicalUnequalURL() {
- final String r = get(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key);
+ final String r = this.get(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key);
if (r == null) return true;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean storeHTCache() {
- final String r = get(CrawlAttribute.STORE_HTCACHE.key);
+ final String r = this.get(CrawlAttribute.STORE_HTCACHE.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean remoteIndexing() {
- final String r = get(CrawlAttribute.REMOTE_INDEXING.key);
+ final String r = this.get(CrawlAttribute.REMOTE_INDEXING.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public int snapshotMaxdepth() {
- final String r = get(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key);
+ final String r = this.get(CrawlAttribute.SNAPSHOTS_MAXDEPTH.key);
if (r == null) return -1;
try {
final int i = Integer.parseInt(r);
@@ -886,20 +909,20 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public boolean snapshotLoadImage() {
- final String r = get(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key);
+ final String r = this.get(CrawlAttribute.SNAPSHOTS_LOADIMAGE.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public boolean snapshotReplaceold() {
- final String r = get(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key);
+ final String r = this.get(CrawlAttribute.SNAPSHOTS_REPLACEOLD.key);
if (r == null) return false;
return (r.equals(Boolean.TRUE.toString()));
}
public Pattern snapshotsMustnotmatch() {
if (this.snapshotsMustnotmatch == null) {
- final String r = get(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key);
+ final String r = this.get(CrawlAttribute.SNAPSHOTS_MUSTNOTMATCH.key);
try {
this.snapshotsMustnotmatch = (r == null || r.equals(CrawlProfile.MATCH_ALL_STRING)) ? CrawlProfile.MATCH_ALL_PATTERN : Pattern.compile(r, Pattern.CASE_INSENSITIVE);
} catch (final PatternSyntaxException e) { this.snapshotsMustnotmatch = CrawlProfile.MATCH_NEVER_PATTERN; }
@@ -908,11 +931,11 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public int timezoneOffset() {
- final String timezoneOffset = get(CrawlAttribute.TIMEZONEOFFSET.key);
+ final String timezoneOffset = this.get(CrawlAttribute.TIMEZONEOFFSET.key);
if (timezoneOffset == null) return 0;
try {
return Integer.parseInt(timezoneOffset);
- } catch (NumberFormatException e) {
+ } catch (final NumberFormatException e) {
return 0;
}
}
@@ -949,7 +972,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
}
public static String subpathFilter(final Collection<? extends MultiProtocolURL> urls) {
- LinkedHashSet<String> filters = new LinkedHashSet<String>(); // first collect in a set to eliminate doubles
+ final LinkedHashSet<String> filters = new LinkedHashSet<>(); // first collect in a set to eliminate doubles
for (final MultiProtocolURL url: urls) filters.add(mustMatchSubpath(url));
final StringBuilder filter = new StringBuilder();
for (final String urlfilter: filters) filter.append('|').append(urlfilter);
@@ -976,8 +999,8 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
final boolean dark,
final int count,
final int domlistlength) {
- boolean terminateButton = active && !CrawlSwitchboard.DEFAULT_PROFILES.contains(this.name());
- boolean deleteButton = !active;
+ final boolean terminateButton = active && !CrawlSwitchboard.DEFAULT_PROFILES.contains(this.name());
+ final boolean deleteButton = !active;
prop.put(CRAWL_PROFILE_PREFIX + count + "_status", terminateButton ? 1 : deleteButton ? 0 : 2);
prop.put(CRAWL_PROFILE_PREFIX + count + "_terminateButton", terminateButton);
prop.put(CRAWL_PROFILE_PREFIX + count + "_terminateButton_handle", this.handle());
@@ -1006,7 +1029,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexURLMustNotMatch", this.get(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key));
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexContentMustMatch", this.get(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key));
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexContentMustNotMatch", this.get(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key));
- prop.put(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL, noindexWhenCanonicalUnequalURL() ? 1 : 0);
+ prop.put(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL, this.noindexWhenCanonicalUnequalURL() ? 1 : 0);
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key, this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key));
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key));
prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, this.get(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key));
@@ -1042,19 +1065,19 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements M
public static void main(String[] args) {
// test to convert the key set from set to string and back
- Set<String> a = new HashSet<>();
+ final Set<String> a = new HashSet<>();
a.add("eins"); a.add("zwei"); a.add("drei");
JSONArray j = new JSONArray(a);
- String s = j.toString();
+ final String s = j.toString();
System.out.println(s);
- JSONTokener o = new JSONTokener(s);
+ final JSONTokener o = new JSONTokener(s);
try {
j = new JSONArray(o);
System.out.println(j);
- Set<String> h = new HashSet<String>();
+ final Set<String> h = new HashSet<>();
for (int i = 0; i < j.length(); i++) h.add(j.getString(i));
System.out.println(h);
- } catch (JSONException e) {
+ } catch (final JSONException e) {
e.printStackTrace();
}
}
diff --git a/source/net/yacy/document/importer/WarcImporter.java b/source/net/yacy/document/importer/WarcImporter.java
index 14ce06bec..02787e4be 100644
--- a/source/net/yacy/document/importer/WarcImporter.java
+++ b/source/net/yacy/document/importer/WarcImporter.java
@@ -22,6 +22,7 @@
*/
package net.yacy.document.importer;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -44,6 +45,7 @@ import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.util.ByteBuffer;
import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.crawler.data.CrawlProfile;
import net.yacy.crawler.retrieval.Request;
import net.yacy.crawler.retrieval.Response;
import net.yacy.document.TextParser;
@@ -83,23 +85,31 @@ public class WarcImporter extends Thread implements Importer {
private final long sourceSize; // length of the input source (for statistic)
private long consumed; // bytes consumed from input source (for statistic)
private boolean abort = false; // flag to signal stop of import
+ private String collection;
- public WarcImporter(MultiProtocolURL url) throws IOException {
+ public WarcImporter(MultiProtocolURL url, String collection) throws IOException {
super("WarcImporter - from InputStream");
this.recordCnt = 0;
this.sourceSize = -1;
this.name = url.toNormalform(true);
this.source = url.getInputStream(ClientIdentification.yacyInternetCrawlerAgent);
if (this.name.endsWith(".gz")) this.source = new GZIPInputStream(this.source);
+ this.collection = collection;
}
- public WarcImporter(File f) throws IOException {
+ public WarcImporter(File f, byte[] data, String collection) throws IOException {
super("WarcImporter - from file " + f.getName());
this.name = f.getName();
- this.sourceSize = f.length();
- this.source = new FileInputStream(f);
- if (this.name.endsWith(".gz")) this.source = new GZIPInputStream(this.source);
- }
+ if (!f.exists() && data != null) {
+ this.sourceSize = data.length;
+ this.source = new ByteArrayInputStream(data);
+ } else {
+ this.sourceSize = f.length();
+ this.source = new FileInputStream(f);
+ if (this.name.endsWith(".gz")) this.source = new GZIPInputStream(this.source);
+ }
+ this.collection = collection;
+ }
/**
* Reads a Warc file and adds all contained responses to the index.
@@ -115,6 +125,10 @@ public class WarcImporter extends Thread implements Importer {
job = this;
this.startTime = System.currentTimeMillis();
+ final CrawlProfile warcProfile = (CrawlProfile) Switchboard.getSwitchboard().crawler.defaultPackProfile.clone();
+ warcProfile.setCollections(this.collection);
+ warcProfile.setHandle();
+
WarcReader localwarcReader = WarcReaderFactory.getReader(f);
WarcRecord wrec = localwarcReader.getNextRecord();
while (wrec != null && !this.abort) {
@@ -165,15 +179,15 @@ public class WarcImporter extends Thread implements Importer {
requestHeader.referer() == null ? null : requestHeader.referer().hash(),
"warc",
responseHeader.lastModified(),
- Switchboard.getSwitchboard().crawler.defaultPackProfile.handle(),
+ warcProfile.handle(),
0,
- Switchboard.getSwitchboard().crawler.defaultPackProfile.timezoneOffset());
+ warcProfile.timezoneOffset());
final Response response = new Response(
request,
requestHeader,
responseHeader,
- Switchboard.getSwitchboard().crawler.defaultPackProfile,
+ warcProfile,
false,
content
);
diff --git a/source/net/yacy/document/importer/ZimImporter.java b/source/net/yacy/document/importer/ZimImporter.java
index 20e946d27..a1c002016 100644
--- a/source/net/yacy/document/importer/ZimImporter.java
+++ b/source/net/yacy/document/importer/ZimImporter.java
@@ -37,6 +37,12 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import org.apache.commons.io.FileUtils;
+import org.openzim.ZIMFile;
+import org.openzim.ZIMReader;
+import org.openzim.ZIMReader.ArticleEntry;
+import org.openzim.ZIMReader.DirectoryEntry;
+
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.protocol.ClientIdentification;
@@ -44,16 +50,12 @@ import net.yacy.cora.protocol.HeaderFramework;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.protocol.ResponseHeader;
import net.yacy.cora.util.ConcurrentLog;
+import net.yacy.crawler.data.CrawlProfile;
import net.yacy.crawler.retrieval.Request;
import net.yacy.crawler.retrieval.Response;
import net.yacy.document.TextParser;
import net.yacy.search.Switchboard;
-import org.openzim.ZIMFile;
-import org.openzim.ZIMReader;
-import org.openzim.ZIMReader.ArticleEntry;
-import org.openzim.ZIMReader.DirectoryEntry;
-
/**
* ZIM importer
* can import ZIM file i.e. from https://download.kiwix.org/zim/ or mirrors like https://ftp.fau.de/kiwix/zim/
@@ -69,10 +71,12 @@ public class ZimImporter extends Thread implements Importer {
static public ZimImporter job;
- private ZIMFile file;
+ private final ZIMFile file;
private ZIMReader reader;
- private String path;
+ private final String path;
private String guessedSource;
+ private final byte[] data;
+ private final String collection;
private int recordCnt;
private long startTime;
@@ -80,62 +84,75 @@ public class ZimImporter extends Thread implements Importer {
private long consumed;
private boolean abort = false;
- public ZimImporter(String path) throws IOException {
+ public ZimImporter(String path, byte[] data, String collection) throws IOException {
super("ZimImporter - from file " + path);
this.path = path;
- this.file = new ZIMFile(this.path); // this will read already some of the metadata and could consume some time
- this.sourceSize = this.file.length();
+ this.data = data;
+ File zimFilePath = new File(path);
+ if (!zimFilePath.exists() && this.data != null) {
+ File tempFile = File.createTempFile(zimFilePath.getName().substring(0, zimFilePath.getName().length() - 4), "zim");
+ FileUtils.writeByteArrayToFile(tempFile, data);
+ this.file = new ZIMFile(tempFile.getPath());
+ } else {
+ this.file = new ZIMFile(this.path); // this will read already some of the metadata and could consume some time
+ }
+ this.sourceSize = data == null ? this.file.length() : data.length;
+ this.collection = collection;
}
@Override
public void run() {
job = this;
this.startTime = System.currentTimeMillis();
- Switchboard sb = Switchboard.getSwitchboard();
+ final Switchboard sb = Switchboard.getSwitchboard();
try {
this.reader = new ZIMReader(this.file);
- this.guessedSource = getSource(this.reader);
- Date guessedDate = getDate(this.reader);
- String dates = HeaderFramework.newRfc1123Format().format(guessedDate);
+ this.guessedSource = getSource(this.reader, this.file.getName());
+ final Date guessedDate = getDate(this.reader);
+ final String dates = HeaderFramework.newRfc1123Format().format(guessedDate);
// verify the source
- DirectoryEntry mainEntry = this.reader.getMainDirectoryEntry();
- DigestURL mainURL = guessURL(this.guessedSource, mainEntry);
- if (!mainURL.exists(ClientIdentification.browserAgent)) {
- sb.log.info("zim importer: file " + this.file.getName() + " failed main url existence test: " + mainURL);
- return;
- }
+ final DirectoryEntry mainEntry = this.reader.getMainDirectoryEntry();
+ final DigestURL mainURL = guessURL(this.guessedSource, mainEntry);
+ //if (!mainURL.exists(ClientIdentification.browserAgent)) {
+ // sb.log.info("zim importer: file " + this.file.getName() + " failed main url existence test: " + mainURL);
+ // return;
+ //}
+
+ final CrawlProfile zimProfile = (CrawlProfile) Switchboard.getSwitchboard().crawler.defaultPackProfile.clone();
+ zimProfile.setCollections(this.collection);
+ zimProfile.setHandle();
// read all documents
for (int i = 0; i < this.file.header_entryCount; i++) {
try {
if (this.abort) break;
- DirectoryEntry de = this.reader.getDirectoryInfo(i);
+ final DirectoryEntry de = this.reader.getDirectoryInfo(i);
if (!(de instanceof ZIMReader.ArticleEntry)) continue;
- ArticleEntry ae = (ArticleEntry) de;
+ final ArticleEntry ae = (ArticleEntry) de;
if (ae.namespace != 'C' && ae.namespace != 'A') continue;
-
+
// check url
- DigestURL guessedUrl = guessURL(this.guessedSource, de);
- if (recordCnt < 10) {
+ final DigestURL guessedUrl = guessURL(this.guessedSource, de);
+ //if (this.recordCnt < 10) {
// critical test for the first 10 urls
- if (!guessedUrl.exists(ClientIdentification.browserAgent)) {
- sb.log.info("zim importer: file " + this.file.getName() + " failed url " + recordCnt + " existence test: " + guessedUrl);
- return;
- }
- }
-
+ // if (!guessedUrl.exists(ClientIdentification.browserAgent)) {
+ // sb.log.info("zim importer: file " + this.file.getName() + " failed url " + this.recordCnt + " existence test: " + guessedUrl);
+ // return;
+ // }
+ //}
+
// check availability of text parser
- String mimeType = ae.getMimeType();
+ final String mimeType = ae.getMimeType();
if (!mimeType.startsWith("text/") && !mimeType.equals("application/epub+zip")) continue; // in this import we want only text, not everything that is possible
if (TextParser.supportsMime(mimeType) != null) continue;
-
+
// read the content
- byte[] b = this.reader.getArticleData(ae);
-
+ final byte[] b = this.reader.getArticleData(ae);
+
// create artificial request and response headers for the indexer
- RequestHeader requestHeader = new RequestHeader();
- ResponseHeader responseHeader = new ResponseHeader(200);
+ final RequestHeader requestHeader = new RequestHeader();
+ final ResponseHeader responseHeader = new ResponseHeader(200);
responseHeader.put(HeaderFramework.CONTENT_TYPE, de.getMimeType()); // very important to tell parser which kind of content
responseHeader.put(HeaderFramework.LAST_MODIFIED, dates); // put in the guessd date to have something that is not the current date
final Request request = new Request(
@@ -144,29 +161,29 @@ public class ZimImporter extends Thread implements Importer {
null, // referrerhash the hash of the referrer URL
de.title, // name the name of the document to crawl
null, // appdate the time when the url was first time appeared
- sb.crawler.defaultPackProfile.handle(), // profileHandle the name of the prefetch profile. This must not be null!
+ zimProfile.handle(), // profileHandle the name of the prefetch profile. This must not be null!
0, // depth the crawling depth of the entry
- sb.crawler.defaultPackProfile.timezoneOffset() // timezone offset
+ zimProfile.timezoneOffset() // timezone offset
);
final Response response = new Response(
request,
requestHeader,
responseHeader,
- Switchboard.getSwitchboard().crawler.defaultPackProfile,
+ zimProfile,
false,
b
);
-
+
// throw this to the indexer
- String error = sb.toIndexer(response);
+ final String error = sb.toIndexer(response);
if (error != null) ConcurrentLog.info("ZimImporter", "error parsing: " + error);
this.recordCnt++;
- } catch (Exception e) {
+ } catch (final Exception e) {
// catch any error that could stop the importer
ConcurrentLog.info("ZimImporter", "error loading: " + e.getMessage());
}
}
- } catch (IOException e) {
+ } catch (final IOException e) {
ConcurrentLog.info("ZimImporter", "error reading: " + e.getMessage());
}
ConcurrentLog.info("ZimImporter", "Indexed " + this.recordCnt + " documents");
@@ -190,7 +207,7 @@ public class ZimImporter extends Thread implements Importer {
@Override
public int speed() {
if (this.recordCnt == 0) return 0;
- return (int) (this.recordCnt / Math.max(0L, runningTime() ));
+ return (int) (this.recordCnt / Math.max(0L, this.runningTime() ));
}
@Override
@@ -203,7 +220,7 @@ public class ZimImporter extends Thread implements Importer {
if (this.consumed == 0) {
return 0;
}
- long speed = this.consumed / runningTime();
+ final long speed = this.consumed / this.runningTime();
return (this.sourceSize - this.consumed) / speed;
}
@@ -217,11 +234,11 @@ public class ZimImporter extends Thread implements Importer {
return null; // Handle null or empty input
}
- String[] parts = fileName.split("_");
+ final String[] parts = fileName.split("_");
if (parts.length == 0) {
return null;
}
- String firstPart = parts[0];
+ final String firstPart = parts[0];
// Handling special cases where the domain name might not be obvious
// These are based on your provided list and can be expanded as needed
@@ -293,8 +310,8 @@ public class ZimImporter extends Thread implements Importer {
// Handling domain patterns
if (firstPart.contains(".stackexchange.com")) {
return firstPart;
- } else if (firstPart.endsWith(".com") || firstPart.endsWith(".org") || firstPart.endsWith(".de") ||
- firstPart.endsWith(".fr") || firstPart.endsWith(".pt") || firstPart.endsWith(".it") ||
+ } else if (firstPart.endsWith(".com") || firstPart.endsWith(".org") || firstPart.endsWith(".de") ||
+ firstPart.endsWith(".fr") || firstPart.endsWith(".pt") || firstPart.endsWith(".it") ||
firstPart.endsWith(".ja") || firstPart.endsWith(".es") || firstPart.endsWith(".eo")) {
return firstPart;
} else if (firstPart.contains("-")) {
@@ -303,7 +320,7 @@ public class ZimImporter extends Thread implements Importer {
// Additional general domain extraction logic
if (firstPart.contains(".")) {
- int lastDotIndex = firstPart.lastIndexOf('.');
+ final int lastDotIndex = firstPart.lastIndexOf('.');
if (lastDotIndex > 0 && lastDotIndex < firstPart.length() - 1) {
// Extract up to the next character beyond the TLD, to support TLDs of variable length
int endIndex = firstPart.indexOf('.', lastDotIndex + 1);
@@ -318,19 +335,20 @@ public class ZimImporter extends Thread implements Importer {
return null;
}
- public static String getSource(ZIMReader r) throws IOException {
+ public static String getSource(ZIMReader r, String fileName) throws IOException {
String source = r.getMetadata("Source");
if (source != null) return source;
- source = "https://" + guessDomainName(r.getZIMFile().getName()) + "/";
+ if (fileName == null || fileName.length() == 0) fileName = r.getZIMFile().getName();
+ source = "https://" + guessDomainName(fileName) + "/";
return source;
}
public static Date getDate(ZIMReader r) throws IOException {
- String date = r.getMetadata("Date");
+ final String date = r.getMetadata("Date");
if (date != null) try {
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd", Locale.US);
+ final SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd", Locale.US);
return format.parse(date);
- } catch (ParseException e) {}
+ } catch (final ParseException e) {}
// failover situation: use file date
return new Date(r.getZIMFile().lastModified());
}
@@ -491,47 +509,47 @@ public class ZimImporter extends Thread implements Importer {
};
public static void main(String[] args) {
- Set<String> skip = new HashSet<>();
- for (String s: skip_files) skip.add(s);
+ final Set<String> skip = new HashSet<>();
+ for (final String s: skip_files) skip.add(s);
// zim file import test
// will test mostly if domain names are included in zim file urls
- String zimFilesPath = args[0];
- File zimFiles = new File(zimFilesPath);
+ final String zimFilesPath = args[0];
+ final File zimFiles = new File(zimFilesPath);
// make ordered file list; order by file size (start with smallest)
- String[] filelist = zimFiles.list();
- Map<Long, File> orderedFileMap = new TreeMap<>();
+ final String[] filelist = zimFiles.list();
+ final Map<Long, File> orderedFileMap = new TreeMap<>();
for (int i = 0; i < filelist.length; i++) {
if (!filelist[i].endsWith(".zim")) continue;
- File f = new File(zimFiles, filelist[i]);
+ final File f = new File(zimFiles, filelist[i]);
orderedFileMap.put(f.length() * 1000 + i, f);
}
- Collection<File> orderedFiles = orderedFileMap.values();
- Set<String> files_ok = new LinkedHashSet<>();
- Set<String> files_nok = new LinkedHashSet<>();
- for (File f: orderedFiles) {
+ final Collection<File> orderedFiles = orderedFileMap.values();
+ final Set<String> files_ok = new LinkedHashSet<>();
+ final Set<String> files_nok = new LinkedHashSet<>();
+ for (final File f: orderedFiles) {
if (skip.contains(f.getName())) continue;
try {
- ZIMFile z = new ZIMFile(f.getAbsolutePath());
- ZIMReader r = new ZIMReader(z);
- DirectoryEntry de = r.getMainDirectoryEntry();
+ final ZIMFile z = new ZIMFile(f.getAbsolutePath());
+ final ZIMReader r = new ZIMReader(z);
+ final DirectoryEntry de = r.getMainDirectoryEntry();
System.out.println("ZIM file: " + f.getAbsolutePath());
- for (String key: ZIMReader.METADATA_KEYS) {String s = r.getMetadata(key); if (s != null) System.out.println("Metadata " + key + ": " + s);};
+ for (final String key: ZIMReader.METADATA_KEYS) {final String s = r.getMetadata(key); if (s != null) System.out.println("Metadata " + key + ": " + s);}
System.out.println("Namespace: " + de.namespace);
System.out.println("Title: " + de.title);
System.out.println("URL: " + de.url);
System.out.println("Mime Type " + de.getMimeType());
System.out.println("guessed domain: " + guessDomainName(f.getName())); // uses a table and rules that deduces a source from the file name
- String source = getSource(r);
+ final String source = getSource(r, null);
System.out.println("guessed Source: " + source); // this uses metadata stored in the zim file
- DigestURL mainURL = guessURL(source, de);
+ final DigestURL mainURL = guessURL(source, de);
System.out.println("guessed main article: " + mainURL);
- boolean ok = mainURL.exists(ClientIdentification.browserAgent);
+ final boolean ok = mainURL.exists(ClientIdentification.browserAgent);
System.out.println("main article exists: " + ok);
if (ok) files_ok.add(f.getName()); else files_nok.add(f.getName());
System.out.println();
- } catch (IOException e) {
+ } catch (final IOException e) {
e.printStackTrace();
}
}
diff --git a/source/net/yacy/htroot/IndexImportWarc_p.java b/source/net/yacy/htroot/IndexImportWarc_p.java
index 325454605..28e9e3957 100644
--- a/source/net/yacy/htroot/IndexImportWarc_p.java
+++ b/source/net/yacy/htroot/IndexImportWarc_p.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.net.MalformedURLException;
import net.yacy.cora.document.id.MultiProtocolURL;
+import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.importer.WarcImporter;
import net.yacy.server.serverObjects;
@@ -52,11 +53,14 @@ public class IndexImportWarc_p {
if (post != null) {
if (post.containsKey("file") || post.containsKey("url")) {
final String filename = post.get("file");
+ final String collection = post.get("collection", "user");
+ final String data64 = post.get("file$file", null); // file uploads are all base64-encoded in YaCyDefaultServlet.parseMultipart
+ final byte[] data = data64 == null ? null : Base64Order.standardCoder.decode(data64);
if (filename != null && filename.length() > 0) {
final File sourcefile = new File(filename);
- if (sourcefile.exists()) {
+ if (sourcefile.exists() || data != null) {
try {
- final WarcImporter wi = new WarcImporter(sourcefile);
+ final WarcImporter wi = new WarcImporter(sourcefile, data, collection);
wi.start();
prop.put("import_thread", "started");
} catch (final IOException ex) {
@@ -73,7 +77,7 @@ public class IndexImportWarc_p {
if (urlstr != null && urlstr.length() > 0) {
try {
final MultiProtocolURL url = new MultiProtocolURL(urlstr);
- final WarcImporter wi = new WarcImporter(url);
+ final WarcImporter wi = new WarcImporter(url, collection);
wi.start();
prop.put("import_thread", "started");
} catch (final MalformedURLException ex) {
diff --git a/source/net/yacy/htroot/IndexImportZim_p.java b/source/net/yacy/htroot/IndexImportZim_p.java
index 8517121d5..07b9c2039 100644
--- a/source/net/yacy/htroot/IndexImportZim_p.java
+++ b/source/net/yacy/htroot/IndexImportZim_p.java
@@ -21,6 +21,7 @@ package net.yacy.htroot;
import java.io.File;
import java.io.IOException;
+import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.document.importer.ZimImporter;
import net.yacy.server.serverObjects;
@@ -29,16 +30,16 @@ import net.yacy.server.serverSwitch;
public class IndexImportZim_p {
public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader request, final serverObjects post, @SuppressWarnings("unused") final serverSwitch env) {
-
+
// read multipart data from post request
-
+
final serverObjects prop = new serverObjects();
-
+
if (ZimImporter.job != null && ZimImporter.job.isAlive()) {
// one import is running, no option to insert anything
prop.put("import", 1);
prop.put("import_thread", "running");
- prop.put("import_warcfile", ZimImporter.job.source());
+ prop.put("import_zimfile", ZimImporter.job.source());
prop.put("import_count", ZimImporter.job.count());
prop.put("import_speed", ZimImporter.job.speed());
prop.put("import_runningHours", (ZimImporter.job.runningTime() / 60) / 60);
@@ -51,13 +52,17 @@ public class IndexImportZim_p {
} else {
prop.put("import", 0);
if (post != null) {
+ //for (final String s: post.keySet()) System.out.println("key: " + s); // print all post key attributes
if (post.containsKey("file")) {
final String filename = post.get("file");
+ final String collection = post.get("collection", "user");
+ final String data64 = post.get("file$file", null); // file uploads are all base64-encoded in YaCyDefaultServlet.parseMultipart
+ final byte[] data = data64 == null ? null : Base64Order.standardCoder.decode(data64);
if (filename != null && filename.length() > 0) {
final File sourcefile = new File(filename);
- if (sourcefile.exists()) {
+ if (data != null || sourcefile.exists()) {
try {
- final ZimImporter zi = new ZimImporter(sourcefile.getAbsolutePath());
+ final ZimImporter zi = new ZimImporter(sourcefile.getAbsolutePath(), data, collection);
zi.start();
prop.put("import_thread", "started");
} catch (final IOException ex) {
@@ -70,7 +75,7 @@ public class IndexImportZim_p {
prop.put("import_thread", "Error: file not found [" + filename + "]");
}
}
-
+
prop.put("import_count", 0);
prop.put("import_speed", 0);
prop.put("import_runningHours", 0);
diff --git a/source/net/yacy/http/servlets/YaCyDefaultServlet.java b/source/net/yacy/http/servlets/YaCyDefaultServlet.java
index 235a3fd18..c9650f466 100644
--- a/source/net/yacy/http/servlets/YaCyDefaultServlet.java
+++ b/source/net/yacy/http/servlets/YaCyDefaultServlet.java
@@ -147,7 +147,7 @@ public class YaCyDefaultServlet extends HttpServlet {
this._htDocsPath = sb.htDocsPath;
this._htLocalePath = sb.getDataPath("locale.translated_html", "DATA/LOCALE/htroot");
- this._servletContext = getServletContext();
+ this._servletContext = this.getServletContext();
this._mimeTypes = new MimeTypes();
final String tmpstr = this.getServletContext().getInitParameter("welcomeFile");
@@ -156,12 +156,12 @@ public class YaCyDefaultServlet extends HttpServlet {
} else {
this._welcomes = new String[]{tmpstr,"index.html"};
}
- this._acceptRanges = getInitBoolean("acceptRanges", this._acceptRanges);
- this._dirAllowed = getInitBoolean("dirAllowed", this._dirAllowed);
+ this._acceptRanges = this.getInitBoolean("acceptRanges", this._acceptRanges);
+ this._dirAllowed = this.getInitBoolean("dirAllowed", this._dirAllowed);
Resource.setDefaultUseCaches(false); // caching is handled internally (prevent double caching)
- final String rb = getInitParameter("resourceBase");
+ final String rb = this.getInitParameter("resourceBase");
try {
if (rb != null) {
this._resourceBase = Resource.newResource(rb);
@@ -176,12 +176,12 @@ public class YaCyDefaultServlet extends HttpServlet {
if (ConcurrentLog.isFine("FILEHANDLER")) {
ConcurrentLog.fine("FILEHANDLER","YaCyDefaultServlet: resource base = " + this._resourceBase);
}
- this.templateMethodCache = new ConcurrentHashMap<String, Method>();
+ this.templateMethodCache = new ConcurrentHashMap<>();
}
/* ------------------------------------------------------------ */
protected boolean getInitBoolean(final String name, final boolean dft) {
- final String value = getInitParameter(name);
+ final String value = this.getInitParameter(name);
if (value == null || value.length() == 0) {
return dft;
}
@@ -243,7 +243,7 @@ public class YaCyDefaultServlet extends HttpServlet {
// Is this a Range request?
reqRanges = request.getHeaders(HeaderFramework.RANGE);
- if (!hasDefinedRange(reqRanges)) {
+ if (!this.hasDefinedRange(reqRanges)) {
reqRanges = null;
}
}
@@ -261,7 +261,7 @@ public class YaCyDefaultServlet extends HttpServlet {
if (reqRanges == null && !endsWithSlash) {
final int p = pathInContext.lastIndexOf('.');
if (p >= 0) {
- final Method rewriteMethod = rewriteMethod(pathInContext);
+ final Method rewriteMethod = this.rewriteMethod(pathInContext);
if (rewriteMethod != null) {
hasClass = true;
} else {
@@ -276,7 +276,7 @@ public class YaCyDefaultServlet extends HttpServlet {
}
// find resource
- resource = getResource(pathInContext);
+ resource = this.getResource(pathInContext);
if (!hasClass && (resource == null || !resource.exists()) && !pathInContext.contains("..")) {
// try to get this in the alternative htDocsPath
@@ -304,10 +304,10 @@ public class YaCyDefaultServlet extends HttpServlet {
response.sendRedirect(response.encodeRedirectURL(URIUtil.addPaths(this._servletContext.getContextPath(), pathInContext)));
} else {
if (hasClass) { // this is a YaCy servlet, handle the template
- handleTemplate(pathInfo, request, response);
+ this.handleTemplate(pathInfo, request, response);
} else {
- if (included || passConditionalHeaders(request, response, resource)) {
- sendData(request, response, included, resource, reqRanges);
+ if (included || this.passConditionalHeaders(request, response, resource)) {
+ this.sendData(request, response, included, resource, reqRanges);
}
}
}
@@ -332,7 +332,7 @@ public class YaCyDefaultServlet extends HttpServlet {
response.sendRedirect(response.encodeRedirectURL(buf.toString()));
}
} // else look for a welcome file
- else if (null != (welcome = getWelcomeFile(pathInContext))) {
+ else if (null != (welcome = this.getWelcomeFile(pathInContext))) {
ConcurrentLog.fine("FILEHANDLER","welcome={}" + welcome);
// Forward to the index
@@ -345,8 +345,8 @@ public class YaCyDefaultServlet extends HttpServlet {
}
}
} else {
- if (included || passConditionalHeaders(request, response, resource)) {
- sendDirectory(request, response, resource, pathInContext);
+ if (included || this.passConditionalHeaders(request, response, resource)) {
+ this.sendDirectory(request, response, resource, pathInContext);
}
}
}
@@ -366,7 +366,7 @@ public class YaCyDefaultServlet extends HttpServlet {
@Override
protected void doPost(final HttpServletRequest request, final HttpServletResponse response)
throws ServletException, IOException {
- doGet(request, response);
+ this.doGet(request, response);
}
/* ------------------------------------------------------------ */
@@ -399,7 +399,7 @@ public class YaCyDefaultServlet extends HttpServlet {
}
for (final String _welcome : this._welcomes) {
final String welcome_in_context = URIUtil.addPaths(pathInContext, _welcome);
- final Resource welcome = getResource(welcome_in_context);
+ final Resource welcome = this.getResource(welcome_in_context);
if (welcome != null && welcome.exists()) {
return _welcome;
}
@@ -524,7 +524,7 @@ public class YaCyDefaultServlet extends HttpServlet {
if (include) {
resource.writeTo(out, 0, content_length);
} else {
- writeHeaders(response, resource, content_length);
+ this.writeHeaders(response, resource, content_length);
resource.writeTo(out, 0, content_length);
}
} else {
@@ -533,7 +533,7 @@ public class YaCyDefaultServlet extends HttpServlet {
// if there are no satisfiable ranges, send 416 response
if (ranges == null || ranges.isEmpty()) {
- writeHeaders(response, resource, content_length);
+ this.writeHeaders(response, resource, content_length);
response.setStatus(HttpServletResponse.SC_REQUESTED_RANGE_NOT_SATISFIABLE);
response.setHeader(HttpHeader.CONTENT_RANGE.asString(),
InclusiveByteRange.to416HeaderRangeString(content_length));
@@ -547,7 +547,7 @@ public class YaCyDefaultServlet extends HttpServlet {
if (ranges.size() == 1) {
final InclusiveByteRange singleSatisfiableRange = ranges.iterator().next();
final long singleLength = singleSatisfiableRange.getSize();
- writeHeaders(response, resource, singleLength);
+ this.writeHeaders(response, resource, singleLength);
response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT);
response.setHeader(HttpHeader.CONTENT_RANGE.asString(),
singleSatisfiableRange.toHeaderRangeString(content_length));
@@ -560,7 +560,7 @@ public class YaCyDefaultServlet extends HttpServlet {
// 216 response which does not require an overall
// content-length header
//
- writeHeaders(response, resource, -1);
+ this.writeHeaders(response, resource, -1);
final String mimetype = response.getContentType();
if (mimetype == null) {
ConcurrentLog.warn("FILEHANDLER","YaCyDefaultServlet: Unknown mimetype for " + request.getRequestURI());
@@ -660,7 +660,7 @@ public class YaCyDefaultServlet extends HttpServlet {
}
protected Object invokeServlet(final Method targetMethod, final RequestHeader request, final serverObjects args) throws IllegalArgumentException, IllegalAccessException, InvocationTargetException {
- return targetMethod.invoke(null, new Object[]{request, args, Switchboard.getSwitchboard()}); // add switchboard
+ return targetMethod.invoke(null, request, args, Switchboard.getSwitchboard()); // add switchboard
}
/**
@@ -813,8 +813,8 @@ public class YaCyDefaultServlet extends HttpServlet {
localeSelection = lng;
}
}
- final File targetLocalizedFile = getLocalizedFile(target, localeSelection);
- final Method targetMethod = rewriteMethod(target);
+ final File targetLocalizedFile = this.getLocalizedFile(target, localeSelection);
+ final Method targetMethod = this.rewriteMethod(target);
final String targetExt = target.substring(target.lastIndexOf('.') + 1);
final long now = System.currentTimeMillis();
@@ -844,19 +844,19 @@ public class YaCyDefaultServlet extends HttpServlet {
// standard attributes are just pushed as string
args.put(argName, request.getParameter(argName));
}
- final RequestHeader legacyRequestHeader = generateLegacyRequestHeader(request, target, targetExt);
+ final RequestHeader legacyRequestHeader = this.generateLegacyRequestHeader(request, target, targetExt);
// add multipart-form fields to parameter
if (ServletFileUpload.isMultipartContent(request)) {
- parseMultipart(request, args);
+ this.parseMultipart(request, args);
}
// eof modification to read attribute
Object tmp;
try {
if (args.isEmpty()) {
// yacy servlets typically test for args != null (but not for args .isEmpty())
- tmp = invokeServlet(targetMethod, legacyRequestHeader, null);
+ tmp = this.invokeServlet(targetMethod, legacyRequestHeader, null);
} else {
- tmp = invokeServlet(targetMethod, legacyRequestHeader, args);
+ tmp = this.invokeServlet(targetMethod, legacyRequestHeader, args);
}
} catch(final InvocationTargetException e) {
if(e.getCause() instanceof InvalidURLLicenceException) {
@@ -929,7 +929,7 @@ public class YaCyDefaultServlet extends HttpServlet {
result = RasterPlotter.exportImage(bi, targetExt);
}
- updateRespHeadersForImages(target, response);
+ this.updateRespHeadersForImages(target, response);
final String mimeType = Classification.ext2mime(targetExt, MimeTypes.Type.TEXT_HTML.asString());
response.setContentType(mimeType);
response.setContentLength(result.length());
@@ -942,9 +942,9 @@ public class YaCyDefaultServlet extends HttpServlet {
if (tmp instanceof InputStream) {
/* Images and favicons can also be written directly from an inputStream */
- updateRespHeadersForImages(target, response);
+ this.updateRespHeadersForImages(target, response);
- writeInputStream(response, targetExt, (InputStream)tmp);
+ this.writeInputStream(response, targetExt, (InputStream)tmp);
return;
}
@@ -1007,9 +1007,9 @@ public class YaCyDefaultServlet extends HttpServlet {
if (targetLocalizedFile.exists() && targetLocalizedFile.isFile() && targetLocalizedFile.canRead()) {
- sb.setConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, appendPath(sb.getConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, ""), target));
+ sb.setConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, this.appendPath(sb.getConfig(SwitchboardConstants.SERVER_SERVLETS_CALLED, ""), target));
if (args != null && !args.isEmpty()) {
- sb.setConfig("server.servlets.submitted", appendPath(sb.getConfig("server.servlets.submitted", ""), target));
+ sb.setConfig("server.servlets.submitted", this.appendPath(sb.getConfig("server.servlets.submitted", ""), target));
}
// add the application version, the uptime and the client name to every rewrite table
@@ -1071,7 +1071,7 @@ public class YaCyDefaultServlet extends HttpServlet {
TemplateEngine.writeTemplate(targetLocalizedFile.getName(), fis, bas, templatePatterns);
// handle SSI
- parseSSI (bas.toByteArray(),request,response);
+ this.parseSSI (bas.toByteArray(),request,response);
} finally {
try {
fis.close();
@@ -1258,7 +1258,7 @@ public class YaCyDefaultServlet extends HttpServlet {
InputStream filecontent = null;
try {
filecontent = item.getInputStream();
- files.put(new AbstractMap.SimpleEntry<String, byte[]>(item.getFieldName(), FileUtils.read(filecontent)));
+ files.put(new AbstractMap.SimpleEntry<>(item.getFieldName(), FileUtils.read(filecontent)));
} catch (final IOException e) {
ConcurrentLog.info("FILEHANDLER", e.getMessage());
} finally {
@@ -1271,7 +1271,7 @@ public class YaCyDefaultServlet extends HttpServlet {
final String n = job.getKey();
final byte[] v = job.getValue();
final String filename = args.get(n);
- if (filename != null && filename.endsWith(".gz")) {
+ if (filename != null && (filename.endsWith(".gz") || filename.endsWith(".zip") || filename.endsWith(".warc") || filename.endsWith(".zim"))) {
// transform this value into base64
final String b64 = Base64Order.standardCoder.encode(v);
args.put(n + "$file", b64);
diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java
index a8ed948b6..9ce73fbe4 100644
--- a/source/net/yacy/search/Switchboard.java
+++ b/source/net/yacy/search/Switchboard.java
@@ -2118,15 +2118,15 @@ public final class Switchboard extends serverSwitch {
return null;
}
- public boolean processPack(final String s) {
- final File infile = new File(this.packsLoadPath, s);
+ public boolean processPack(final String inFileName, String collectionName) {
+ final File infile = new File(this.packsLoadPath, inFileName);
if ( !infile.exists() || !infile.canWrite() || !infile.canRead() ) {
return false;
}
- final File outfile = new File(this.packsLoadedPath, s);
+ final File outfile = new File(this.packsLoadedPath, inFileName);
//if (outfile.exists()) return false;
boolean moved = false;
- if ( s.endsWith("xml.zip") ) {
+ if ( inFileName.endsWith("xml.zip") ) {
// open the zip file with all the xml files in it
ZipInputStream zis = null;
try {
@@ -2141,7 +2141,7 @@ public final class Switchboard extends serverSwitch {
baos.write(buffer, 0, size);
}
baos.flush();
- this.processXMLPack(new ByteArrayInputStream(baos.toByteArray()), entry.getName());
+ this.processXMLPack(new ByteArrayInputStream(baos.toByteArray()), entry.getName(), collectionName);
baos.close();
if (this.shallTerminate()) break;
}
@@ -2154,9 +2154,9 @@ public final class Switchboard extends serverSwitch {
}
}
return moved;
- } else if (s.endsWith(".warc") || s.endsWith(".warc.gz")) {
+ } else if (inFileName.endsWith(".warc") || inFileName.endsWith(".warc.gz")) {
try {
- final WarcImporter wri = new WarcImporter(infile);
+ final WarcImporter wri = new WarcImporter(infile, null, collectionName);
wri.start();
try {
wri.join();
@@ -2168,9 +2168,9 @@ public final class Switchboard extends serverSwitch {
this.log.warn("IO Error processing warc file " + infile);
}
return moved;
- } else if (s.endsWith(".zim")) {
+ } else if (inFileName.endsWith(".zim")) {
try {
- final ZimImporter wri = new ZimImporter(infile.getAbsolutePath());
+ final ZimImporter wri = new ZimImporter(infile.getAbsolutePath(), null, collectionName);
wri.start();
try {
wri.join();
@@ -2183,16 +2183,16 @@ public final class Switchboard extends serverSwitch {
}
return moved;
} else if (
- s.endsWith(".jsonl") || s.endsWith(".jsonl.gz") ||
- s.endsWith(".jsonlist") || s.endsWith(".jsonlist.gz") ||
- s.endsWith(".flatjson") || s.endsWith(".flatjson.gz")) {
+ inFileName.endsWith(".jsonl") || inFileName.endsWith(".jsonl.gz") ||
+ inFileName.endsWith(".jsonlist") || inFileName.endsWith(".jsonlist.gz") ||
+ inFileName.endsWith(".flatjson") || inFileName.endsWith(".flatjson.gz")) {
return this.processPackJson(infile, outfile);
}
InputStream is = null;
try {
is = new BufferedInputStream(new FileInputStream(infile));
- if (s.endsWith(".gz")) is = new GZIPInputStream(is, 65535);
- this.processXMLPack(is, infile.getName());
+ if (inFileName.endsWith(".gz")) is = new GZIPInputStream(is, 65535);
+ this.processXMLPack(is, infile.getName(), collectionName);
} catch (final IOException e ) {
ConcurrentLog.logException(e);
} finally {
@@ -2244,9 +2244,13 @@ public final class Switchboard extends serverSwitch {
return moved;
}
- private void processXMLPack(final InputStream is, final String name) throws IOException {
+ private void processXMLPack(final InputStream is, final String name, String collection) throws IOException {
final int concurrency = Runtime.getRuntime().availableProcessors();
+ final CrawlProfile xmlProfile = (CrawlProfile) Switchboard.getSwitchboard().crawler.defaultPackProfile.clone();
+ xmlProfile.setCollections(collection);
+ xmlProfile.setHandle();
+
// start reader thread
final XMLPackReader reader = new XMLPackReader(is, 100, this.crawlStacker, this.index.fulltext().getDefaultConfiguration(), concurrency);
final Thread readerThread = new Thread(reader, name);
@@ -2305,10 +2309,10 @@ public final class Switchboard extends serverSwitch {
null,
"",
entry.getDate(),
- Switchboard.this.crawler.defaultPackProfile.handle(),
+ xmlProfile.handle(),
0,
- Switchboard.this.crawler.defaultPackProfile.timezoneOffset());
- final Response response = new Response(request, null, null, Switchboard.this.crawler.defaultPackProfile, false, null);
+ xmlProfile.timezoneOffset());
+ final Response response = new Response(request, null, null, xmlProfile, false, null);
final IndexingQueueEntry queueEntry =
new IndexingQueueEntry(response, new Document[] {document}, null);
@@ -2401,7 +2405,7 @@ public final class Switchboard extends serverSwitch {
if (this.isPackFile(pack)) {
// read the pack file and store entry in index
- if ( this.processPack(pack) ) {
+ if ( this.processPack(pack, "user") ) {
return true;
}
}
generated by cgit v1.2.3 (git 2.46.0) at 2026-08-08 10:31:11 +0000