diff options
| -rw-r--r-- | examples/surrogate_dublin_core.xml | 13 | ||||
| -rw-r--r-- | source/net/yacy/document/content/SurrogateReader.java | 18 | ||||
| -rw-r--r-- | source/net/yacy/search/schema/CollectionConfiguration.java | 17 |
3 files changed, 40 insertions, 8 deletions
diff --git a/examples/surrogate_dublin_core.xml b/examples/surrogate_dublin_core.xml index 2558b1a62..aa574c2df 100644 --- a/examples/surrogate_dublin_core.xml +++ b/examples/surrogate_dublin_core.xml @@ -15,7 +15,9 @@ --> <surrogates - xmlns:dc="http://purl.org/dc/elements/1.1/"> + xmlns:dc="http://purl.org/dc/elements/1.1/" + xmlns:md="http://localhost:8090/api/schema.xml?core=collection1" +> <record> <dc:Title><![CDATA[Alan Smithee]]></dc:Title> @@ -23,6 +25,15 @@ <dc:Description><![CDATA[Der als Filmregisseur oft genannte '''Alan Smithee''' ist ein Anagramm von „The Alias Men“.]]></dc:Description> <dc:Language>de</dc:Language> <dc:Date>2009-03-02T11:12:36Z</dc:Date> <!-- date is in ISO 8601 --> + <md:h1_txt>Alan Smithee</md:h1_txt> + <md:h2_txt>Geschichte</md:h2_txt> + <md:h3_txt>Entstehung</md:h3_txt> + <md:h3_txt>Aufdeckung und Abkehr</md:h3_txt> + <md:h2_txt>Verwendung</md:h2_txt> + <md:h2_txt>Literatur</md:h2_txt> + <md:h2_txt>Weblinks</md:h2_txt> + <md:h2_txt>Referenzen</md:h2_txt> + <md:collection_sxt>surrogate</md:collection_sxt> </record> </surrogates> diff --git a/source/net/yacy/document/content/SurrogateReader.java b/source/net/yacy/document/content/SurrogateReader.java index 3e0199d25..4eb01e07b 100644 --- a/source/net/yacy/document/content/SurrogateReader.java +++ b/source/net/yacy/document/content/SurrogateReader.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
+import java.util.Map;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.zip.GZIPInputStream;
@@ -140,7 +141,7 @@ public class SurrogateReader extends DefaultHandler implements Runnable { } else if ("value".equals(tag)) {
this.buffer.setLength(0);
this.parsingValue = true;
- } else if (tag.startsWith("dc:") || tag.startsWith("geo:")) {
+ } else if (tag.startsWith("dc:") || tag.startsWith("geo:") || tag.startsWith("md:")) {
// parse dublin core attribute
this.elementName = tag;
this.parsingValue = true;
@@ -173,15 +174,18 @@ public class SurrogateReader extends DefaultHandler implements Runnable { }
this.buffer.setLength(0);
this.parsingValue = false;
- } else if (tag.startsWith("dc:") || tag.startsWith("geo:")) {
+ } else if (tag.startsWith("dc:") || tag.startsWith("geo:") || tag.startsWith("md:")) {
final String value = buffer.toString().trim();
if (this.elementName != null && tag.equals(this.elementName)) {
- value.replaceAll(";", ",");
- String oldcontent = this.surrogate.get(this.elementName);
- if (oldcontent == null) {
- this.surrogate.getMap().put(this.elementName, new String[]{value});
+ Map<String,String[]> map = this.surrogate.getMap();
+ String[] oldcontent = map.get(this.elementName);
+ if (oldcontent == null || oldcontent.length == 0) {
+ map.put(this.elementName, new String[]{value});
} else {
- this.surrogate.getMap().put(this.elementName, new String[]{oldcontent + ";" + value});
+ String[] newcontent = new String[oldcontent.length + 1];
+ System.arraycopy(oldcontent, 0, newcontent, 0, oldcontent.length);
+ newcontent[oldcontent.length] = value;
+ map.put(this.elementName, newcontent);
}
}
this.buffer.setLength(0);
diff --git a/source/net/yacy/search/schema/CollectionConfiguration.java b/source/net/yacy/search/schema/CollectionConfiguration.java index 9e98b5487..c6eef588c 100644 --- a/source/net/yacy/search/schema/CollectionConfiguration.java +++ b/source/net/yacy/search/schema/CollectionConfiguration.java @@ -70,6 +70,7 @@ import net.yacy.crawler.retrieval.Response; import net.yacy.document.Condenser; import net.yacy.document.Document; import net.yacy.document.SentenceReader; +import net.yacy.document.content.DCEntry; import net.yacy.document.parser.html.ContentScraper; import net.yacy.document.parser.html.ImageEntry; import net.yacy.kelondro.data.citation.CitationReference; @@ -788,6 +789,22 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri add(doc, CollectionSchema.publisher_url_s, html.getPublisherLink().toNormalform(true)); } } + + if (parser instanceof DCEntry) { + // the document was created with a surrogate parsing; overwrite all md: -entries to Solr + DCEntry dcentry = (DCEntry) parser; + for (Map.Entry<String, String[]> entry: dcentry.getMap().entrySet()) { + String tag = entry.getKey(); + if (!tag.startsWith("md:") || tag.length() < 4) continue; + CollectionSchema solr_field = CollectionSchema.valueOf(tag.substring(3)); + if (solr_field == null) continue; + String[] values = entry.getValue(); + if (values == null || values.length == 0) continue; + if (allAttr || contains(solr_field)) { + add(doc, solr_field, values); + } + } + } String content = document.getTextString(); String tokens = digestURL.toTokens(); |