diff options
| author | orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542> | 2010-04-30 14:03:51 +0000 |
|---|---|---|
| committer | orbiter <orbiter@6c8d7289-2bf4-0310-a012-ef5d649a1542> | 2010-04-30 14:03:51 +0000 |
| commit | fc5efcc05a7817c2d5c2a7e1c1b958eb5bbc3d87 (patch) | |
| tree | a1345968d0b82fbe99fb2651c1b19f0eaa71eacc /htroot | |
| parent | c2098f9399b78b3de4de87ca6fdb95529911a130 (diff) | |
enhanced and fixed OAI-PMH import
- now importing OAI-PMH server list fron two sources
- simultanous import from several servers (even > 2000)
- check buttons on OAI-PMH server list to select multiple servers for import start
- it is possible to select all servers at once for import
- imported XML data is gzipped after import from surrogate reader
git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@6847 6c8d7289-2bf4-0310-a012-ef5d649a1542
Diffstat (limited to 'htroot')
| -rw-r--r-- | htroot/CrawlResults.java | 4 | ||||
| -rw-r--r-- | htroot/IndexImportOAIPMHList_p.html | 33 | ||||
| -rw-r--r-- | htroot/IndexImportOAIPMHList_p.java | 40 | ||||
| -rw-r--r-- | htroot/IndexImportOAIPMH_p.html | 2 | ||||
| -rw-r--r-- | htroot/IndexImportOAIPMH_p.java | 44 |
5 files changed, 96 insertions, 27 deletions
diff --git a/htroot/CrawlResults.java b/htroot/CrawlResults.java index f3c3f2f75..7efe7dd8c 100644 --- a/htroot/CrawlResults.java +++ b/htroot/CrawlResults.java @@ -187,8 +187,8 @@ public class CrawlResults { entry = i.next();
try {
urle = sb.indexSegments.urlMetadata(Segments.Process.LOCALCRAWLING).load(entry.getKey().getBytes(), null, 0);
- if(urle == null) {
- Log.logWarning("PLASMA", "CrawlResults: URL not in index for crawl result "+ i +" with hash "+ entry.getKey());
+ if (urle == null) {
+ Log.logWarning("PLASMA", "CrawlResults: URL not in index with url hash "+ entry.getKey());
urlstr = null;
urltxt = null;
metadata = null;
diff --git a/htroot/IndexImportOAIPMHList_p.html b/htroot/IndexImportOAIPMHList_p.html index 99b0756c3..168a1f85d 100644 --- a/htroot/IndexImportOAIPMHList_p.html +++ b/htroot/IndexImportOAIPMHList_p.html @@ -4,20 +4,49 @@ <title>YaCy '#[clientname]#': OAI-PMH source import list</title>
#%env/templates/metas.template%#
#(refresh)#::<meta http-equiv="REFRESH" content="6" />#(/refresh)#
+ <script>
+ <!--
+ function setall(name, check){
+ var selectForm = document.forms.namedItem(name);
+ var count = selectForm.elements["num"].value;
+ if (check) for(i = 0; i < count; i++) {
+ if (selectForm.elements["item_" + i].checked) {
+ check = false;
+ break;
+ }
+ }
+ for(i = 0; i < count; i++){
+ selectForm.elements["item_" + i].checked = check;
+ }
+ }
+ -->
+ </script>
+ <script src="/js/sorttable.js"></script>
</head>
<body>
#(source)#::
- <h3>OAI Source List</h3>
+ <h3>List of #[num]# OAI-PMH Servers</h3>
+ <form action="IndexImportOAIPMH_p.html" target="_top" method="post" enctype="multipart/form-data" accept-charset="UTF-8" name="oaipmhimport">
+ <p>
+ <input type="hidden" name="num" value="#[num]#" />
+ <input type="submit" name="loadrows" value="Load Selected Sources" />
+ </p>
<table cellpadding="2" cellspacing="1" >
<tr class="TableHeader">
+ <td><input type="checkbox" name="allswitch" onclick="setall(this.form.name, this.value)" /></td>
<td>Source</td>
</tr>
#{table}#
<tr class="TableCell#(dark)#Light::Dark#(/dark)#">
- <td>#[source]#</td>
+ <td align="left"><input type="checkbox" name="item_#[count]#" value="mark_#[source]#" /></td>
+ <td>#[loadurl]#</td>
</tr>
#{/table}#
</table>
+ <p>
+ <input type="submit" name="loadrows" value="Load Selected Sources" />
+ </p>
+ </form>
#(/source)#
#(import)#::
diff --git a/htroot/IndexImportOAIPMHList_p.java b/htroot/IndexImportOAIPMHList_p.java index 9c2dd257b..ceba573a2 100644 --- a/htroot/IndexImportOAIPMHList_p.java +++ b/htroot/IndexImportOAIPMHList_p.java @@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.Set;
+import net.yacy.document.importer.OAIListFriendsLoader;
import net.yacy.document.importer.OAIPMHImporter;
import de.anomic.http.server.RequestHeader;
@@ -43,39 +44,42 @@ public class IndexImportOAIPMHList_p { prop.put("source", 0);
if (post != null && post.containsKey("source")) {
- Set<String> oaiRoots = OAIPMHImporter.getAllListedOAIServer(sb.loader);
+ Set<String> oaiRoots = OAIListFriendsLoader.load(sb.loader).keySet();
boolean dark = false;
- int cnt = 0;
+ int count = 0;
for (String root: oaiRoots) {
- prop.put("source_table_" + cnt + "_dark", (dark) ? "1" : "0");
- prop.put("source_table_" + cnt + "_source", "<a href=\"/IndexImportOAIPMH_p.html?importroot=&urlstartall=" + root + "\" target=\"_top\">" + root+ "</a>");
+ prop.put("source_table_" + count + "_dark", (dark) ? "1" : "0");
+ prop.put("source_table_" + count + "_count", count);
+ prop.put("source_table_" + count + "_source", root);
+ prop.put("source_table_" + count + "_loadurl", "<a href=\"/IndexImportOAIPMH_p.html?urlstart=" + root + "\" target=\"_top\">" + root + "</a>");
dark = !dark;
- cnt++;
+ count++;
}
- prop.put("source_table", cnt);
+ prop.put("source_table", count);
+ prop.put("source_num", count);
prop.put("source", 1);
}
if (post != null && post.containsKey("import")) {
ArrayList<OAIPMHImporter> jobs = new ArrayList<OAIPMHImporter>();
- for (OAIPMHImporter job: OAIPMHImporter.runningJobs) jobs.add(job);
- for (OAIPMHImporter job: OAIPMHImporter.startedJobs) jobs.add(job);
- for (OAIPMHImporter job: OAIPMHImporter.finishedJobs) jobs.add(job);
+ for (OAIPMHImporter job: OAIPMHImporter.runningJobs.keySet()) jobs.add(job);
+ for (OAIPMHImporter job: OAIPMHImporter.startedJobs.keySet()) jobs.add(job);
+ for (OAIPMHImporter job: OAIPMHImporter.finishedJobs.keySet()) jobs.add(job);
boolean dark = false;
- int cnt = 0;
+ int count = 0;
for (OAIPMHImporter job: jobs) {
- prop.put("import_table_" + cnt + "_dark", (dark) ? "1" : "0");
- prop.put("import_table_" + cnt + "_thread", (job.isAlive()) ? "<img src=\"/env/grafics/loading.gif\" alt=\"running\" />" : "finished");
- prop.put("import_table_" + cnt + "_source", job.source());
- prop.put("import_table_" + cnt + "_chunkCount", job.chunkCount());
- prop.put("import_table_" + cnt + "_recordsCount", job.count());
- prop.put("import_table_" + cnt + "_speed", job.speed());
+ prop.put("import_table_" + count + "_dark", (dark) ? "1" : "0");
+ prop.put("import_table_" + count + "_thread", (job.isAlive()) ? "<img src=\"/env/grafics/loading.gif\" alt=\"running\" />" : "finished");
+ prop.put("import_table_" + count + "_source", job.source());
+ prop.put("import_table_" + count + "_chunkCount", job.chunkCount());
+ prop.put("import_table_" + count + "_recordsCount", job.count());
+ prop.put("import_table_" + count + "_speed", job.speed());
dark = !dark;
- cnt++;
+ count++;
}
- prop.put("import_table", cnt);
+ prop.put("import_table", count);
prop.put("import", 1);
prop.put("refresh", 1);
}
diff --git a/htroot/IndexImportOAIPMH_p.html b/htroot/IndexImportOAIPMH_p.html index 325bedf01..be20d2527 100644 --- a/htroot/IndexImportOAIPMH_p.html +++ b/htroot/IndexImportOAIPMH_p.html @@ -33,7 +33,7 @@ <fieldset>
<legend>Import all Records from a server</legend>
Import all records that follow according to resumption elements into index<br />
- <input name="urlstartall" type="text" value="" size="80" />
+ <input name="urlstart" type="text" value="" size="80" />
<input name="importroot" type="submit" value="import this source" />
#(optiongetlist)#::or <input name="getlist" type="submit" value="import from a list" />#(/optiongetlist)#
#(status)#::<p>Import started!</p>::<p>Bad input data: #[message]# </p>#(/status)#
diff --git a/htroot/IndexImportOAIPMH_p.java b/htroot/IndexImportOAIPMH_p.java index 55a62dcec..dc6985979 100644 --- a/htroot/IndexImportOAIPMH_p.java +++ b/htroot/IndexImportOAIPMH_p.java @@ -24,9 +24,13 @@ import java.io.IOException;
import java.net.MalformedURLException;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeSet;
import net.yacy.document.importer.OAIPMHImporter;
-import net.yacy.document.importer.OAIPMHReader;
+import net.yacy.document.importer.OAIPMHLoader;
import net.yacy.document.importer.ResumptionToken;
import net.yacy.kelondro.data.meta.DigestURI;
import net.yacy.kelondro.logging.Log;
@@ -55,7 +59,7 @@ public class IndexImportOAIPMH_p { DigestURI url = null;
try {
url = new DigestURI(oaipmhurl, null);
- OAIPMHReader r = new OAIPMHReader(sb.loader, url, sb.surrogatesInPath, "oaipmh-one");
+ OAIPMHLoader r = new OAIPMHLoader(sb.loader, url, sb.surrogatesInPath, "oaipmh-one");
ResumptionToken rt = r.getResumptionToken();
prop.put("import-one", 1);
prop.put("import-one_count", (rt == null) ? "not available" : Integer.toString(rt.getRecordCounter()));
@@ -83,8 +87,8 @@ public class IndexImportOAIPMH_p { }
}
- if (post.containsKey("importroot")) {
- String oaipmhurl = post.get("urlstartall", "");
+ if (post.get("urlstart", "").length() > 0) {
+ String oaipmhurl = post.get("urlstart", "");
DigestURI url = null;
try {
url = new DigestURI(oaipmhurl, null);
@@ -100,6 +104,38 @@ public class IndexImportOAIPMH_p { }
}
+
+ if (post.get("loadrows", "").length() > 0) {
+ // create a time-ordered list of events to execute
+ TreeSet<String> sources = new TreeSet<String>();
+ for (Map.Entry<String, String> entry: post.entrySet()) {
+ if (entry.getValue().startsWith("mark_")) {
+ sources.add(entry.getValue().substring(5));
+ }
+ }
+ prop.put("status", 1);
+ prop.put("optiongetlist", 1);
+ prop.put("iframetype", 1);
+
+ // prepare the set for random read from it (to protect the servers at the beginning of the list)
+ ArrayList<String> sourceList = new ArrayList<String>(sources.size());
+ for (String oaipmhurl: sources) sourceList.add(oaipmhurl);
+ Random r = new Random(System.currentTimeMillis());
+
+ // start jobs for the sources
+ DigestURI url = null;
+ while (sourceList.size() > 0) {
+ String oaipmhurl = sourceList.remove(r.nextInt(sourceList.size()));
+ try {
+ url = new DigestURI(oaipmhurl, null);
+ OAIPMHImporter job = new OAIPMHImporter(sb.loader, url);
+ job.start();
+ } catch (MalformedURLException e) {
+ Log.logException(e);
+ }
+ }
+ }
+
if (post.containsKey("getlist")) {
prop.put("iframetype", 2);
}
|