diff options
47 files changed, 1278 insertions, 847 deletions
diff --git a/defaults/yacy.init b/defaults/yacy.init index 4bd2d8fa0..63db597c0 100644 --- a/defaults/yacy.init +++ b/defaults/yacy.init @@ -304,11 +304,12 @@ proxyCache.sync.lockTimeout = 2000 # you can use the proxy with fresh/stale rules or in a always-fresh mode
proxyAlwaysFresh = false
-# a path to the surrogate input directory
-surrogates.in = DATA/SURROGATES/in
-
-# a path to the surrogate output directory
-surrogates.out = DATA/SURROGATES/out
+# paths to the packs directories
+packs.hold = DATA/PACKS/hold
+packs.live = DATA/PACKS/live
+packs.load = DATA/PACKS/load
+packs.loaded = DATA/PACKS/loaded
+packs.unload = DATA/PACKS/unload
# a path to the dictionaries directory
# this directory also contains subdirectories for input sources, the did-you-mean function and other
@@ -726,10 +727,10 @@ recrawlindex_memprereq=1048576 62_remotetriggeredcrawl_memprereq=12582912
62_remotetriggeredcrawl_loadprereq=8.0
62_remotetriggeredcrawl_isPaused=false
-70_surrogates_idlesleep=10000
-70_surrogates_busysleep=0
-70_surrogates_memprereq=12582912
-70_surrogates_loadprereq=8.0
+70_packs_idlesleep=10000
+70_packs_busysleep=0
+70_packs_memprereq=12582912
+70_packs_loadprereq=8.0
720_ccimport_idlesleep=100
720_ccimport_busysleep=1000
720_ccimport_memprereq=1048576
diff --git a/htroot/ContentIntegrationPHPBB3_p.html b/htroot/ContentIntegrationPHPBB3_p.html index c22651f80..3c886f36f 100644 --- a/htroot/ContentIntegrationPHPBB3_p.html +++ b/htroot/ContentIntegrationPHPBB3_p.html @@ -22,8 +22,8 @@ <li>deselect the partial import flag</li>
</ul>
<p>
- When an export is started, surrogate files are generated into DATA/SURROGATE/in which are automatically fetched by an indexer thread.
- All indexed surrogate files are then moved to DATA/SURROGATE/out and can be re-cycled when an index is deleted.
+ When an export is started, pack files are generated into DATA/PACKS/load which are automatically fetched by an indexer thread.
+ All indexed pack files are then moved to DATA/PACKS/loaded and can be re-cycled when an index is deleted.
</p>
<form action="ContentIntegrationPHPBB3_p.html" accept-charset="UTF-8">
@@ -53,7 +53,7 @@ <dt><b>Password</b> for the account of that user given above</dt>
<dd><input type="text" name="content.phpbb3.dbpw" value="#[content.phpbb3.dbpw]#" size="20" /></dd>
- <dt><b>Posts per file</b><br />in exported surrogates</dt>
+ <dt><b>Posts per file</b><br />in exported packs</dt>
<dd><input type="text" name="content.phpbb3.ppf" value="#[content.phpbb3.ppf]#" size="20" /></dd>
<dt> </dt>
@@ -62,7 +62,7 @@ </dd>
<dt> </dt>
<dd>
- <input type="submit" name="export" value="Export Content to Surrogates" class="btn btn-primary" style="width:240px"/>
+ <input type="submit" name="export" value="Export Content to Packs" class="btn btn-primary" style="width:240px"/>
</dd>
</dl>
</fieldset>
@@ -98,7 +98,7 @@ #(/check)#
#(export)#::
- <p>Export successful! Wrote #[files]# files in DATA/SURROGATES/in</p>::
+ <p>Export successful! Wrote #[files]# files in DATA/PACKS/load</p>::
<p>Export failed: #[error]#</p>
#(/export)#
diff --git a/htroot/CrawlResults.html b/htroot/CrawlResults.html index 3eccec1e8..11ee431dd 100644 --- a/htroot/CrawlResults.html +++ b/htroot/CrawlResults.html @@ -17,7 +17,7 @@ <p>Case (6) is a monitor of the local receipt-generator, the opposed case of (1). It contains also an indexing result monitor but is not considered private
since it shows crawl requests from other peers.
</p>
- <p>Case (7) occurs if surrogate files are imported</p>
+ <p>Case (7) occurs if pack files are imported</p>
<p><img src="env/grafics/indexmonitor.png" width="600" height="308" alt="An illustration how yacy works" /></p>
<p>The image above illustrates the data flow initiated by web index acquisition.
Some processes occur double to document the complex index migration structure.
@@ -64,9 +64,9 @@ <p><em>Use Case:</em> This list may fill if you check the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page</p>
#(remoteCrawlerDisabled)#::<div class="info"><p>The remote crawler is currently disabled<p></div>#(/remoteCrawlerDisabled)#
::
- <h2>(7) Results from surrogates import</h2>
- <p>These records had been imported from surrogate files in DATA/SURROGATES/in</p>
- <p><em>Use Case:</em> place files with dublin core metadata content into DATA/SURROGATES/in or use an index import method (i.e. <a href="IndexImportMediawiki_p.html">MediaWiki import</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH retrieval</a>)</p>
+ <h2>(7) Results from pack import</h2>
+ <p>These records had been imported from pack files in DATA/PACKS/load</p>
+ <p><em>Use Case:</em> place files with dublin core metadata content into DATA/PACKS/load or use an index import method (i.e. <a href="IndexImportMediawiki_p.html">MediaWiki import</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH retrieval</a>)</p>
#(/process)#
diff --git a/htroot/IndexExportImportSolr_p.html b/htroot/IndexExportImportSolr_p.html new file mode 100644 index 000000000..c6f687557 --- /dev/null +++ b/htroot/IndexExportImportSolr_p.html @@ -0,0 +1,51 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "DTD/xhtml1-transitional.dtd"> +<!-- This page is only XHTML 1.0 Transitional because target is being used in a links --> +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <title>YaCy '#[clientname]#': URL Database Administration</title> + #%env/templates/metas.template%# + </head> + <body id="IndexControl"> + #%env/templates/header.template%# + #%env/templates/submenuIndexImport.template%# + + <h2>Solr Index Export/Import</h2> + <p>The local index currently contains #[ucount]# documents (including non-http-200 error pages).</p> + + #(dumprestore)#:: + <form action="IndexExportImportSolr_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8"> + <fieldset><legend>Dump and Restore of Solr Index</legend> + #(dumpRestoreEnabled)#<div class="alert alert-info">This feature is available only when a local embedded Solr is active.</div>::#(/dumpRestoreEnabled)# + <dl> + <dt> </dt> + <dd><input type="submit" name="indexdump" value="Create Dump" class="btn btn-primary" style="width:240px;" #(dumpRestoreEnabled)#disabled="disabled"::#(/dumpRestoreEnabled)#/><br/> + (This may take several minutes. Please be patient and wait until the page reloads.)</dd> + </dd> + </dl> + <dl> + <dt class="TableCellDark">Dump File (full path)</dt> + <dd><input type="text" name="dumpfile" value="#[dumpfile]#" size="80" maxlength="250" #(dumpRestoreEnabled)#disabled="disabled"::#(/dumpRestoreEnabled)#/> + </dd> + <dt> </dt> + <dd><input type="submit" name="indexrestore" value="Restore Dump" class="btn btn-primary" style="width:240px;" #(dumpRestoreEnabled)#disabled="disabled"::#(/dumpRestoreEnabled)#/> + </dd> + </dl> + </fieldset> + </form>:: + #(/dumprestore)# + + #(indexdump)#:: + <div class="alert alert-success" role="alert">Stored a solr dump to file #[dumpfile]#</div>:: + <div class="alert alert-danger" role="alert">Could not create the Solr dump : no embedded Solr is available.</div>:: + <div class="alert alert-danger" role="alert">An error occurred while trying to create the Solr dump.</div> + #(/indexdump)# + + #(indexRestore)#:: + <div class="alert alert-success" role="alert">Successfully restored Solr index from dump file!</div>:: + <div class="alert alert-danger" role="alert">Could not restore the Solr dump : no embedded Solr is available.</div>:: + <div class="alert alert-danger" role="alert">An error occurred while trying to restore the Solr dump.</div> + #(/indexRestore)# + + #%env/templates/footer.template%# + </body> +</html> diff --git a/htroot/IndexExport_p.html b/htroot/IndexExport_p.html index df58837c2..feb449643 100644 --- a/htroot/IndexExport_p.html +++ b/htroot/IndexExport_p.html @@ -40,38 +40,8 @@ <dt class="TableCellDark">Export Format</dt> <dd> <dl> - <dt>Full Data Records:</dt> - <dd><input type="radio" name="format" value="full-elasticsearch" checked="checked" /> - JSON (Rich and full-text Elasticsearch data, one document per line in one flat JSON file, - can be bulk-imported to elasticsearch. Here is an example for opensearch, using docker:<br /> -Start docker container of opensearch:<br /> -<code>docker run --name opensearch -p 9200:9200 -d -e OPENSEARCH_JAVA_OPTS="-Xms2G -Xmx2G" -e discovery.type=single-node -e DISABLE_SECURITY_PLUGIN=true -v $(pwd)/opensearch_data:/usr/share/opensearch/data opensearchproject/opensearch:latest</code><br /> -Unblock index creation:<br /> -<code>curl -X PUT "http://localhost:9200/_cluster/settings" -H 'Content-Type: application/json' -d' -{ - "persistent": { - "cluster.blocks.create_index": null - } -}'</code><br /> -Create the search index:<br /> -<code>curl -X PUT "http://localhost:9200/collection1/yacy"</code><br /> -Bulk-upload the index file:<br /> -<code>curl -XPOST "http://localhost:9200/collection1/yacy/_bulk?filter_path=took,errors" -H "Content-Type: application/x-ndjson" --data-binary @yacy_dump_XXX.flatjson</code><br /> -Make a search, get 10 results, search in fields text_t, title, description with boosts:<br /> -<code>curl -X POST "http://localhost:9200/collection1/yacy/_search" -H 'Content-Type: application/json' -d' -{"size": 10, "query": {"multi_match": { - "query": "one two three", - "fields": ["text_t", "title^10", "description^3"], "fuzziness": "AUTO" -}}}'</code><br /> - <input type="radio" name="format" value="full-solr" /> - XML (Rich and full-text Solr data, one document per line in one large xml file, - can be processed with shell tools, can be imported with DATA/SURROGATE/in/) - <br /> - <input type="radio" name="format" value="full-rss" /> - XML (RSS) - </dd> <dt>Full URL List:</dt> - <dd><input type="radio" name="format" value="url-text" /> Plain Text List (URLs only)<br /> + <dd><input type="radio" name="format" value="url-text" checked="checked"/> Plain Text List (URLs only)<br /> <input type="radio" name="format" value="url-html" /> HTML (URLs with title)</dd> <dt>Only Domain:</dt> <dd><input type="radio" name="format" value="dom-text" /> Plain Text List (domains only)<br /> @@ -79,7 +49,6 @@ Make a search, get 10 results, search in fields text_t, title, description with <dt>Only Text:</dt> <dd><input type="radio" name="format" value="text-text" /> Fulltext of Search Index Text</dd> </dl> - </dd> <dt> </dt> <dd><input type="submit" name="lurlexport" value="Export" class="btn btn-primary" style="width:240px;"/> </dd> @@ -91,46 +60,13 @@ Make a search, get 10 results, search in fields text_t, title, description with #(lurlexportfinished)#:: <div class="alert alert-success">Finished export of #[urlcount]# Documents to file <a href="file://#[exportfile]#" target="_">#[exportfile]#</a><br/> - <em>Import this file by moving it to DATA/SURROGATES/in</em></div>:: + <em>Import this file by moving it to DATA/PACKS/load</em></div>:: #(/lurlexportfinished)# #(lurlexporterror)#:: <div class="alert alert-warning">Export to file #[exportfile]# failed: #[exportfailmsg]#</div>:: #(/lurlexporterror)# - #(dumprestore)#:: - <form action="IndexExport_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8"> - <fieldset><legend>Dump and Restore of Solr Index</legend> - #(dumpRestoreEnabled)#<div class="alert alert-info">This feature is available only when a local embedded Solr is active.</div>::#(/dumpRestoreEnabled)# - <dl> - <dt> </dt> - <dd><input type="submit" name="indexdump" value="Create Dump" class="btn btn-primary" style="width:240px;" #(dumpRestoreEnabled)#disabled="disabled"::#(/dumpRestoreEnabled)#/> - </dd> - </dl> - <dl> - <dt class="TableCellDark">Dump File</dt> - <dd><input type="text" name="dumpfile" value="#[dumpfile]#" size="80" maxlength="250" #(dumpRestoreEnabled)#disabled="disabled"::#(/dumpRestoreEnabled)#/> - </dd> - <dt> </dt> - <dd><input type="submit" name="indexrestore" value="Restore Dump" class="btn btn-primary" style="width:240px;" #(dumpRestoreEnabled)#disabled="disabled"::#(/dumpRestoreEnabled)#/> - </dd> - </dl> - </fieldset> - </form>:: - #(/dumprestore)# - - #(indexdump)#:: - <div class="alert alert-success" role="alert">Stored a solr dump to file #[dumpfile]#</div>:: - <div class="alert alert-danger" role="alert">Could not create the Solr dump : no embedded Solr is available.</div>:: - <div class="alert alert-danger" role="alert">An error occurred while trying to create the Solr dump.</div> - #(/indexdump)# - - #(indexRestore)#:: - <div class="alert alert-success" role="alert">Successfully restored Solr index from dump file!</div>:: - <div class="alert alert-danger" role="alert">Could not restore the Solr dump : no embedded Solr is available.</div>:: - <div class="alert alert-danger" role="alert">An error occurred while trying to restore the Solr dump.</div> - #(/indexRestore)# - #%env/templates/footer.template%# </body> </html> diff --git a/htroot/IndexImportMediawiki_p.html b/htroot/IndexImportMediawiki_p.html index 5af34631b..5d3e5f92c 100644 --- a/htroot/IndexImportMediawiki_p.html +++ b/htroot/IndexImportMediawiki_p.html @@ -80,11 +80,11 @@ </surrogates> </pre> </li> - <li>Each 10000 wiki records are combined in one output file which is written to /DATA/SURROGATES/in into a temporary file.</li> + <li>Each 10000 wiki records are combined in one output file which is written to /DATA/PACKS/load into a temporary file.</li> <li>When each of the generated output file is finished, it is renamed to a .xml file</li> - <li>Each time a xml surrogate file appears in /DATA/SURROGATES/in, the YaCy indexer fetches the file and indexes the record entries.</li> - <li>When a surrogate file is finished with indexing, it is moved to /DATA/SURROGATES/out</li> - <li>You can recycle processed surrogate files by moving them from /DATA/SURROGATES/out to /DATA/SURROGATES/in</li> + <li>Each time a xml pack file appears in /DATA/PACKS/load, the YaCy indexer fetches the file and indexes the record entries.</li> + <li>When a pack file is finished with indexing, it is moved to /DATA/PACKS/loaded</li> + <li>You can recycle processed pack files by moving them from /DATA/PACKS/loaded to /DATA/PACKS/load</li> </ul> <br /> :: diff --git a/htroot/IndexImportOAIPMH_p.html b/htroot/IndexImportOAIPMH_p.html index 84c9da15f..a4f642db6 100644 --- a/htroot/IndexImportOAIPMH_p.html +++ b/htroot/IndexImportOAIPMH_p.html @@ -10,7 +10,7 @@ #%env/templates/header.template%#
#%env/templates/submenuIndexImport.template%#
<h2>OAI-PMH Import</h2>
- <p>Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for surrogates</a></p>
+ <p>Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for packs</a></p>
<form action="IndexImportOAIPMH_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8">
<fieldset>
diff --git a/htroot/IndexPackGenerator_p.html b/htroot/IndexPackGenerator_p.html new file mode 100644 index 000000000..ef5df6f07 --- /dev/null +++ b/htroot/IndexPackGenerator_p.html @@ -0,0 +1,137 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "DTD/xhtml1-transitional.dtd"> +<!-- This page is only XHTML 1.0 Transitional because target is being used in a links --> +<html xmlns="http://www.w3.org/1999/xhtml"> +#(reload)#::<meta http-equiv="REFRESH" content="5; url=/IndexPackGenerator_p.html">#(/reload)# + <head> + <title>YaCy '#[clientname]#': URL Database Administration</title> + #%env/templates/metas.template%# + </head> + <body id="IndexControl"> + #%env/templates/header.template%# + #%env/templates/submenuIndexImport.template%# + + <h2>Index Pack Generator</h2> + <p>The local index currently contains #[ucount]# documents, only #[ucount200]# exportable with status code 200 - the remaining are error documents.</p> + + #(lurlexport)#:: + <form action="IndexPackGenerator_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8"> + <fieldset><legend>Index Pack Generator</legend> + <dl> + <dt class="TableCellDark">Set a Category (this goes into the filename)</dt> + <dd> + <select name="category" id="category" class="form-control"> + <option value="core">core - technical documentation, operation systems, computer hardware, open source and free software, manuals, protocol standards</option> + <option value="scroll" selected="selected">scroll - non-technical documents: knowledge, encyclopedia, linguistic corpora, dictionaries, translation memories, texts, non-fiction books, historical books</option> + <option value="codex">codex - non-technical standards: industry standards, laws, rules, compliance</option> + <option value="gem">gem - research, papers, university publications, science</option> + <option value="fiction">fiction - fictional documents: movies, stories, series, books (fiction, science-fiction)</option> + <option value="map">map - geological data, geolocation-data, earth/world information</option> + <option value="echo">echo – micro-content (tweets, toots, short headlines, SMS corpora), podcasts, radio archives, audio lectures, spoken-word datasets, logs, incidents, telemetry</option> + <option value="spirit">spirit – related to non-textual data (possibly only metadata): art, music, game assets, creative-commons media (non-text culture loot)</option> + <option value="vault">vault - sensitive data: secrets, leaks, non-public documents, security advisories</option> + </select> + </dd> + + <dt class="TableCellDark">Index Collection</dt> + <dd> + <select id="collection" name="collection" class="form-control"> + #{collections}# + <option #(selected)#::selected="selected"#(/selected)#>#[collection]#</option> + #{/collections}# + </select> the collection name is used as part of the filename to describe the content. Exception: if the collection is "user", then you can name the content with a slug. + </dd> + + <dt class="TableCellDark">Slug - describe the content<br>(only if collection is "user")</dt> + <dd><input type="text" name="slug" id="slug" value="export" size="20" maxlength="40" class="form-control"/> This will become a part of the filename, spaces will be replaced by "-"; must not be empty; should end with a language description, e.g. "-en" + </dd> + + + <script> + const collectionSelect = document.getElementById('collection'); + const slugInput = document.getElementById('slug'); + + function toggleSlug() { + const enable = collectionSelect.value === 'user'; + slugInput.disabled = !enable; + slugInput.style.opacity = enable ? 1 : 0.5; // optional visual hint + } + + // run once on load + toggleSlug(); + + // keep it in sync + collectionSelect.addEventListener('change', toggleSlug); + </script> + + <dt class="TableCellDark">URL Filter</dt> + <dd><input type="text" name="exportfilter" value=".*.*" size="20" maxlength="250" class="form-control"/> .*.* (default) is a catch-all; format: java regex + </dd> + + <dt class="TableCellDark">Search Query - </dt> + <dd><input type="text" name="exportquery" value="*:*" size="20" maxlength="250" class="form-control"/> *:* (default) is a catch-all; format: <field-name>:<solr-pattern> + </dd> + <dt class="TableCellDark">Export Format</dt> + <dd><span class="info" style="float:right"><img src="env/grafics/i16.gif" width="16" height="16" alt="info"/><span style="right:0px;"> + This JSON is an elasticsearch index dump format and can be bulk-imported to elasticsearch. Here is an example for opensearch, using docker:<br /> + Start docker container of opensearch:<br /> + <code>docker run --name opensearch -p 9200:9200 -d -e OPENSEARCH_JAVA_OPTS="-Xms2G -Xmx2G" -e discovery.type=single-node -e DISABLE_SECURITY_PLUGIN=true -v $(pwd)/opensearch_data:/usr/share/opensearch/data opensearchproject/opensearch:latest</code><br /> + Unblock index creation:<br /> + <code>curl -X PUT "http://localhost:9200/_cluster/settings" -H 'Content-Type: application/json' -d' + { + "persistent": { + "cluster.blocks.create_index": null + } + }'</code><br /> + Create the search index:<br /> + <code>curl -X PUT "http://localhost:9200/collection1/yacy"</code><br /> + Bulk-upload the index file:<br /> + <code>curl -XPOST "http://localhost:9200/collection1/yacy/_bulk?filter_path=took,errors" -H "Content-Type: application/x-ndjson" --data-binary @yacy_dump_XXX.flatjson</code><br /> + Make a search, get 10 results, search in fields text_t, title, description with boosts:<br /> + <code>curl -X POST "http://localhost:9200/collection1/yacy/_search" -H 'Content-Type: application/json' -d' + {"size": 10, "query": {"multi_match": { + "query": "one two three", + "fields": ["text_t", "title^10", "description^3"], "fuzziness": "AUTO" + }}}'</code> + </span></span> + <input type="radio" name="format" value="full-elasticsearch" checked="checked" /> + JSON (Rich and full-text Elasticsearch data, one document per line in one flat JSON file) + <br /> + + <input type="radio" name="format" value="full-solr" /> + XML (Rich and full-text Solr data, one document per line in one large xml file, + can be processed with shell tools, can be imported with DATA/PACKS/load/) + <br /> + <input type="radio" name="format" value="full-rss" /> + XML (RSS) + </dd> + + <dt> </dt> + <dd><input type="submit" name="lurlexport" value="Generate Data Pack" class="btn btn-primary" style="width:240px;"/> + </dd> + </dl> + </fieldset> + </form>:: + <div class="alert alert-info" style="text-decoration:blink">Export to file #[exportfile]# is running .. #[urlcount]# Documents so far</div>:: + #(/lurlexport)# + + #(lurlexportfinished)#:: + <div class="alert alert-success">Finished export of #[urlcount]# Documents to file <a href="file://#[exportfile]#" target="_">#[exportfile]#</a><br/> + <em>Import this file by moving it to DATA/PACKS/load</em></div>:: + #(/lurlexportfinished)# + + #(lurlexporterror)#:: + <div class="alert alert-warning">Export to file #[exportfile]# failed: #[exportfailmsg]#</div>:: + #(/lurlexporterror)# + + <fieldset><legend>Pack List</legend> + <table border="0" summary="Pack List Archive"> + <tr class="TableHeader"><td>Pack</td><td>Process</td><td>Size (KB)</td></tr> + #{packs}# + <tr class="TableCell#(dark)#Light::Dark#(/dark)#"><td>#[file]#</td><td>#[type]#</td><td>#[size]#</td></tr> + #{/packs}# + </table> + </fieldset> + + #%env/templates/footer.template%# + </body> +</html> diff --git a/htroot/env/base.css b/htroot/env/base.css index 0a99d779d..a82e0b998 100644 --- a/htroot/env/base.css +++ b/htroot/env/base.css @@ -150,7 +150,6 @@ tt, *.tt { } .info { - font-weight:bold; color:olive; } @@ -1064,8 +1063,9 @@ div#info:hover span { display: block; position: absolute; font-size:1em; - width: 200px; + width: 600px; padding: 3px; + margin-right: 10px; color: #000000; background: #DDDDDD; text-align: center; @@ -1086,8 +1086,9 @@ div#tagcloud { display: block; position: absolute; font-size:1em; - width: 200px; + width: 600px; padding: 3px; + margin-right: 10px; color: #000000; background: #DDDDDD; text-align: left; diff --git a/htroot/env/templates/header.template b/htroot/env/templates/header.template index c5267f2db..b42d9a2c2 100644 --- a/htroot/env/templates/header.template +++ b/htroot/env/templates/header.template @@ -202,7 +202,7 @@ <ul class="nav nav-sidebar menugroup"> <li><h3>Production</h3></li> <li><a href="CrawlStartExpert.html" class="MenuItemLink">Advanced Crawler</a></li> - <li><a href="IndexExport_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Index Export/Import</a></li> + <li><a href="IndexPackGenerator_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Index Export/Import</a></li> <li><a href="Vocabulary_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Content Semantic</a></li> <li><a href="CrawlCheck_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Target Analysis</a></li> </ul> diff --git a/htroot/env/templates/submenuCrawlMonitor.template b/htroot/env/templates/submenuCrawlMonitor.template index 4bbcff639..6da2ac3e1 100644 --- a/htroot/env/templates/submenuCrawlMonitor.template +++ b/htroot/env/templates/submenuCrawlMonitor.template @@ -45,7 +45,7 @@ <li><a href="CrawlResults.html?process=4" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">(4) Proxy Use</a></li> <li><a href="CrawlResults.html?process=5" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">(5) Local Crawling</a></li> #(navigation-p2p)#::<li><a href="CrawlResults.html?process=6" class="MenuItemLink">(6) Global Crawling</a></li>#(/navigation-p2p)# - <li><a href="CrawlResults.html?process=7" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">(7) Surrogate Import</a></li> + <li><a href="CrawlResults.html?process=7" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">(7) Pack Import</a></li> </ul> </div> </div>
\ No newline at end of file diff --git a/htroot/env/templates/submenuIndexImport.template b/htroot/env/templates/submenuIndexImport.template index ccbc49412..8df07e446 100644 --- a/htroot/env/templates/submenuIndexImport.template +++ b/htroot/env/templates/submenuIndexImport.template @@ -3,27 +3,34 @@ </div> <div class="SubMenu"> <div class="SubMenugroup"> + <h3>Index Packs</h3> + <ul class="SubMenu"> + <li><a href="IndexPackGenerator_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Index Pack Generator</a></li> + </ul> + </div> + <div class="SubMenugroup"> <h3>Export</h3> <ul class="SubMenu"> - <li><a href="IndexExport_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Internal Index Export</a></li> + <li><a href="IndexExport_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Index Export</a></li> + <li><a href="IndexExportImportSolr_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Solr Dump Export/Import</a></li> </ul> </div> <div class="SubMenugroup"> <h3>Import</h3> <ul class="SubMenu"> - <li><a href="Load_RSS_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">RSS Feed Importer</a></li> - <li><a href="IndexImportOAIPMH_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">OAI-PMH Importer</a></li> - <li><a href="IndexImportWarc_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Warc Importer</a></li> - <li><a href="IndexImportZim_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Zim Importer</a></li> - <li><a href="IndexImportJsonList_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">JsonList Importer</a></li> + <li><a href="Load_RSS_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">RSS</a></li> + <li><a href="IndexImportOAIPMH_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">OAI-PMH</a></li> + <li><a href="IndexImportWarc_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">WARC</a></li> + <li><a href="IndexImportZim_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">ZIM</a></li> + <li><a href="IndexImportJsonList_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">JsonList</a></li> </ul> </div> <div class="SubMenugroup"> <h3>Database Reader</h3> <ul class="SubMenu"> - <li><a href="ContentIntegrationPHPBB3_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Database Reader for phpBB3 Forums</a></li> - <li><a href="IndexImportMediawiki_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Dump Reader for MediaWiki dumps</a></li> + <li><a href="ContentIntegrationPHPBB3_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">phpBB3 Database</a></li> + <li><a href="IndexImportMediawiki_p.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">MediaWiki Dump</a></li> </ul> </div> </div>
\ No newline at end of file diff --git a/locales/de.lng b/locales/de.lng index ca7c241b2..054488687 100644 --- a/locales/de.lng +++ b/locales/de.lng @@ -1092,7 +1092,7 @@ above which also will show you a table with indexing results so far. The informa so you need to log-in with your administration password.==also müssen Sie sich mit Ihrem Administrator Passwort einloggen, um sie zu sehen. Case (6) is a monitor of the local receipt-generator, the opposed case of (1). It contains also an indexing result monitor but is not considered private==Fall (6) ist ein Monitor des lokalen Empfangs-Generator, der Gegensatz zu (1). Er enthält ausserdem einen Index Resulate Monitor, ist aber nicht privat, since it shows crawl requests from other peers.==da er Crawl Anfragen von anderen Peers zeigt. -Case (7) occurs if surrogate files are imported==Fall (7) tritt ein wenn Surrogat Dateien importiert werden. +Case (7) occurs if pack files are imported==Fall (7) tritt ein wenn Pack-Dateien importiert werden. The image above illustrates the data flow initiated by web index acquisition.==Das obige Bild zeigt den Datenfluss, der durch die Index Erwerbung über das Internet entsteht. Some processes occur double to document the complex index migration structure.==Einige Prozesse erscheinen doppelt, um die Komplexizität der Index Verteilungs Struktur zu erklären. (1) Results of Remote Crawl Receipts==(1) Ergebnisse der Remote Crawl Rückmeldungen @@ -1128,9 +1128,9 @@ This is the 'mirror'-case of process (1).==Dies ist der 'Gegensatz' Prozess zu ( <em>Use Case:</em> This list may fill if you check the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page==<em>Anwendung:</em> Diese Liste füllt sich, wenn Sie 'Akzeptiere Remote Crawl Anfragen' auf der '<a href="RemoteCrawl_p.html">Remote Crawling</a>' Seite aktiviert haben. The stack is empty.==Die Liste ist leer. Statistics about #[domains]# domains in this stack:==Statistiken über #[domains]# Domains in diesem Bereich: -(7) Results from surrogates import==(7) Ergebnisse aus dem Surrogat Import -These records had been imported from surrogate files in DATA/SURROGATES/in==Diese Datensätze wurden aus Surrogat Dateien in DATA/SURROGATES/in importiert -<em>Use Case:</em> place files with dublin core metadata content into DATA/SURROGATES/in or use an index import method==<em>Anwendungsfall:</em> Dateien mit Dublin Core Metadaten Inhalt in das DATA/SURROGATES/in kopieren oder eine der Index Import Funktionen nutzen +(7) Results from pack import==(7) Ergebnisse aus dem Pack Import +These records had been imported from pack files in DATA/PACKS/load==Diese Datensätze wurden aus Pack Dateien in DATA/PACKS/load importiert +<em>Use Case:</em> place files with dublin core metadata content into DATA/PACKS/load or use an index import method==<em>Anwendungsfall:</em> Dateien mit Dublin Core Metadaten Inhalt in das DATA/PACKS/load kopieren oder eine der Index Import Funktionen nutzen (i.e. <a href="IndexImportMediawiki_p.html">MediaWiki import</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH retrieval</a>)==(z.B. <a href="IndexImportMediawiki_p.html">MediaWiki Dump Import</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH Import</a>) #Domain==Domain #URLs=URLs @@ -1752,7 +1752,7 @@ Export File==Export-Datei #>query< #Export Format Full Data Records:==Komplette Datensätze -(Rich and full-text Solr data, one document per line in one large xml file, can be processed with shell tools, can be imported with DATA/SURROGATE/in/)==(vollständige Solr Daten, ein Dokument pro Zeile in einer grossen xml Datei, kann mit Kommandozeilentools bearbeitet werden, kann aus DATA/SURROGATE/in/ importiert werden) +(Rich and full-text Solr data, one document per line in one large xml file, can be processed with shell tools, can be imported with DATA/PACKS/load/)==(vollständige Solr Daten, ein Dokument pro Zeile in einer grossen xml Datei, kann mit Kommandozeilentools bearbeitet werden, kann aus DATA/PACKS/load/ importiert werden) #> XML (RSS)< Full URL List:==Liste mit vollständiger URL: Plain Text List (URLs only)==Einfache Text Liste (nur URLs) @@ -1765,7 +1765,7 @@ Fulltext of Search Index Text==Voller Text der Indexdokumente #"Export" Export to file #[exportfile]# is running .. #[urlcount]# Documents so far==Export nach Datei #[exportfile]# läuft .. #[urlcount]# Dokumente bisher Finished export of #[urlcount]# Documents to file==Export beendet und #[urlcount]# Dokumente gespeichert in Datei -Import this file by moving it to DATA/SURROGATES/in==Verschiebe diese Datei nach DATA/SURROGATES/in um sie zu importieren +Import this file by moving it to DATA/PACKS/load==Verschiebe diese Datei nach DATA/PACKS/load um sie zu importieren Export to file #[exportfile]# failed:==Export in Datei #[exportfile]# fehlgeschlagen: Dump and Restore of Solr Index==Dump und Wiederherstellen des Solr Index "Create Dump"=="Erstelle Dump" @@ -1784,8 +1784,8 @@ If you read from an imported database, here are some hints to get around problem before importing large database dumps, set==Bevor große Datenbankdumps importiert werden die folgende Zeile in phpmyadmin/config.inc.php and place your dump file in /tmp (Otherwise it is not possible to upload files larger than 2MB)==in die Datei phpmyadmin/config.inc.php schreiben und die Datenbank Datei in /tmp ablegen (Andernfalls ist es nicht möglich Dateien größer als 2MB hochzuladen) deselect the partial import flag==Den teilweisen Import Flag abschalten -When an export is started, surrogate files are generated into DATA/SURROGATE/in which are automatically fetched by an indexer thread.==Wenn ein Export gestartet wird werden Hilfsdateien in DATA/SURROGATE/in erzeugt, die automatisch von einem Indexer Thread geholt und verarbeitet werden. -All indexed surrogate files are then moved to DATA/SURROGATE/out and can be re-cycled when an index is deleted.==Alle indexierten Hilfsdateien werden dann nach DATA/SURROGATE/out verschoben und können recycelt werden wenn ein Index gelöscht wird. +When an export is started, pack files are generated into DATA/PACKS/load which are automatically fetched by an indexer thread.==Wenn ein Export gestartet wird werden Hilfsdateien in DATA/PACKS/load erzeugt, die automatisch von einem Indexer Thread geholt und verarbeitet werden. +All indexed pack files are then moved to DATA/PACKS/loaded and can be re-cycled when an index is deleted.==Alle indexierten Hilfsdateien werden dann nach DATA/PACKS/loaded verschoben und können recycelt werden wenn ein Index gelöscht wird. The URL stub==Der Teil der URL like https://community.searchlab.eu==wie z.B. https://community.searchlab.eu this must be the path right in front of '/viewtopic.php?'==dies muss den kompletten Pfad vor '/viewtopic.php?' enthalten @@ -1805,16 +1805,16 @@ that can access the database==mit Zugriff auf die Datenbank Password==Passwort for the account of that user given above==für den Zugang des oben angegebenen Benutzers Posts per file==Beiträge pro Datei -in exported surrogates==in der exportierten Hilfsdatei +in exported packs==in der exportierten Pack-Datei Check database connection==Datenbankverbindung überprüfen -Export Content to Surrogates==Exportiere Inhalt in Hilfsdateien +Export Content to Packs==Exportiere Inhalt in Pack-Dateien Import a database dump==Importieren eines Datenbankauszugs Import Dump==Datenbankdump importieren Posts in database==Beiträge in Datenbank first entry==Erster Eintrag last entry==Letzter Eintrag Info failed:==Info Fehlgeschlagen: -Export successful! Wrote #[files]# files in DATA/SURROGATES/in==Export erfolgreich! #[files]# Dateien in DATA/SURROGATES/in geschrieben +Export successful! Wrote #[files]# files in DATA/PACKS/load==Export erfolgreich! #[files]# Dateien in DATA/PACKS/load geschrieben Export failed:==Export fehlgeschlagen: Import successful!==Import erfolgreich! Import failed:==Import fehlgeschlagen: @@ -1986,11 +1986,11 @@ Dumps must be in XML format and may be compressed in gz or bz2. Place the file i "Import MediaWiki Dump"=="Importiere MediaWiki Dump" When the import is started, the following happens:==Wenn der Import gestartet wird passiert Folgendes: The dump is extracted on the fly and wiki entries are translated into Dublin Core data format. The output looks like this:==Der Dump wird zur Laufzeit extrahiert und die Wiki Einträge werden in das Dublin Core Datenformat übersetzt. Die Ausgabe schaut wie folgt aus: -Each 10000 wiki records are combined in one output file which is written to /DATA/SURROGATES/in into a temporary file.==Je 10000 Wiki Einträge werden zusammen in eine Ausgabedatei geschrieben und in /DATA/SURROGATES/in temporär gespeichert. +Each 10000 wiki records are combined in one output file which is written to /DATA/PACKS/load into a temporary file.==Je 10000 Wiki Einträge werden zusammen in eine Ausgabedatei geschrieben und in /DATA/PACKS/load temporär gespeichert. When each of the generated output file is finished, it is renamed to a .xml file==Wenn jede der generierten Ausgabedateien abgearbeitet wurde wird diese in eine .xml Datei umbenannt. -Each time a xml surrogate file appears in /DATA/SURROGATES/in, the YaCy indexer fetches the file and indexes the record entries.==Immer wenn eine xml Surrogat Datei in /DATA/SURROGATES/in erscheint, holt der YaCy Indexer diese Datei und indexiert die enthaltenen Datensätze. -When a surrogate file is finished with indexing, it is moved to /DATA/SURROGATES/out==Wenn eine Surrogat Datei vollständig indexiert wurde, wird sie nach /DATA/SURROGATES/out verschoben -You can recycle processed surrogate files by moving them from /DATA/SURROGATES/out to /DATA/SURROGATES/in==Sie können schon abgearbeitete Surrogat Dateien durch Verschieben von /DATA/SURROGATES/out nach /DATA/SURROGATES/in recyclen. +Each time a xml pack file appears in /DATA/PACKS/load, the YaCy indexer fetches the file and indexes the record entries.==Immer wenn eine xml Pack Datei in /DATA/PACKS/load erscheint, holt der YaCy Indexer diese Datei und indexiert die enthaltenen Datensätze. +When a pack file is finished with indexing, it is moved to /DATA/PACKS/loaded==Wenn eine Pack-Datei vollständig indexiert wurde, wird sie nach /DATA/PACKS/loaded verschoben +You can recycle processed pack files by moving them from /DATA/PACKS/loaded to /DATA/PACKS/load==Sie können schon abgearbeitete Pack-Dateien durch Verschieben von /DATA/PACKS/loaded nach /DATA/PACKS/load recyclen. Import Process==Import Prozess #Thread:==Thread: #Dump:==Dump: @@ -2009,7 +2009,7 @@ Remaining Time:==Verbleibende Zeit: #File: IndexImportOAIPMH_p.html #--------------------------- #OAI-PMH Import==OAI-PMH Import -Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for surrogates==Ergebnisse aus dem Import finden Sie auf der Seite <a href="CrawlResults.html?process=7">Ergebnisse aus dem Surrogat Import +Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for packs==Ergebnisse aus dem Import finden Sie auf der Seite <a href="CrawlResults.html?process=7">Ergebnisse aus dem Pack-Datei Import Single request import==Einfacher Anfrage Import This will submit only a single request as given here to a OAI-PMH server and imports records into the index==Mit dieser Methode wird nur eine einzelne Abfrage an einen OAI-PMH Server geschickt und die Datensätze in den Index aufgenommen "Import OAI-PMH source"=="Importiere OAI-PMH Quelle" @@ -3878,7 +3878,7 @@ DHT Transfer==DHT-Verteilung Proxy Use==Proxy Benutzung Local Crawling</a>==Lokales Crawlen</a> Global Crawling</a>==Globales Crawlen</a> -Surrogate Import==Surrogat Import +Pack Import==Pack Import Processing Monitor==Prozess Monitor Loader<==Lade Puffer< Rejected URLs==Abgelehnte URLs diff --git a/locales/hi.lng b/locales/hi.lng index 04f9bcccc..ce52843ea 100644 --- a/locales/hi.lng +++ b/locales/hi.lng @@ -877,7 +877,7 @@ DHT Transfer==DHT के ट्रांसफर Proxy Use==प्रॉक्सी उपयोग Local Crawling</a>==लोकल क्रॉलिंग</a> Global Crawling</a>==ग्लोबल क्रॉलिंग</a> -Surrogate Import</a>==सरोगेट इम्पोर्ट</a> +Pack Import</a>==सरोगेट इम्पोर्ट</a> >Crawl Results Overview<==>क्रॉल का परिणाम ओवरव्यू< These are monitoring pages for the different indexing queues.==ये अलग अनुक्रमण कतारों के लिए पृष्ठों की निगरानी कर रहे हैं. YaCy knows 5 different ways to acquire web indexes. The details of these processes (1-5) are described within the submenu's listed==YaCy वेब अनुक्रमित प्राप्त करने के लिए 5 अलग अलग तरीकों से जानता है. इन प्रक्रियाओं का ब्यौरा (1-5 ) सबमेनू सूचीबद्ध भीतर वर्णित हैं @@ -885,7 +885,7 @@ above which also will show you a table with indexing results so far. The informa so you need to log-in with your administration password.==इसलिए आप लॉग इन करने के लिए अपने प्रशासन पासवर्ड के साथ की जरूरत. Case (6) is a monitor of the local receipt-generator, the opposed case of (1). It contains also an indexing result monitor but is not considered private==प्रकरण (6 ) स्थानीय रसीद जनरेटर, (1 ) का विरोध करने के मामले की निगरानी है. यह भी एक अनुक्रमण परिणाम की निगरानी शामिल है, लेकिन निजी विचार नहीं किया है since it shows crawl requests from other peers.==यह अन्य साथियों से क्रॉल अनुरोधों से पता चलता है -Case (7) occurs if surrogate files are imported==किराए की फ़ाइलें आयात कर रहे हैं यदि प्रकरण (7 ) होता है +Case (7) occurs if pack files are imported==किराए की फ़ाइलें आयात कर रहे हैं यदि प्रकरण (7 ) होता है The image above illustrates the data flow initiated by web index acquisition.==छवि ऊपर वेब सूचकांक अधिग्रहण द्वारा शुरू डाटा प्रवाह दिखाता है. Some processes occur double to document the complex index migration structure.==कुछ प्रक्रियाओं को जटिल सूचकांक प्रवास संरचना करने के लिए दस्तावेज़ डबल घटित. (1) Results of Remote Crawl Receipts==(1 ) रिमोट क्रॉल प्राप्तियों के का परिणाम @@ -921,9 +921,9 @@ This is the 'mirror'-case of process (1).==इस प्रक्रिया <em>Use Case:</em> This list may fill if you check the 'Accept Remote Crawl Requests'-flag on the 'Index Crate' page==<em> प्रयोग करें प्रकरण: </ em> आप जाँच अगर इस सूची में भर सकते हैं 'स्वीकार रिमोट पर requests' झंडा रेंगने' सूचकांक टोकरा 'पेज The stack is empty.==ढेर खाली है. Statistics about #[domains]# domains in this stack:==इस ढेर में # [डोमेन ] # डोमेन के बारे में सांख्यिकी: -(7) Results from surrogates import==(7) Surrogates के आयात से का परिणाम -These records had been imported from surrogate files in DATA/SURROGATES/in==ये रिकॉर्ड आंकड़े / surrogates में / में सरोगेट फ़ाइलों से आयात किया गया था -<em>Use Case:</em> place files with dublin core metadata content into DATA/SURROGATES/in or use an index import method==<em> प्रयोग करें प्रकरण: </ em> डबलिन कोर मेटाडेटा सामग्री के साथ जगह फाइलें आंकड़े / surrogates में / में या एक सूचकांक आयात विधि का उपयोग +(7) Results from pack import==(7) Packs के आयात से का परिणाम +These records had been imported from pack files in DATA/PACKS/load==ये रिकॉर्ड आंकड़े / packs में / में सरोगेट फ़ाइलों से आयात किया गया था +<em>Use Case:</em> place files with dublin core metadata content into DATA/PACKS/load or use an index import method==<em> प्रयोग करें प्रकरण: </ em> डबलिन कोर मेटाडेटा सामग्री के साथ जगह फाइलें आंकड़े / packs में / में या एक सूचकांक आयात विधि का उपयोग (i.e. <a href="IndexImportMediawiki_p.html">MediaWiki import</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH retrieval</a>)== (यानी <a href="IndexImportMediawiki_p.html"> साँचा आयात </ a> <a href="IndexImportOAIPMH_p.html"> OAI-PMH पुनर्प्राप्ति </ a> ) #Domain==डोमेन #URLs=यूआरएल diff --git a/locales/master.lng.xlf b/locales/master.lng.xlf index 25bebd73c..0e347ff73 100644 --- a/locales/master.lng.xlf +++ b/locales/master.lng.xlf @@ -2453,10 +2453,10 @@ <source>deselect the partial import flag</source> </trans-unit> <trans-unit id="e5986233" xml:space="preserve" approved="no" translate="yes"> - <source>When an export is started, surrogate files are generated into DATA/SURROGATE/in which are automatically fetched by an indexer thread.</source> + <source>When an export is started, pack files are generated into DATA/PACKS/load which are automatically fetched by an indexer thread.</source> </trans-unit> <trans-unit id="7f68bfb2" xml:space="preserve" approved="no" translate="yes"> - <source>All indexed surrogate files are then moved to DATA/SURROGATE/out and can be re-cycled when an index is deleted.</source> + <source>All indexed pack files are then moved to DATA/PACKS/loaded and can be re-cycled when an index is deleted.</source> </trans-unit> <trans-unit id="dfe68bce" xml:space="preserve" approved="no" translate="yes"> <source>The URL stub</source> @@ -2495,13 +2495,13 @@ <source><b>Password</b> for the account of that user given above</source> </trans-unit> <trans-unit id="777c2e3" xml:space="preserve" approved="no" translate="yes"> - <source><b>Posts per file</b><br />in exported surrogates</source> + <source><b>Posts per file</b><br />in exported packs</source> </trans-unit> <trans-unit id="eb1fd76b" xml:space="preserve" approved="no" translate="yes"> <source>Check database connection</source> </trans-unit> <trans-unit id="febcecad" xml:space="preserve" approved="no" translate="yes"> - <source>Export Content to Surrogates</source> + <source>Export Content to Packs</source> </trans-unit> <trans-unit id="7a28359f" xml:space="preserve" approved="no" translate="yes"> <source>Import a database dump</source> @@ -2522,7 +2522,7 @@ <source>Info failed:</source> </trans-unit> <trans-unit id="62f6028b" xml:space="preserve" approved="no" translate="yes"> - <source>Export successful! Wrote #[files]# files in DATA/SURROGATES/in</source> + <source>Export successful! Wrote #[files]# files in DATA/PACKS/load</source> </trans-unit> <trans-unit id="4e2c7b51" xml:space="preserve" approved="no" translate="yes"> <source>Export failed:</source> @@ -2579,7 +2579,7 @@ <source>Posts per file</source> </trans-unit> <trans-unit id="142e0c0d" xml:space="preserve" approved="no" translate="yes"> - <source>in exported surrogates</source> + <source>in exported packs</source> </trans-unit> </body> </file> @@ -2846,7 +2846,7 @@ <source>since it shows crawl requests from other peers.</source> </trans-unit> <trans-unit id="51121b61" xml:space="preserve" approved="no" translate="yes"> - <source>Case (7) occurs if surrogate files are imported</source> + <source>Case (7) occurs if pack files are imported</source> </trans-unit> <trans-unit id="3953b093" xml:space="preserve" approved="no" translate="yes"> <source>The image above illustrates the data flow initiated by web index acquisition.</source> @@ -2951,13 +2951,13 @@ <source>Statistics about #[domains]# domains in this stack:</source> </trans-unit> <trans-unit id="e2b69744" xml:space="preserve" approved="no" translate="yes"> - <source>(7) Results from surrogates import</source> + <source>(7) Results from pack import</source> </trans-unit> <trans-unit id="742eeb80" xml:space="preserve" approved="no" translate="yes"> - <source>These records had been imported from surrogate files in DATA/SURROGATES/in</source> + <source>These records had been imported from pack files in DATA/PACKS/load</source> </trans-unit> <trans-unit id="dc86ab52" xml:space="preserve" approved="no" translate="yes"> - <source><em>Use Case:</em> place files with dublin core metadata content into DATA/SURROGATES/in or use an index import method</source> + <source><em>Use Case:</em> place files with dublin core metadata content into DATA/PACKS/load or use an index import method</source> </trans-unit> <trans-unit id="3f1a6e99" xml:space="preserve" approved="no" translate="yes"> <source>(i.e. <a href="IndexImportMediawiki_p.html">MediaWiki import</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH retrieval</a>)</source> @@ -4751,7 +4751,7 @@ <source>Full Data Records:</source> </trans-unit> <trans-unit id="aa903601" xml:space="preserve" approved="no" translate="yes"> - <source>(Rich and full-text Solr data, one document per line in one large xml file, can be processed with shell tools, can be imported with DATA/SURROGATE/in/)</source> + <source>(Rich and full-text Solr data, one document per line in one large xml file, can be processed with shell tools, can be imported with DATA/PACKS/load/)</source> </trans-unit> <trans-unit id="Line0038" xml:space="preserve" approved="no" translate="yes"> <source>(Rich and full-text Elasticsearch data, one document per line in one flat JSON file, can be bulk-imported to elasticsearch with the command "curl -XPOST localhost:9200/collection1/yacy/_bulk --data-binary @yacy_dump_XXX.flatjson")</source> @@ -4787,7 +4787,7 @@ <source>Finished export of #[urlcount]# Documents to file</source> </trans-unit> <trans-unit id="1bfecfea" xml:space="preserve" approved="no" translate="yes"> - <source>Import this file by moving it to DATA/SURROGATES/in</source> + <source>Import this file by moving it to DATA/PACKS/load</source> </trans-unit> <trans-unit id="64110976" xml:space="preserve" approved="no" translate="yes"> <source>Export to file #[exportfile]# failed:</source> @@ -4947,19 +4947,19 @@ <source>The dump is extracted on the fly and wiki entries are translated into Dublin Core data format. The output looks like this:</source> </trans-unit> <trans-unit id="d504cb55" xml:space="preserve" approved="no" translate="yes"> - <source>Each 10000 wiki records are combined in one output file which is written to /DATA/SURROGATES/in into a temporary file.</source> + <source>Each 10000 wiki records are combined in one output file which is written to /DATA/PACKS/load into a temporary file.</source> </trans-unit> <trans-unit id="b61c5bd1" xml:space="preserve" approved="no" translate="yes"> <source>When each of the generated output file is finished, it is renamed to a .xml file</source> </trans-unit> <trans-unit id="4508732e" xml:space="preserve" approved="no" translate="yes"> - <source>Each time a xml surrogate file appears in /DATA/SURROGATES/in, the YaCy indexer fetches the file and indexes the record entries.</source> + <source>Each time a xml pack file appears in /DATA/PACKS/load, the YaCy indexer fetches the file and indexes the record entries.</source> </trans-unit> <trans-unit id="e0a572b8" xml:space="preserve" approved="no" translate="yes"> - <source>When a surrogate file is finished with indexing, it is moved to /DATA/SURROGATES/out</source> + <source>When a pack file is finished with indexing, it is moved to /DATA/PACKS/loaded</source> </trans-unit> <trans-unit id="b60d0ab2" xml:space="preserve" approved="no" translate="yes"> - <source>You can recycle processed surrogate files by moving them from /DATA/SURROGATES/out to /DATA/SURROGATES/in</source> + <source>You can recycle processed pack files by moving them from /DATA/PACKS/loaded to /DATA/PACKS/load</source> </trans-unit> <trans-unit id="278c1cf4" xml:space="preserve" approved="no" translate="yes"> <source>Import Process</source> @@ -5038,7 +5038,7 @@ <source>OAI-PMH Import</source> </trans-unit> <trans-unit id="55381611" xml:space="preserve" approved="no" translate="yes"> - <source>Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for surrogates</source> + <source>Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for packs</source> </trans-unit> <trans-unit id="52d1b50e" xml:space="preserve" approved="no" translate="yes"> <source>Single request import</source> @@ -10238,7 +10238,7 @@ <source>Global Crawling</a></source> </trans-unit> <trans-unit id="16f4722d" xml:space="preserve" approved="no" translate="yes"> - <source>Surrogate Import</source> + <source>Pack Import</source> </trans-unit> <trans-unit id="ffd9ecbd" xml:space="preserve" approved="no" translate="yes"> <source>Crawl Results</source> diff --git a/locales/ru.lng b/locales/ru.lng index 34d1c1dd6..9b782f49a 100644 --- a/locales/ru.lng +++ b/locales/ru.lng @@ -1178,7 +1178,7 @@ above which also will show you a table with indexing results so far. The informa so you need to log-in with your administration password.==то потребуется ввести пароль администратора. Case (6) is a monitor of the local receipt-generator, the opposed case of (1). It contains also an indexing result monitor but is not considered private==Монитор (6) показывает локальный приём индекса, в отличие от монитора (1). Здесь также показываются результаты индексирования не содержащие личные данные, since it shows crawl requests from other peers.==а только запросы индексаторов от других узлов. -Case (7) occurs if surrogate files are imported==Монитор (7) показывает замещающий импорт файлов +Case (7) occurs if pack files are imported==Монитор (7) показывает замещающий импорт файлов The image above illustrates the data flow initiated by web index acquisition.==На изображении выше показан поток данных, инициированный вэб-индексом. Some processes occur double to document the complex index migration structure.==Некоторые процессы повторяются дважды. (1) Results of Remote Crawl Receipts==(1) Получение результатов удалённого индексирования @@ -1214,9 +1214,9 @@ This is the 'mirror'-case of process (1).==Это противоположнос <em>Use Case:</em> This list may fill if you check the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page==Этот список будет пополняться, если вы включите "Удалённое индексирование" на странице '<a href="RemoteCrawl_p.html">Удалённое индексирование</a>'. The stack is empty.==Список пуст. Statistics about #[domains]# domains in this stack:==В этой таблице данные о #[domains]# доменах: -(7) Results from surrogates import==(7) Результаты замещающего импорта -These records had been imported from surrogate files in DATA/SURROGATES/in==Эти данные импортированы из замещающих файлов в DATA/SURROGATES/in -<em>Use Case:</em> place files with dublin core metadata content into DATA/SURROGATES/in or use an index import method==Поместите файлы с метаданными контента стандарта дублинского ядра в DATA/SURROGATES/in или используйте способ импортирования индекса +(7) Results from pack import==(7) Результаты замещающего импорта +These records had been imported from pack files in DATA/PACKS/load==Эти данные импортированы из замещающих файлов в DATA/PACKS/load +<em>Use Case:</em> place files with dublin core metadata content into DATA/PACKS/load or use an index import method==Поместите файлы с метаданными контента стандарта дублинского ядра в DATA/PACKS/load или используйте способ импортирования индекса (i.e. <a href="IndexImportMediawiki_p.html">MediaWiki import</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH retrieval</a>)==(например, <a href="IndexImportMediawiki_p.html">импорт MediaWiki дампов</a>, <a href="IndexImportOAIPMH_p.html">импорт OAI-PMH</a>) >Domain==>Домен #>URLs==>Ссылки @@ -1911,8 +1911,8 @@ If you read from an imported database, here are some hints to get around problem before importing large database dumps, set==перед импортированием большого дампа базы данных, установите the following Line in phpmyadmin/config.inc.php and place your dump file in /tmp (Otherwise it is not possible to upload files larger than 2MB)==указанную ниже строку в phpmyadmin/config.inc.php и разместите ваш файл дампа в директории /tmp (иначе будет невозможна загрузка файлов размером больше 2МБайт) deselect the partial import flag==снимите флаг частичного импорта -When an export is started, surrogate files are generated into DATA/SURROGATE/in which are automatically fetched by an indexer thread.==После начала экспорта, замещающие файлы создаются в DATA/SURROGATE/in и автоматически добавляются в индексатор. -All indexed surrogate files are then moved to DATA/SURROGATE/out and can be re-cycled when an index is deleted.==Все проиндексированные замещающие файлы перемещаются в DATA/SURROGATE/out и могут быть использованы повторно в случае удаления индекса. +When an export is started, pack files are generated into DATA/PACKS/load which are automatically fetched by an indexer thread.==После начала экспорта, замещающие файлы создаются в DATA/PACKS/load и автоматически добавляются в индексатор. +All indexed pack files are then moved to DATA/PACKS/loaded and can be re-cycled when an index is deleted.==Все проиндексированные замещающие файлы перемещаются в DATA/PACKS/loaded и могут быть использованы повторно в случае удаления индекса. The URL stub==Часть ссылки like https://community.searchlab.eu==например, https://community.searchlab.eu this must be the path right in front of '/viewtopic.php?'==(путь до '/viewtopic.php?') @@ -1925,16 +1925,16 @@ use either 'mysql' or 'pgsql'==используйте 'mysql' или 'pgsql' <b>Table prefix string</b> for table names==Префикс для таблицы имён <b>User</b> that can access the database==Пользователь базы данных <b>Password</b> for the account of that user given above==Пароль от учётной записи пользователя -<b>Posts per file</b><br />in exported surrogates==Постов на файл в экспортированных замещающих файлах +<b>Posts per file</b><br />in exported packs==Постов на файл в экспортированных замещающих файлах Check database connection==Проверить соединение -Export Content to Surrogates==Экспортировать содержимое +Export Content to Packs==Экспортировать содержимое <b>Import a database dump</b>,<br />==Импортировать дамп базы данных Import Dump==Импортировать Posts in database==Постов в базе данных first entry==первая запись last entry==последняя запись Info failed:==Получение информации неудачно: -Export successful! Wrote #[files]# files in DATA/SURROGATES/in==Экспорт успешно выполнен! #[files]# файлов записано в DATA/SURROGATES/in +Export successful! Wrote #[files]# files in DATA/PACKS/load==Экспорт успешно выполнен! #[files]# файлов записано в DATA/PACKS/load Export failed:==Экспорт не был выполнен: Import successful!==Импорт успешно выполнен! Import failed:==Импорт не был выполнен: @@ -2092,11 +2092,11 @@ Dumps must be in XML format and may be compressed in gz or bz2. Place the file i "Import MediaWiki Dump"=="Импортировать дамп MediaWiki" When the import is started, the following happens:==После начала импорта произойдет следующее: The dump is extracted on the fly and wiki entries are translated into Dublin Core data format. The output looks like this:==Дамп извлекается "на лету" и значения wiki переводятся в формат дублинского ядра. Результат выглядит примерно так: -Each 10000 wiki records are combined in one output file which is written to /DATA/SURROGATES/in into a temporary file.==Каждые 10000 записей Wiki формируются в один временный файл, который размещается в /DATA/SURROGATES/in. +Each 10000 wiki records are combined in one output file which is written to /DATA/PACKS/load into a temporary file.==Каждые 10000 записей Wiki формируются в один временный файл, который размещается в /DATA/PACKS/load. When each of the generated output file is finished, it is renamed to a .xml file==После завершения импортирования полученный файл переименовывается в .xml. -Each time a xml surrogate file appears in /DATA/SURROGATES/in, the YaCy indexer fetches the file and indexes the record entries.==Каждый раз xml-файл появляется в /DATA/SURROGATES/in, а YaCy индексирует полученный файл. -When a surrogate file is finished with indexing, it is moved to /DATA/SURROGATES/out==После завершения индексирования файл перемещается в /DATA/SURROGATES/out. -You can recycle processed surrogate files by moving them from /DATA/SURROGATES/out to /DATA/SURROGATES/in==Вы можете повторно использовать замещающие файлы для индексирования, переместив их из /DATA/SURROGATES/out в /DATA/SURROGATES/in. +Each time a xml pack file appears in /DATA/PACKS/load, the YaCy indexer fetches the file and indexes the record entries.==Каждый раз xml-файл появляется в /DATA/PACKS/load, а YaCy индексирует полученный файл. +When a pack file is finished with indexing, it is moved to /DATA/PACKS/loaded==После завершения индексирования файл перемещается в /DATA/PACKS/loaded. +You can recycle processed pack files by moving them from /DATA/PACKS/loaded to /DATA/PACKS/load==Вы можете повторно использовать замещающие файлы для индексирования, переместив их из /DATA/PACKS/loaded в /DATA/PACKS/load. Import Process==Выполнение импорта Thread:==Поток: Dump:==Дамп: @@ -2115,7 +2115,7 @@ Remaining Time:==Осталось времени: #File: IndexImportOAIPMH_p.html #--------------------------- OAI-PMH Import==Импорт OAI-PMH -Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for surrogates==Результаты импорта можно увидеть на странице <a href="CrawlResults.html?process=7">результатов индексации замещающего импорта. +Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for packs==Результаты импорта можно увидеть на странице <a href="CrawlResults.html?process=7">результатов индексации замещающего импорта. Single request import==Один запрос импорта This will submit only a single request as given here to a OAI-PMH server and imports records into the index==Это позволит отправить только один запрос на сервер OAI-PMH и импортировать записи в индекс "Import OAI-PMH source"=="Импортировать источник OAI-PMH" @@ -4249,7 +4249,7 @@ DHT Transfer==DHT-передача Proxy Use==Использование прокси Local Crawling</a>==Локальное индексирование</a> Global Crawling</a>==Глобальное индексирование</a> -Surrogate Import</a>==Замещающий импорт</a> +Pack Import</a>==Замещающий импорт</a> Crawl Results==Результаты индексирования Processing Monitor==Монитор процессов Crawler Queues==Очереди индексирования diff --git a/locales/tr.lng b/locales/tr.lng index 7c0957d80..b0a6d9619 100644 --- a/locales/tr.lng +++ b/locales/tr.lng @@ -1104,7 +1104,7 @@ above which also will show you a table with indexing results so far. The informa so you need to log-in with your administration password.==Bu tablolardaki bilgiler özel kabul edildiği için yönetim şifrenizle giriş yapmanız gerekiyor. Case (6) is a monitor of the local receipt-generator, the opposed case of (1). It contains also an indexing result monitor but is not considered private==Durum (6), yerel alım üreticisinin bir izleyicisidir, (1) durumunun zıttı. Ayrıca bir indeksleme sonuç izleyici içerir, ancak özel kabul edilmez, since it shows crawl requests from other peers.==çünkü başka eşlerden gelen tarama isteklerini gösterir. -Case (7) occurs if surrogate files are imported==Durum (7) ortaya çıkar eğer surrogat dosyaları içe aktarılıyorsa. +Case (7) occurs if pack files are imported==Durum (7) ortaya çıkar eğer pack dosyaları içe aktarılıyorsa. The image above illustrates the data flow initiated by web index acquisition.==Yukarıdaki resim, web indeksi edinimi tarafından başlatılan veri akışını göstermektedir. Some processes occur double to document the complex index migration structure.==Bazı süreçler karmaşık indeks göç yapısını belgelemek için çift gerçekleşir. (1) Results of Remote Crawl Receipts==(1) Uzaktan Tarama Alındı Sonuçları @@ -1139,9 +1139,9 @@ This is the 'mirror'-case of process (1). == Bu, işlem (1)'in 'ayna' durumudur. <em>Use Case:</em> This list may fill if you check the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page == <em>Kullanım Durumu:</em> Bu liste, '<a href="RemoteCrawl_p.html">Uzaktan Tarama</a>' sayfasındaki 'Uzaktan Tarama İsteklerini Kabul Et' bayrağını işaretlerseniz dolabilir. The stack is empty. == Yığın boş. Statistics about #[domains]# domains in this stack: == Bu yığındaki #[domains]# alanlarıyla ilgili istatistikler: -(7) Results from surrogates import == (7) Surrogat İçe Aktarma Sonuçları -These records had been imported from surrogate files in DATA/SURROGATES/in == Bu kayıtlar, DATA/SURROGATES/in klasöründeki surrogat dosyalarından içe aktarılmıştır. -<em>Use Case:</em> place files with dublin core metadata content into DATA/SURROGATES/in or use an index import method == <em>Kullanım Durumu:</em> Dublin Core meta veri içeriğine sahip dosyaları DATA/SURROGATES/in klasörüne yerleştirin veya bir indeks içe aktarma yöntemi kullanın +(7) Results from pack import == (7) Packs İçe Aktarma Sonuçları +These records had been imported from pack files in DATA/PACKS/load == Bu kayıtlar, DATA/PACKS/load klasöründeki pack dosyalarından içe aktarılmıştır. +<em>Use Case:</em> place files with dublin core metadata content into DATA/PACKS/load or use an index import method == <em>Kullanım Durumu:</em> Dublin Core meta veri içeriğine sahip dosyaları DATA/PACKS/load klasörüne yerleştirin veya bir indeks içe aktarma yöntemi kullanın (i.e. <a href="IndexImportMediawiki_p.html">MediaWiki import</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH retrieval</a>) == (örneğin, <a href="IndexImportMediawiki_p.html">MediaWiki İçe Aktarma</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH İndirme</a>) #Domain == Alan #URLs == URL'ler @@ -1761,7 +1761,7 @@ Export File==Dışa Aktarma Dosyası #>query< #Export Format Full Data Records:==Tam Veri Kayıtları -(Rich and full-text Solr data, one document per line in one large xml file, can be processed with shell tools, can be imported with DATA/SURROGATE/in/)==(Zengin ve tam metin Solr verileri, bir belge satırı başına bir büyük xml dosyasında, kabuk araçları ile işlenebilir, DATA/SURROGATE/in/ ile içe aktarılabilir) +(Rich and full-text Solr data, one document per line in one large xml file, can be processed with shell tools, can be imported with DATA/PACKS/load/)==(Zengin ve tam metin Solr verileri, bir belge satırı başına bir büyük xml dosyasında, kabuk araçları ile işlenebilir, DATA/PACKS/load/ ile içe aktarılabilir) #> XML (RSS)< Full URL List:==Tam URL Listesi: Plain Text List (URLs only)==Düz Metin Listesi (Yalnızca URL'ler) @@ -1774,7 +1774,7 @@ Fulltext of Search Index Text==Arama İndeksi Metni'nin Tamamı #"Export" Export to file #[exportfile]# is running .. #[urlcount]# Documents so far==Dosyaya #[exportfile]# dışa aktarılıyor .. Şu ana kadar #[urlcount]# Belge Finished export of #[urlcount]# Documents to file==Dosyaya #[urlcount]# belgenin dışa aktarımı tamamlandı -Import this file by moving it to DATA/SURROGATES/in==Bu dosyayı DATA/SURROGATES/in dizinine taşıyarak içe aktarın +Import this file by moving it to DATA/PACKS/load==Bu dosyayı DATA/PACKS/load dizinine taşıyarak içe aktarın Export to file #[exportfile]# failed:==Dosyaya #[exportfile]# dışa aktarım başarısız oldu: Dump and Restore of Solr Index==Solr İndeksinin Dökümü ve Geri Yüklenmesi "Create Dump"=="Döküm Oluştur" @@ -1793,8 +1793,8 @@ If you read from an imported database, here are some hints to get around problem before importing large database dumps, set==Büyük veritabanı dökümlerini içe aktarmadan önce, şu satırı ayarlayın in phpmyadmin/config.inc.php and place your dump file in /tmp (Otherwise it is not possible to upload files larger than 2MB)==phpmyadmin/config.inc.php dosyasına ve döküm dosyanızı /tmp dizinine yerleştirin (Aksi takdirde 2 MB'den büyük dosyalar yüklenemez) deselect the partial import flag==Kısmi içe aktarma bayrağını kaldırın -When an export is started, surrogate files are generated into DATA/SURROGATE/in which are automatically fetched by an indexer thread.==Bir dışa aktarma başlatıldığında, yedek dosyalar DATA/SURROGATE/in dizinine oluşturulur ve otomatik olarak bir dizin thread tarafından alınır. -All indexed surrogate files are then moved to DATA/SURROGATE/out and can be re-cycled when an index is deleted.==Tüm dizinlenmiş yedek dosyalar daha sonra DATA/SURROGATE/out'a taşınır ve bir dizin silindiğinde yeniden döngüye alınabilir. +When an export is started, pack files are generated into DATA/PACKS/load which are automatically fetched by an indexer thread.==Bir dışa aktarma başlatıldığında, yedek dosyalar DATA/PACKS/load dizinine oluşturulur ve otomatik olarak bir dizin thread tarafından alınır. +All indexed pack files are then moved to DATA/PACKS/loaded and can be re-cycled when an index is deleted.==Tüm dizinlenmiş yedek dosyalar daha sonra DATA/PACKS/loaded'a taşınır ve bir dizin silindiğinde yeniden döngüye alınabilir. The URL stub==URL parçası like https://community.searchlab.eu==örneğin https://community.searchlab.eu this must be the path right in front of '/viewtopic.php?'==bu, '/viewtopic.php?' hemen önündeki yol olmalıdır @@ -1814,16 +1814,16 @@ that can access the database==veritabanına erişebilen Password==Parola for the account of that user given above==yukarıda belirtilen kullanıcının hesabı için Posts per file==Dosya başına gönderiler -in exported surrogates==dışa aktarılan yedek dosyalarda +in exported packs==dışa aktarılan yedek dosyalarda Check database connection==Veritabanı bağlantısını kontrol et -Export Content to Surrogates==İçeriği Yedek Dosyalara Aktar +Export Content to Packs==İçeriği Yedek Dosyalara Aktar Import a database dump==Bir veritabanı dökümü içe aktar Import Dump==Dökümü İçe Aktar Posts in database==Veritabanındaki Gönderiler first entry==ilk giriş last entry==son giriş Info failed:==Bilgi başarısız oldu: -Export successful! Wrote #[files]# files in DATA/SURROGATES/in==Dışa aktarma başarılı! DATA/SURROGATES/in dizininde #[files]# dosya yazıldı +Export successful! Wrote #[files]# files in DATA/PACKS/load==Dışa aktarma başarılı! DATA/PACKS/load dizininde #[files]# dosya yazıldı Export failed:==Dışa aktarma başarısız oldu: Import successful!==İçe aktarma başarılı! Import failed:==İçe aktarma başarısız oldu: @@ -1994,11 +1994,11 @@ Dumps must be in XML format and may be compressed in gz or bz2. Place the file i "Import MediaWiki Dump"=="MediaWiki Dökümü İçe Aktar" When the import is started, the following happens:==İçe aktarma başladığında şu işlemler gerçekleşir: The dump is extracted on the fly and wiki entries are translated into Dublin Core data format. The output looks like this:==Döküm uçarken çıkarılır ve wiki girişleri Dublin Core veri formatına çevrilir. Çıktı şu şekildedir: -Each 10000 wiki records are combined in one output file which is written to /DATA/SURROGATES/in into a temporary file.==Her 10000 wiki kaydı, geçici bir dosyaya /DATA/SURROGATES/in yazılan bir çıkış dosyasında birleştirilir. +Each 10000 wiki records are combined in one output file which is written to /DATA/PACKS/load into a temporary file.==Her 10000 wiki kaydı, geçici bir dosyaya /DATA/PACKS/load yazılan bir çıkış dosyasında birleştirilir. When each of the generated output file is finished, it is renamed to a .xml file==Her bir oluşturulan çıkış dosyası tamamlandığında, .xml dosyasına yeniden adlandırılır. -Each time a xml surrogate file appears in /DATA/SURROGATES/in, the YaCy indexer fetches the file and indexes the record entries.==Her bir xml yedek dosyası /DATA/SURROGATES/in içinde göründüğünde, YaCy dizini dosyayı alır ve kayıt girişlerini indeksler. -When a surrogate file is finished with indexing, it is moved to /DATA/SURROGATES/out==Bir yedek dosya indeksleme ile tamamlandığında, /DATA/SURROGATES/out dizinine taşınır -You can recycle processed surrogate files by moving them from /DATA/SURROGATES/out to /DATA/SURROGATES/in==İşlenmiş yedek dosyaları /DATA/SURROGATES/out'dan /DATA/SURROGATES/in'a taşıyarak geri dönüştürebilirsiniz. +Each time a xml pack file appears in /DATA/PACKS/load, the YaCy indexer fetches the file and indexes the record entries.==Her bir xml yedek dosyası /DATA/PACKS/load içinde göründüğünde, YaCy dizini dosyayı alır ve kayıt girişlerini indeksler. +When a pack file is finished with indexing, it is moved to /DATA/PACKS/loaded==Bir yedek dosya indeksleme ile tamamlandığında, /DATA/PACKS/loaded dizinine taşınır +You can recycle processed pack files by moving them from /DATA/PACKS/loaded to /DATA/PACKS/load==İşlenmiş yedek dosyaları /DATA/PACKS/loaded'dan /DATA/PACKS/load'a taşıyarak geri dönüştürebilirsiniz. Import Process==İçe Aktarma Süreci #Thread:==#İş Parçacığı: #Dump:==#Döküm: @@ -2017,7 +2017,7 @@ Remaining Time:==Kalan Zaman: #File: IndexImportOAIPMH_p.html #--------------------------- #OAI-PMH Import==OAI-PMH İçe Aktarma -Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for surrogates==İçe aktarmadan gelen sonuçlar, <a href="CrawlResults.html?process=7">yedekler için dizinleme sonuçları +Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for packs==İçe aktarmadan gelen sonuçlar, <a href="CrawlResults.html?process=7">yedekler için dizinleme sonuçları Single request import==Tek bir istek içe aktarım This will submit only a single request as given here to an OAI-PMH server and imports records into the index==Bu, yalnızca burada verilen şekilde tek bir isteği bir OAI-PMH sunucusuna gönderecek ve kayıtları dizine aktaracaktır "Import OAI-PMH source"=="OAI-PMH Kaynağı İçe Aktar" @@ -3890,7 +3890,7 @@ DHT Transfer==DHT Transferi Proxy Use==Proxy Kullanımı Local Crawling</a>==Yerel Tarama</a> Global Crawling</a>==Genel Tarama</a> -Surrogate Import==Yedek İçe Aktarma +Pack Import==Yedek İçe Aktarma Processing Monitor==İşlem Monitörü Loader<==Yükleyici< Rejected URLs==Reddedilen URL'lar diff --git a/locales/uk.lng b/locales/uk.lng index 3b8e146fb..b95079df5 100644 --- a/locales/uk.lng +++ b/locales/uk.lng @@ -952,7 +952,7 @@ DHT Transfer==DHT-Розподіл Proxy Use==Використання проксі Local Crawling</a>==Місцеве сканування</a> Global Crawling</a>==Загальне сканування</a> -Surrogate Import</a>==Імпорт "Заміщень"</a> +Pack Import</a>==Імпорт "Заміщень"</a> >Crawl Results Overview<==>Огляд результатів сканування< These are monitoring pages for the different indexing queues.==Ці сторінки для спостереження за різними чергами індексації. YaCy knows 5 different ways to acquire web indexes. The details of these processes (1-5) are described within the submenu's listed==YaCy знає 5 різних способів упорядкування індексу. Деталі цих процесів (1-5) описані в меню вище. @@ -960,7 +960,7 @@ above which also will show you a table with indexing results so far. The informa so you need to log-in with your administration password.==тому вам необхідно увійти в систему з паролем адміністратора для його перегляду. Case (6) is a monitor of the local receipt-generator, the opposed case of (1). It contains also an indexing result monitor but is not considered private==В (6) знаходиться спостереження за місцевим генератором прийому на відміну від (1). Він також містить спостереження за результатами індексування, але не є приватним, since it shows crawl requests from other peers.==тому-що показує запити сканування від інших вузлів. -Case (7) occurs if surrogate files are imported==Вибір (7) використовується при імпортуванні файлів заміщень. +Case (7) occurs if pack files are imported==Вибір (7) використовується при імпортуванні файлів заміщень. The image above illustrates the data flow initiated by web index acquisition.==На малюнку вище показаний потік даних, розпочатих придбанням індексу з інтернет. Some processes occur double to document the complex index migration structure.==Поява деяких процесів двічі пояснюється складністю структури розподілу індексу. (1) Results of Remote Crawl Receipts==(1) Результати надходжень віддаленого сканування @@ -1003,9 +1003,9 @@ This is the 'mirror'-case of process (1).==Це в "дзеркальний" пр <em>Use Case:</em> This list may fill if you check the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page==<em>Використання:</em> Цей список заповнюється, якщо ви ввімкнули "Приймати запити на сканування" на сторінці '<a href="RemoteCrawl_p.html">Віддалений сканування</a>'. The stack is empty.==Список порожній. Statistics about #[domains]# domains in this stack:==Статистика щодо #[domains]# доменів в цій черзі: -(7) Results from surrogates import==(7) Результати з імпорту "заміщень" -These records had been imported from surrogate files in DATA/SURROGATES/in==Ці записи було імпортовано з сурогатних файлів з DATA/SURROGATES/in -<em>Use Case:</em> place files with dublin core metadata content into DATA/SURROGATES/in or use an index import method==<em>Використання:</em> Розмістіть файли з вмістом dublin core metadata в DATA/SURROGATES/in або використайте в функцію імпорту індексу +(7) Results from pack import==(7) Результати з імпорту "заміщень" +These records had been imported from pack files in DATA/PACKS/load==Ці записи було імпортовано з сурогатних файлів з DATA/PACKS/load +<em>Use Case:</em> place files with dublin core metadata content into DATA/PACKS/load or use an index import method==<em>Використання:</em> Розмістіть файли з вмістом dublin core metadata в DATA/PACKS/load або використайте в функцію імпорту індексу (i.e. <a href="IndexImportMediawiki_p.html">MediaWiki import</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH retrieval</a>)==(Наприклад, <a href="IndexImportMediawiki_p.html">Імпорт Dump'у MediaWiki</a>, <a href="IndexImportOAIPMH_p.html">Імпорт OAI-PMH</a>) Domain==Домен URLs=URL @@ -1556,8 +1556,8 @@ If you read from an imported database, here are some hints to get around problem before importing large database dumps, set the following Line==Перед імпортуванням великих dump'ів бази даних впишіть наступний рядок in phpmyadmin/config.inc.php and place your dump file in /tmp (Otherwise it is not possible to upload files larger than 2MB)==в phpmyadmin/config.inc.php і збережіть файл бази даних в /tmp (в іншому випадку неможливо завантажити файл розміром більше 2 Мб) deselect the partial import flag==Зніміть прапорець часткового імпорту -When an export is started, surrogate files are generated into DATA/SURROGATE/in which are automatically fetched by an indexer thread.==При запуску експорту в DATA/SURROGATE/in створюються допоміжні файли, які автоматично вилучаються та опрацьовуються потоком індексувача. -All indexed surrogate files are then moved to DATA/SURROGATE/out and can be re-cycled when an index is deleted.==Всі проіндексовані допоміжні файли потім переміщуються в DATA/SURROGATE/out, і можуть бути знову оброблені, якщо індекс буде видалено. +When an export is started, pack files are generated into DATA/PACKS/load which are automatically fetched by an indexer thread.==При запуску експорту в DATA/PACKS/load створюються допоміжні файли, які автоматично вилучаються та опрацьовуються потоком індексувача. +All indexed pack files are then moved to DATA/PACKS/loaded and can be re-cycled when an index is deleted.==Всі проіндексовані допоміжні файли потім переміщуються в DATA/PACKS/loaded, і можуть бути знову оброблені, якщо індекс буде видалено. <b>The URL stub</b>,<br />like https://community.searchlab.eu==<b>Частина URL, як наприклад,</b><br />https://community.searchlab.eu this must be the path right in front of '/viewtopic.php?'==повний шлях перед "/viewtopic.php?" Type==Тип @@ -1574,16 +1574,16 @@ that can access the database==з доступом до БД Password==Пароль for the account of that user given above==для доступу вищезазначеним користувачем Posts per file==Повідомлень на файл -in exported surrogates==в експортованому допоміжному файлі +in exported packs==в експортованому допоміжному файлі Check database connection==Перевірити підключення до бази даних -Export Content to Surrogates==Експортувати вміст до допоміжних файлів +Export Content to Packs==Експортувати вміст до допоміжних файлів Import a database dump==Імпорт виписок бази даних Import Dump==Імпортувати dump бази даних Posts in database==Повідомлень у базі даних first entry==перший запис last entry==останній запис Info failed:==Інформаційна помилка: -Export successful! Wrote #[files]# files in DATA/SURROGATES/in==Експортування завершилось успіхом! #[files]# файлів записано в DATA/SURROGATES/in +Export successful! Wrote #[files]# files in DATA/PACKS/load==Експортування завершилось успіхом! #[files]# файлів записано в DATA/PACKS/load Export failed:==Збій при експорті: Import successful!==Імпорт завершився успіхом! Import failed:==Не вдалося імпортувати: @@ -1769,11 +1769,11 @@ Dumps must be in XML format and may be compressed in gz or bz2. Place the file i "Import MediaWiki Dump"=="Імпортувати Dump MediaWiki" When the import is started, the following happens:==Коли імпортування запущене, відбувається наступне: The dump is extracted on the fly and wiki entries are translated into Dublin Core data format. The output looks like this:==Dump видобувається під час виконання і wiki-записи переводяться у формат Dublin Core. Результат виглядає наступним чином: -Each 10000 wiki records are combined in one output file which is written to /DATA/SURROGATES/in into a temporary file.==Кожні 10000 записів wiki пишуться разом в один вихідний файл і тимчасово зберігаються в /DATA/SURROGATES/in. +Each 10000 wiki records are combined in one output file which is written to /DATA/PACKS/load into a temporary file.==Кожні 10000 записів wiki пишуться разом в один вихідний файл і тимчасово зберігаються в /DATA/PACKS/load. When each of the generated output file is finished, it is renamed to a .xml file==Коли кожен із створених вихідних файлів було оброблено, вони перейменовуються в файл .xml. -Each time a xml surrogate file appears in /DATA/SURROGATES/in, the YaCy indexer fetches the file and indexes the record entries.==Кожного разу, коли сурогатний XML-файл з’являється в /DATA/SURROGATES/in, індексувач YaCy розбирає цей файл і індексує записи, що там містяться. -When a surrogate file is finished with indexing, it is moved to /DATA/SURROGATES/out==Коли сурогатний файл повністю проіндексований, він переноситься в /DATA/SURROGATES/out -You can recycle processed surrogate files by moving them from /DATA/SURROGATES/out to /DATA/SURROGATES/in==Ви можете переобробити вже оброблені сурогатні файли, перемістивши їх з /DATA/SURROGATES/out в /DATA/SURROGATES/in. +Each time a xml pack file appears in /DATA/PACKS/load, the YaCy indexer fetches the file and indexes the record entries.==Кожного разу, коли сурогатний XML-файл з’являється в /DATA/PACKS/load, індексувач YaCy розбирає цей файл і індексує записи, що там містяться. +When a pack file is finished with indexing, it is moved to /DATA/PACKS/loaded==Коли сурогатний файл повністю проіндексований, він переноситься в /DATA/PACKS/loaded +You can recycle processed pack files by moving them from /DATA/PACKS/loaded to /DATA/PACKS/load==Ви можете переобробити вже оброблені сурогатні файли, перемістивши їх з /DATA/PACKS/loaded в /DATA/PACKS/load. Import Process==Процес імпорту Thread:==Потік: #Dump:==Dump: @@ -1792,7 +1792,7 @@ Remaining Time:==Залишилось часу: #File: IndexImportOAIPMH_p.html #--------------------------- OAI-PMH Import==Імпорт OAI-PMH -Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for surrogates==Наслідки імпортування знаходяться на сторінці <a href="CrawlResults.html?process=7">наслідків імпорту "заміщень" +Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for packs==Наслідки імпортування знаходяться на сторінці <a href="CrawlResults.html?process=7">наслідків імпорту "заміщень" Single request import==Імпортування одиночним запитом This will submit only a single request as given here to a OAI-PMH server and imports records into the index==При цьому способі тільки один запит відправляється до OAI-PMH сервера і отримані записи додаються в індекс "Import OAI-PMH source"=="Імпортувати джерело OAI-PMH" @@ -2342,7 +2342,7 @@ milliseconds==мілісекунд > kbytes<==> кбайтів< #Сурогати->Заміщення -</body>==<script>window.onload = function () {$("td:contains('unlimited')").text('необмежений'); $("td:contains('HTTPD Server/Proxy')").text('HTTPD сервер/проксі'); $("td:contains('DHT Distribution')").text('DHT розподіл'); $("td:contains('YaCy Core')").text('Ядро YaCy'); $("td:contains('Seed-List Upload')").text('Вивантаження списку насіння'); $("a:contains('Local Crawl')").text('Місцеве сканування'); $("td:contains('Remote Crawl URL Loader')").text('URL завантажувач віддаленого сканування'); $("a:contains('Remote Crawl Job')").text('Завдання віддаленого сканування'); $("td:contains('Surrogates')").text('Заміщення'); $("td:contains('Cleanup')").text('Очищення'); $("td:contains('Crawler Pool')").text('Басейн сканувача'); $("td:contains('httpd Session Pool')").text('Басейн HTTPD сесії'); $("td:contains('the HTTPD, used as web server and proxy')").text('HTTPD, використовується як веб-сервер і проксі'); $("td:contains('selection, transfer and deletion of index entries that are not searched on your peer, but on others')").text('відбір, передача та видалення записів індексу, які шукались не на вашому вузлі, а на інших'); $("td:contains('this is the p2p-control and peer-ping task')").text('це завдання p2p-керування та пінгу вузла'); $("td:contains('task that a principal peer performes to generate and upload a seed-list to a ftp account')").text('завдання, яке головний вузол здійснює для створення і завантаження списку насіння для облікового запису ftp'); $("td:contains('thread that performes a single crawl step from the local crawl queue')").text('потік, який виконує один крок сканування з місцевої черги сканування'); $("td:contains('thread that loads remote crawl lists from other peers')").text('потік, який завантажує списки віддалених сканувань від інших вузлів'); $("td:contains('thread that performes a single crawl/indexing step triggered by a remote peer')").text('потік, який проводить одиночний крок сканування/індексації, викликаний віддаленим вузлом'); $("td:contains('A thread that polls the SURROGATES path and puts all Documents in one surroagte file into the indexing queue.')").text('потік, що опитує шлях сурогатів і розміщує всі документи в один surroagte файл в черзі індексації.'); $("td:contains('simple cleaning process for monitoring information')").text('простий процес очищення інформації моніторингу');}</script></body> +</body>==<script>window.onload = function () {$("td:contains('unlimited')").text('необмежений'); $("td:contains('HTTPD Server/Proxy')").text('HTTPD сервер/проксі'); $("td:contains('DHT Distribution')").text('DHT розподіл'); $("td:contains('YaCy Core')").text('Ядро YaCy'); $("td:contains('Seed-List Upload')").text('Вивантаження списку насіння'); $("a:contains('Local Crawl')").text('Місцеве сканування'); $("td:contains('Remote Crawl URL Loader')").text('URL завантажувач віддаленого сканування'); $("a:contains('Remote Crawl Job')").text('Завдання віддаленого сканування'); $("td:contains('Packs')").text('Заміщення'); $("td:contains('Cleanup')").text('Очищення'); $("td:contains('Crawler Pool')").text('Басейн сканувача'); $("td:contains('httpd Session Pool')").text('Басейн HTTPD сесії'); $("td:contains('the HTTPD, used as web server and proxy')").text('HTTPD, використовується як веб-сервер і проксі'); $("td:contains('selection, transfer and deletion of index entries that are not searched on your peer, but on others')").text('відбір, передача та видалення записів індексу, які шукались не на вашому вузлі, а на інших'); $("td:contains('this is the p2p-control and peer-ping task')").text('це завдання p2p-керування та пінгу вузла'); $("td:contains('task that a principal peer performes to generate and upload a seed-list to a ftp account')").text('завдання, яке головний вузол здійснює для створення і завантаження списку насіння для облікового запису ftp'); $("td:contains('thread that performes a single crawl step from the local crawl queue')").text('потік, який виконує один крок сканування з місцевої черги сканування'); $("td:contains('thread that loads remote crawl lists from other peers')").text('потік, який завантажує списки віддалених сканувань від інших вузлів'); $("td:contains('thread that performes a single crawl/indexing step triggered by a remote peer')").text('потік, який проводить одиночний крок сканування/індексації, викликаний віддаленим вузлом'); $("td:contains('A thread that polls the PACKS path and puts all Documents in one surroagte file into the indexing queue.')").text('потік, що опитує шлях сурогатів і розміщує всі документи в один surroagte файл в черзі індексації.'); $("td:contains('simple cleaning process for monitoring information')").text('простий процес очищення інформації моніторингу');}</script></body> #----------------------------- #File: PerformanceConcurrency_p.html @@ -3597,7 +3597,7 @@ DHT Transfer==DHT-Передача Proxy Use==Проксі Local Crawling==Місцеві Global Crawling==Глобальні -Surrogate Import==Імпорт "Заміщень" +Pack Import==Імпорт "Заміщень" #----------------------------- #File: env/templates/submenuDesign.template diff --git a/locales/zh.lng b/locales/zh.lng index 396e1af13..d85b84271 100644 --- a/locales/zh.lng +++ b/locales/zh.lng @@ -1043,8 +1043,8 @@ If you read from an imported database, here are some hints to get around problem before importing large database dumps, set==在导入尺寸较大的数据库时, in phpmyadmin/config.inc.php and place your dump file in /tmp (Otherwise it is not possible to upload files larger than 2MB)==设置phpmyadmin/config.inc.php的内容, 并将你的数据库文件放到 /tmp 目录下(否则不能上传大于2MB的文件) deselect the partial import flag==取消部分导入 -When an export is started, surrogate files are generated into DATA/SURROGATE/in which are automatically fetched by an indexer thread.==导出过程开始时, 在 DATA/SURROGATE/in 目录下自动生成备份文件, 并且会被索引器自动爬取. -All indexed surrogate files are then moved to DATA/SURROGATE/out and can be re-cycled when an index is deleted.==所有被索引的备份文件都在 DATA/SURROGATE/out 目录下, 并被索引器循环利用. +When an export is started, pack files are generated into DATA/PACKS/load which are automatically fetched by an indexer thread.==导出过程开始时, 在 DATA/PACKS/load 目录下自动生成备份文件, 并且会被索引器自动爬取. +All indexed pack files are then moved to DATA/PACKS/loaded and can be re-cycled when an index is deleted.==所有被索引的备份文件都在 DATA/PACKS/loaded 目录下, 并被索引器循环利用. The URL stub==URL根域名 like https://searchlab.eu==比如链接 https://searchlab.eu this must be the path right in front of '/viewtopic.php?'==必须在'/viewtopic.php?'前面 @@ -1064,16 +1064,16 @@ that can access the database==用户名 Password==给定用户名的 for the account of that user given above==访问密码 Posts per file==导出备份中 -in exported surrogates==每个文件拥有的最多帖子数 +in exported packs==每个文件拥有的最多帖子数 Check database connection==检查数据库连接 -Export Content to Surrogates==导出到备份 +Export Content to Packs==导出到备份 Import a database dump==导入数据库 Import Dump==导入 Posts in database==数据库中帖子 first entry==第一个 last entry==最后一个 Info failed:==错误信息: -Export successful! Wrote #[files]# files in DATA/SURROGATES/in==导出成功! #[files]# 已写入到 DATA/SURROGATES/in 目录 +Export successful! Wrote #[files]# files in DATA/PACKS/load==导出成功! #[files]# 已写入到 DATA/PACKS/load 目录 Export failed:==导出失败: Import successful!==导入成功! Import failed:==导入失败: @@ -1275,7 +1275,7 @@ above which also will show you a table with indexing results so far. The informa so you need to log-in with your administration password.==所以你需要以管理员账户来查看. Case (6) is a monitor of the local receipt-generator, the opposed case of (1). It contains also an indexing result monitor but is not considered private==事件(6)是本地回执生成器的监控器, (1)的相反事件. 它也包含一个索引结果监控器, 但不是私有的. since it shows crawl requests from other peers.==因为它显示了来自其他节点的爬取请求. -Case (7) occurs if surrogate files are imported==事件(7)发生在导入备份文件时 +Case (7) occurs if pack files are imported==事件(7)发生在导入备份文件时 The image above illustrates the data flow initiated by web index acquisition.==上图解释了由网页索引查询发起的数据流. Some processes occur double to document the complex index migration structure.==某些进程发生了两次以记录复杂的索引迁移结构. (1) Results of Remote Crawl Receipts==(1) 远端爬取回执的结果 @@ -1311,9 +1311,9 @@ This is the 'mirror'-case of process (1).==这是进程(1)的'镜像'事件. <em>Use Case:</em> This list may fill if you check the 'Accept Remote Crawl Requests'-flag on the '<a href="RemoteCrawl_p.html">Remote Crawling</a>' page==<em>用法:</em> 如果你在 '<a href="RemoteCrawl_p.html">远端爬取</a>' 页面勾选'接受远端爬取请求'-标记，此列表会填写 The stack is empty.==此栈为空. Statistics about #[domains]# domains in this stack:==此栈显示有关 #[domains]# 域的数据: -(7) Results from surrogates import==(7) 备份导入结果 -These records had been imported from surrogate files in DATA/SURROGATES/in==这些记录从 DATA/SURROGATES/in 中的备份文件中导入 -<em>Use Case:</em> place files with dublin core metadata content into DATA/SURROGATES/in or use an index import method==将包含Dublin核心元数据的文件放在 DATA/SURROGATES/in 中, 或者使用索引导入方式 +(7) Results from pack import==(7) 备份导入结果 +These records had been imported from pack files in DATA/PACKS/load==这些记录从 DATA/PACKS/load 中的备份文件中导入 +<em>Use Case:</em> place files with dublin core metadata content into DATA/PACKS/load or use an index import method==将包含Dublin核心元数据的文件放在 DATA/PACKS/load 中, 或者使用索引导入方式 (i.e. <a href="IndexImportMediawiki_p.html">MediaWiki import</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH retrieval</a>)==(例如 <a href="IndexImportMediawiki_p.html">MediaWiki 导入</a>, <a href="IndexImportOAIPMH_p.html">OAI-PMH 导入</a>) >Domain==>域名 "delete all"=="全部删除" @@ -2065,11 +2065,11 @@ Dumps must be in XML format and may be compressed in gz or bz2. Place the file i "Import MediaWiki Dump"=="导入MediaWiki备份" When the import is started, the following happens:==:开始导入时, 会进行以下工作 The dump is extracted on the fly and wiki entries are translated into Dublin Core data format. The output looks like this:==备份文件即时被解压, 并被译为Dublin核心元数据格式: -Each 10000 wiki records are combined in one output file which is written to /DATA/SURROGATES/in into a temporary file.==每个输出文件都含有10000个百科记录, 并都被保存在 /DATA/SURROGATES/in 的临时目录中. +Each 10000 wiki records are combined in one output file which is written to /DATA/PACKS/load into a temporary file.==每个输出文件都含有10000个百科记录, 并都被保存在 /DATA/PACKS/load 的临时目录中. When each of the generated output file is finished, it is renamed to a .xml file==生成的输出文件都以 .xml结尾 -Each time a xml surrogate file appears in /DATA/SURROGATES/in, the YaCy indexer fetches the file and indexes the record entries.==只要 /DATA/SURROGATES/in 中含有 xml文件, YaCy索引器就会读取它们并为其中的词条制作索引. -When a surrogate file is finished with indexing, it is moved to /DATA/SURROGATES/out==当索引完成时, xml文件会被移动到 /DATA/SURROGATES/out -You can recycle processed surrogate files by moving them from /DATA/SURROGATES/out to /DATA/SURROGATES/in==你可以将文件从/DATA/SURROGATES/out 移动到 /DATA/SURROGATES/in 以重复索引. +Each time a xml pack file appears in /DATA/PACKS/load, the YaCy indexer fetches the file and indexes the record entries.==只要 /DATA/PACKS/load 中含有 xml文件, YaCy索引器就会读取它们并为其中的词条制作索引. +When a pack file is finished with indexing, it is moved to /DATA/PACKS/loaded==当索引完成时, xml文件会被移动到 /DATA/PACKS/loaded +You can recycle processed pack files by moving them from /DATA/PACKS/loaded to /DATA/PACKS/load==你可以将文件从/DATA/PACKS/loaded 移动到 /DATA/PACKS/load 以重复索引. Import Process==导入进程 Thread:==线程: Processed:==已完成: @@ -2085,7 +2085,7 @@ Remaining Time:==剩余时间: #File: IndexImportOAIPMH_p.html #--------------------------- OAI-PMH Import==OAI-PMH导入 -Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for surrogates==导入结果<a href="CrawlResults.html?process=7">监控 +Results from the import can be monitored in the <a href="CrawlResults.html?process=7">indexing results for packs==导入结果<a href="CrawlResults.html?process=7">监控 Single request import==单个导入请求 This will submit only a single request as given here to a OAI-PMH server and imports records into the index==向OAI-PMH服务器提交如下导入请求, 并将返回记录导入索引 "Import OAI-PMH source"=="导入OAI-PMH源" @@ -4279,7 +4279,7 @@ DHT Transfer==DHT 传输 Proxy Use==代理使用 Local Crawling</a>==本地爬取</a> Global Crawling</a>==全球爬取</a> -Surrogate Import==代理导入 +Pack Import==代理导入 Crawl Results==爬取结果 Crawler<==爬虫< Global==全球 diff --git a/source/net/yacy/crawler/CrawlStacker.java b/source/net/yacy/crawler/CrawlStacker.java index 1b4874a96..e62bfaad2 100644 --- a/source/net/yacy/crawler/CrawlStacker.java +++ b/source/net/yacy/crawler/CrawlStacker.java @@ -327,7 +327,7 @@ public final class CrawlStacker implements WorkflowTask<Request>{ * @return null if successfull, a reason string if not successful
*/
public String stackSimpleCrawl(final DigestURL url) {
- final CrawlProfile pe = this.crawler.defaultSurrogateProfile;
+ final CrawlProfile pe = this.crawler.defaultPackProfile;
return this.stackCrawl(new Request(
this.peers.mySeed().hash.getBytes(),
url,
diff --git a/source/net/yacy/crawler/CrawlSwitchboard.java b/source/net/yacy/crawler/CrawlSwitchboard.java index 3c17c75cf..153b39539 100644 --- a/source/net/yacy/crawler/CrawlSwitchboard.java +++ b/source/net/yacy/crawler/CrawlSwitchboard.java @@ -1,6 +1,6 @@ // CrawlSwitchboard.java -// SPDX-FileCopyrightText: 2005 2006 Michael Peter Christen <mc@yacy.net)> -// SPDX-License-Identifier: GPL-2.0-or-later +// SPDX-FileCopyrightText: 2005 2006 Michael Peter Christen <mc@yacy.net)> +// SPDX-License-Identifier: GPL-2.0-or-later // Frankfurt a. M., Germany // first published 2005 on http://www.anomic.de // @@ -61,7 +61,7 @@ import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; public final class CrawlSwitchboard { - + public static final String CRAWL_PROFILE_AUTOCRAWL_DEEP = "autocrawlDeep"; public static final String CRAWL_PROFILE_AUTOCRAWL_SHALLOW = "autocrawlShallow"; public static final String CRAWL_PROFILE_RECRAWL_JOB = "recrawlJob"; @@ -72,9 +72,9 @@ public final class CrawlSwitchboard { public static final String CRAWL_PROFILE_GREEDY_LEARNING_TEXT = "snippetGreedyLearningText"; public static final String CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA = "snippetLocalMedia"; public static final String CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA = "snippetGlobalMedia"; - public static final String CRAWL_PROFILE_SURROGATE = "surrogates"; + public static final String CRAWL_PROFILE_PACKS = "packs"; - public static Set<String> DEFAULT_PROFILES = new HashSet<String>(); + public static Set<String> DEFAULT_PROFILES = new HashSet<>(); static { DEFAULT_PROFILES.add(CRAWL_PROFILE_AUTOCRAWL_DEEP); DEFAULT_PROFILES.add(CRAWL_PROFILE_AUTOCRAWL_SHALLOW); @@ -86,9 +86,9 @@ public final class CrawlSwitchboard { DEFAULT_PROFILES.add(CRAWL_PROFILE_GREEDY_LEARNING_TEXT); DEFAULT_PROFILES.add(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA); DEFAULT_PROFILES.add(CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA); - DEFAULT_PROFILES.add(CRAWL_PROFILE_SURROGATE); + DEFAULT_PROFILES.add(CRAWL_PROFILE_PACKS); } - + public static final String DBFILE_ACTIVE_CRAWL_PROFILES = "crawlProfilesActive1.heap"; public static final String DBFILE_PASSIVE_CRAWL_PROFILES = "crawlProfilesPassive1.heap"; @@ -106,7 +106,7 @@ public final class CrawlSwitchboard { public static final long CRAWL_PROFILE_GREEDY_LEARNING_TEXT_RECRAWL_CYCLE = 60L * 24L * 30L; // 30 days public static final long CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE = 60L * 24L * 30L; // 30 days public static final long CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA_RECRAWL_CYCLE = 60L * 24L * 30L; // 30 days - public static final long CRAWL_PROFILE_SURROGATE_RECRAWL_CYCLE = 60L * 24L * 30L; // 30 days + public static final long CRAWL_PROFILE_PACK_RECRAWL_CYCLE = 60L * 24L * 30L; // 30 days private final ConcurrentLog log; private MapHeap profilesActiveCrawls; @@ -114,11 +114,11 @@ public final class CrawlSwitchboard { private final Map<byte[], CrawlProfile> profilesActiveCrawlsCache; //TreeMap<byte[], DigestURI>(Base64Order.enhancedCoder); private final Map<String, RowHandleSet> profilesActiveCrawlsCounter; public CrawlProfile defaultProxyProfile, defaultRemoteProfile, defaultTextSnippetLocalProfile, defaultTextSnippetGlobalProfile; - public CrawlProfile defaultTextGreedyLearningProfile, defaultMediaSnippetLocalProfile, defaultMediaSnippetGlobalProfile, defaultSurrogateProfile; + public CrawlProfile defaultTextGreedyLearningProfile, defaultMediaSnippetLocalProfile, defaultMediaSnippetGlobalProfile, defaultPackProfile; public CrawlProfile defaultAutocrawlDeepProfile, defaultAutocrawlShallowProfile, defaultRecrawlJobProfile; - private Map<String, CrawlProfile> defaultPushProfiles; // for each collection one profile + private final Map<String, CrawlProfile> defaultPushProfiles; // for each collection one profile private final File queuesRoot; - private Switchboard switchboard; + private final Switchboard switchboard; public CrawlSwitchboard(Switchboard switchboard) { @@ -127,13 +127,13 @@ public final class CrawlSwitchboard { this.queuesRoot = this.switchboard.queuesRoot; this.defaultPushProfiles = new ConcurrentHashMap<>(); this.profilesActiveCrawlsCache = Collections.synchronizedMap(new TreeMap<byte[], CrawlProfile>(Base64Order.enhancedCoder)); - this.profilesActiveCrawlsCounter = new ConcurrentHashMap<String, RowHandleSet>(); + this.profilesActiveCrawlsCounter = new ConcurrentHashMap<>(); // make crawl profiles database and default profiles this.queuesRoot.mkdirs(); this.log.config("Initializing Crawl Profiles"); - final File profilesActiveFile = new File(queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES); + final File profilesActiveFile = new File(this.queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES); this.profilesActiveCrawls = loadFromDB(profilesActiveFile); for ( final byte[] handle : this.profilesActiveCrawls.keySet() ) { CrawlProfile p; @@ -147,14 +147,14 @@ public final class CrawlSwitchboard { continue; } } - initActiveCrawlProfiles(); - log.info("Loaded active crawl profiles from file " + this.initActiveCrawlProfiles(); + this.log.info("Loaded active crawl profiles from file " + profilesActiveFile.getName() + ", " + this.profilesActiveCrawls.size() + " entries"); - final File profilesPassiveFile = new File(queuesRoot, DBFILE_PASSIVE_CRAWL_PROFILES); + final File profilesPassiveFile = new File(this.queuesRoot, DBFILE_PASSIVE_CRAWL_PROFILES); this.profilesPassiveCrawls = loadFromDB(profilesPassiveFile); for ( final byte[] handle : this.profilesPassiveCrawls.keySet() ) { CrawlProfile p; @@ -167,7 +167,7 @@ public final class CrawlSwitchboard { continue; } } - log.info("Loaded passive crawl profiles from file " + this.log.info("Loaded passive crawl profiles from file " + profilesPassiveFile.getName() + ", " + this.profilesPassiveCrawls.size() @@ -184,9 +184,9 @@ public final class CrawlSwitchboard { * @return */ public CrawlProfile get(final byte[] profileKey) { - CrawlProfile profile = getActive(profileKey); + CrawlProfile profile = this.getActive(profileKey); if (profile != null) return profile; - profile = getPassive(profileKey); + profile = this.getPassive(profileKey); if (profile == null) return null; // clean up this.putActive(profileKey, profile); @@ -487,13 +487,13 @@ public final class CrawlSwitchboard { UTF8.getBytes(this.defaultTextSnippetGlobalProfile.handle()), this.defaultTextSnippetGlobalProfile); this.defaultTextSnippetGlobalProfile.setCacheStrategy(CacheStrategy.IFEXIST); - - // generate new default entry for RecrawlBusyThread + + // generate new default entry for RecrawlBusyThread this.defaultRecrawlJobProfile = RecrawlBusyThread.buildDefaultCrawlProfile(); this.profilesActiveCrawls.put( UTF8.getBytes(this.defaultRecrawlJobProfile.handle()), this.defaultRecrawlJobProfile); - + // generate new default entry for greedy learning this.defaultTextGreedyLearningProfile = new CrawlProfile( @@ -596,10 +596,10 @@ public final class CrawlSwitchboard { this.profilesActiveCrawls.put( UTF8.getBytes(this.defaultMediaSnippetGlobalProfile.handle()), this.defaultMediaSnippetGlobalProfile); - // generate new default entry for surrogate parsing - this.defaultSurrogateProfile = + // generate new default entry for pack parsing + this.defaultPackProfile = new CrawlProfile( - CRAWL_PROFILE_SURROGATE, + CRAWL_PROFILE_PACKS, CrawlProfile.MATCH_ALL_STRING, //crawlerUrlMustMatch CrawlProfile.MATCH_NEVER_STRING, //crawlerUrlMustNotMatch CrawlProfile.MATCH_ALL_STRING, //crawlerIpMustMatch @@ -613,7 +613,7 @@ public final class CrawlSwitchboard { true, //noindexWhenCanonicalUnequalURL 0, false, - CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SURROGATE_RECRAWL_CYCLE), + CrawlProfile.getRecrawlDate(CRAWL_PROFILE_PACK_RECRAWL_CYCLE), -1, true, true, false, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow, true, @@ -622,16 +622,16 @@ public final class CrawlSwitchboard { false, -1, false, true, CrawlProfile.MATCH_NEVER_STRING, CacheStrategy.NOCACHE, - "robot_" + CRAWL_PROFILE_SURROGATE, + "robot_" + CRAWL_PROFILE_PACKS, ClientIdentification.yacyIntranetCrawlerAgentName, TagValency.EVAL, null, null, 0); this.profilesActiveCrawls.put( - UTF8.getBytes(this.defaultSurrogateProfile.handle()), - this.defaultSurrogateProfile); + UTF8.getBytes(this.defaultPackProfile.handle()), + this.defaultPackProfile); } - + public CrawlProfile getPushCrawlProfile(String collection) { CrawlProfile genericPushProfile = this.defaultPushProfiles.get(collection); if (genericPushProfile != null) return genericPushProfile; @@ -668,7 +668,7 @@ public final class CrawlSwitchboard { this.defaultPushProfiles.put(collection, genericPushProfile); return genericPushProfile; } - + private void resetProfiles() { this.profilesActiveCrawlsCache.clear(); final File pdb = new File(this.queuesRoot, DBFILE_ACTIVE_CRAWL_PROFILES); @@ -682,7 +682,7 @@ public final class CrawlSwitchboard { ConcurrentLog.logException(e1); this.profilesActiveCrawls = null; } - initActiveCrawlProfiles(); + this.initActiveCrawlProfiles(); } public boolean clear() throws InterruptedException { @@ -712,7 +712,7 @@ public final class CrawlSwitchboard { } } } catch (final kelondroException e ) { - resetProfiles(); + this.resetProfiles(); hasDoneSomething = true; } return hasDoneSomething; @@ -720,7 +720,7 @@ public final class CrawlSwitchboard { public Set<String> getActiveProfiles() { // find all profiles that are candidates for deletion - Set<String> profileKeys = new HashSet<String>(); + final Set<String> profileKeys = new HashSet<>(); for (final byte[] handle: this.getActive()) { CrawlProfile entry; entry = new CrawlProfile(this.getActive(handle)); @@ -730,59 +730,59 @@ public final class CrawlSwitchboard { } return profileKeys; } - + public Set<String> getFinishedProfiles(CrawlQueues crawlQueues) { // clear the counter cache - this.profilesActiveCrawlsCounter.clear(); - + this.profilesActiveCrawlsCounter.clear(); + // find all profiles that are candidates for deletion - Set<String> deletionCandidate = getActiveProfiles(); - if (deletionCandidate.size() == 0) return new HashSet<String>(0); - + final Set<String> deletionCandidate = this.getActiveProfiles(); + if (deletionCandidate.size() == 0) return new HashSet<>(0); + // iterate through all the queues and see if one of these handles appear there // this is a time-consuming process, set a time-out - long timeout = System.currentTimeMillis() + 60000L; // one minute time + final long timeout = System.currentTimeMillis() + 60000L; // one minute time try { - for (StackType stack: StackType.values()) { - Iterator<Request> sei = crawlQueues.noticeURL.iterator(stack); + for (final StackType stack: StackType.values()) { + final Iterator<Request> sei = crawlQueues.noticeURL.iterator(stack); if (sei == null) continue; Request r; while (sei.hasNext()) { r = sei.next(); if (r == null) continue; - String handle = r.profileHandle(); + final String handle = r.profileHandle(); RowHandleSet us = this.profilesActiveCrawlsCounter.get(handle); if (us == null) {us = new RowHandleSet(Word.commonHashLength, Word.commonHashOrder, 0); this.profilesActiveCrawlsCounter.put(handle, us);} if (us.size() < 100) us.put(r.url().hash()); // store the hash, but not too many deletionCandidate.remove(handle); - if (deletionCandidate.size() == 0) return new HashSet<String>(0); - if (System.currentTimeMillis() > timeout) return new HashSet<String>(0); // give up; this is too large + if (deletionCandidate.size() == 0) return new HashSet<>(0); + if (System.currentTimeMillis() > timeout) return new HashSet<>(0); // give up; this is too large } - if (deletionCandidate.size() == 0) return new HashSet<String>(0); + if (deletionCandidate.size() == 0) return new HashSet<>(0); } // look into the CrawlQueues.worker as well - Map<DigestURL, Request> map = switchboard.crawlQueues.activeWorkerEntries(); - for (Request request: map.values()) { + final Map<DigestURL, Request> map = this.switchboard.crawlQueues.activeWorkerEntries(); + for (final Request request: map.values()) { deletionCandidate.remove(request.profileHandle()); } } catch (final Throwable e) { ConcurrentLog.logException(e); - return new HashSet<String>(0); + return new HashSet<>(0); } return deletionCandidate; } - + public boolean allCrawlsFinished(CrawlQueues crawlQueues) { if (!crawlQueues.noticeURL.isEmpty()) return false; // look into the CrawlQueues.worker as well - if (switchboard.crawlQueues.activeWorkerEntries().size() > 0) return false; + if (this.switchboard.crawlQueues.activeWorkerEntries().size() > 0) return false; return true; } - + public void cleanProfiles(Set<String> deletionCandidate) { // all entries that are left are candidates for deletion; do that now - for (String h: deletionCandidate) { - byte[] handle = ASCII.getBytes(h); + for (final String h: deletionCandidate) { + final byte[] handle = ASCII.getBytes(h); final CrawlProfile p = this.getActive(handle); if (p != null) { this.putPassive(handle, p); @@ -790,7 +790,7 @@ public final class CrawlSwitchboard { } } } - + public synchronized void close() { this.profilesActiveCrawlsCache.clear(); this.profilesActiveCrawls.close(); diff --git a/source/net/yacy/crawler/data/ResultURLs.java b/source/net/yacy/crawler/data/ResultURLs.java index 402c0d7db..683c5abd7 100644 --- a/source/net/yacy/crawler/data/ResultURLs.java +++ b/source/net/yacy/crawler/data/ResultURLs.java @@ -1,7 +1,7 @@ // ResultURLs.java // ----------------------- // part of YaCy -// SPDX-FileCopyrightText: 2004 Michael Peter Christen <mc@yacy.net)> +// SPDX-FileCopyrightText: 2004 Michael Peter Christen <mc@yacy.net)> // SPDX-License-Identifier: GPL-2.0-or-later // first published on http://yacy.net // Frankfurt, Germany, 2004 @@ -55,11 +55,11 @@ public final class ResultURLs { PROXY_LOAD(4), LOCAL_CRAWLING(5), GLOBAL_CRAWLING(6), - SURROGATES(7); + PACKS(7); protected int code; private static final EventOrigin[] list = { - UNKNOWN, REMOTE_RECEIPTS, QUERIES, DHT_TRANSFER, PROXY_LOAD, LOCAL_CRAWLING, GLOBAL_CRAWLING, SURROGATES}; + UNKNOWN, REMOTE_RECEIPTS, QUERIES, DHT_TRANSFER, PROXY_LOAD, LOCAL_CRAWLING, GLOBAL_CRAWLING, PACKS}; private EventOrigin(final int code) { this.code = code; } @@ -71,13 +71,13 @@ public final class ResultURLs { } } - private final static Map<EventOrigin, Map<String, InitExecEntry>> resultStacks = new ConcurrentHashMap<EventOrigin, Map<String, InitExecEntry>>(); // a mapping from urlHash to Entries - private final static Map<EventOrigin, ScoreMap<String>> resultDomains = new ConcurrentHashMap<EventOrigin, ScoreMap<String>>(); + private final static Map<EventOrigin, Map<String, InitExecEntry>> resultStacks = new ConcurrentHashMap<>(); // a mapping from urlHash to Entries + private final static Map<EventOrigin, ScoreMap<String>> resultDomains = new ConcurrentHashMap<>(); static { for (final EventOrigin origin: EventOrigin.values()) { - resultStacks.put(origin, new LinkedHashMap<String, InitExecEntry>()); - resultDomains.put(origin, new ClusteredScoreMap<String>(true)); + resultStacks.put(origin, new LinkedHashMap<>()); + resultDomains.put(origin, new ClusteredScoreMap<>(true)); } } @@ -133,7 +133,7 @@ public final class ResultURLs { public static Iterator<Map.Entry<String, InitExecEntry>> results(final EventOrigin stack) { final Map<String, InitExecEntry> resultStack = getStack(stack); if (resultStack == null) return new LinkedHashMap<String, InitExecEntry>().entrySet().iterator(); - return new ReverseMapIterator<String, InitExecEntry>(resultStack); + return new ReverseMapIterator<>(resultStack); } /** diff --git a/source/net/yacy/crawler/retrieval/Response.java b/source/net/yacy/crawler/retrieval/Response.java index 567685510..4a912938e 100644 --- a/source/net/yacy/crawler/retrieval/Response.java +++ b/source/net/yacy/crawler/retrieval/Response.java @@ -1,5 +1,5 @@ // Response.java -// SPDX-FileCopyrightText: 2008 Michael Peter Christen <mc@yacy.net)> +// SPDX-FileCopyrightText: 2008 Michael Peter Christen <mc@yacy.net)> // SPDX-License-Identifier: GPL-2.0-or-later // Frankfurt a. M., Germany // first published 19.08.2008 on http://yacy.net @@ -74,7 +74,7 @@ public class Response { private byte[] content; private int status; // tracker indexing status, see status defs below private final boolean fromCache; - + /** Maximum file size to put in cache for crawler */ public static final long CRAWLER_MAX_SIZE_TO_CACHE = 10 * 1024L * 1024L; @@ -119,14 +119,14 @@ public class Response { if (ext.equals("asf")) return DT_FLASH; return DT_UNKNOWN; } - + /** * doctype calculation based on file extensions; this is the url wrapper * @param url * @return a character denoting the file type */ public static char docType(final MultiProtocolURL url) { - String ext = MultiProtocolURL.getFileExtension(url.getFileName()); + final String ext = MultiProtocolURL.getFileExtension(url.getFileName()); if (ext == null) return DT_UNKNOWN; return docTypeExt(ext); } @@ -171,8 +171,8 @@ public class Response { if (doctype == DT_FLASH) return new String[]{"application/x-shockwave-flash"}; if (doctype == DT_SHARE) return new String[]{"text/plain"}; if (doctype == DT_BINARY) return new String[]{"application/octet-stream"}; - String mime = Classification.ext2mime(ext); - int p = mime.indexOf('/'); + final String mime = Classification.ext2mime(ext); + final int p = mime.indexOf('/'); if (p < 0) return new String[]{mime}; if (doctype == DT_TEXT) return new String[]{"text" + mime.substring(p)}; if (doctype == DT_IMAGE) return new String[]{"image" + mime.substring(p)}; @@ -196,7 +196,7 @@ public class Response { final boolean fromCache, final byte[] content) { this.request = request; - // request and response headers may be zero in case that we process surrogates + // request and response headers may be zero in case that we process packs this.requestHeader = requestHeader; this.responseHeader = responseHeader; this.profile = profile; @@ -204,7 +204,7 @@ public class Response { this.content = content; this.fromCache = fromCache; if (this.responseHeader != null && content != null && Integer.parseInt(this.responseHeader.get(HeaderFramework.CONTENT_LENGTH, "0")) <= content.length) { - this.responseHeader.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(content.length)); // repair length + this.responseHeader.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(content.length)); // repair length } } @@ -216,7 +216,7 @@ public class Response { */ public Response(final Request request, final CrawlProfile profile) { this.request = request; - // request and response headers may be zero in case that we process surrogates + // request and response headers may be zero in case that we process packs this.requestHeader = null; this.responseHeader = new ResponseHeader(200); this.responseHeader.put(HeaderFramework.CONTENT_TYPE, Classification.ext2mime(MultiProtocolURL.getFileExtension(request.url().getFileName()), "text/plain")); // tell parser how to handle the content @@ -226,22 +226,22 @@ public class Response { this.fromCache = true; if (this.responseHeader != null) this.responseHeader.put(HeaderFramework.CONTENT_LENGTH, "0"); // 'virtual' length, shows that the resource was not loaded } - + public void updateStatus(final int newStatus) { this.status = newStatus; } - + /** * @return the original request that produced this response */ public Request getRequest() { - return request; + return this.request; } public ResponseHeader getResponseHeader() { return this.responseHeader; } - + public RequestHeader getRequestHeader() { return this.requestHeader; } @@ -270,8 +270,8 @@ public class Response { } public char docType() { - char doctype = docType(getMimeType()); - if (doctype == DT_UNKNOWN) doctype = docType(url()); + char doctype = docType(this.getMimeType()); + if (doctype == DT_UNKNOWN) doctype = docType(this.url()); return doctype; } @@ -301,7 +301,7 @@ public class Response { } public boolean proxy() { - return initiator() == null; + return this.initiator() == null; } public long size() { @@ -320,8 +320,8 @@ public class Response { public void setContent(final byte[] data) { this.content = data; - if (this.responseHeader != null && this.content != null && Integer.parseInt(this.responseHeader.get(HeaderFramework.CONTENT_LENGTH, "0")) <= content.length) { - this.responseHeader.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(content.length)); // repair length + if (this.responseHeader != null && this.content != null && Integer.parseInt(this.responseHeader.get(HeaderFramework.CONTENT_LENGTH, "0")) <= this.content.length) { + this.responseHeader.put(HeaderFramework.CONTENT_LENGTH, Integer.toString(this.content.length)); // repair length } } @@ -338,7 +338,7 @@ public class Response { */ public String shallStoreCacheForProxy() { - final String crawlerReason = shallStoreCacheForCrawler(); + final String crawlerReason = this.shallStoreCacheForCrawler(); if (crawlerReason != null) return crawlerReason; // check profile (disabled: we will check this in the plasmaSwitchboard) @@ -351,15 +351,15 @@ public class Response { // -CGI access in request // CGI access makes the page very individual, and therefore not usable // in caches - if (url().isPOST() && this.profile != null && !this.profile.crawlingQ()) { + if (this.url().isPOST() && this.profile != null && !this.profile.crawlingQ()) { return "dynamic_post"; } - if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(url().getFileName()))) { + if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(this.url().getFileName()))) { return "dynamic_cgi"; } - if (url().isLocal()) { + if (this.url().isLocal()) { return "local_URL_no_cache_needed"; } @@ -415,10 +415,10 @@ public class Response { public String shallStoreCacheForCrawler() { // check storage size: all files will be handled in RAM before storage, so they must not exceed // a given size, which we consider as 1MB - if (size() > CRAWLER_MAX_SIZE_TO_CACHE) return "too_large_for_caching_" + size(); + if (this.size() > CRAWLER_MAX_SIZE_TO_CACHE) return "too_large_for_caching_" + this.size(); // check status code - if (!validResponseStatus()) { + if (!this.validResponseStatus()) { return "bad_status_" + this.responseHeader.getStatusCode(); } @@ -449,14 +449,14 @@ public class Response { public boolean isFreshForProxy() { if (Switchboard.getSwitchboard().getConfigBool("proxyAlwaysFresh", false)) return true; - + // -CGI access in request // CGI access makes the page very individual, and therefore not usable // in caches - if (url().isPOST()) { + if (this.url().isPOST()) { return false; } - if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(url().getFileName()))) { + if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(this.url().getFileName()))) { return false; } @@ -494,7 +494,7 @@ public class Response { if (d2.after(d1)) { return false; } } - final String mimeType = getMimeType(); + final String mimeType = this.getMimeType(); if (!Classification.isPictureMime(mimeType)) { // -cookies in request // unfortunately, we should reload in case of a cookie @@ -593,22 +593,22 @@ public class Response { * This function is used by plasmaSwitchboard#processResourceStack */ public final String shallIndexCacheForProxy() { - if (profile() == null) { + if (this.profile() == null) { return "shallIndexCacheForProxy: profile() is null !"; } // check profile - if (!profile().indexText() && !profile().indexMedia()) { + if (!this.profile().indexText() && !this.profile().indexMedia()) { return "indexing not allowed - indexText and indexMedia not set (for proxy = " + this.profile.collectionName()+ ")"; } // -CGI access in request // CGI access makes the page very individual, and therefore not usable in caches - if (!profile().crawlingQ()) { - if (url().isPOST()) { + if (!this.profile().crawlingQ()) { + if (this.url().isPOST()) { return "Dynamic_(POST)"; } - if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(url().getFileName()))) { + if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(this.url().getFileName()))) { return "Dynamic_(CGI)"; } } @@ -622,7 +622,7 @@ public class Response { // -cookies in request // unfortunately, we cannot index pages which have been requested with a cookie // because the returned content may be special for the client - if (requestWithCookie()) { + if (this.requestWithCookie()) { // System.out.println("***not indexed because cookie"); return "Dynamic_(Requested_With_Cookie)"; } @@ -723,20 +723,20 @@ public class Response { * This function is used by plasmaSwitchboard#processResourceStack */ public final String shallIndexCacheForCrawler() { - if (profile() == null) { + if (this.profile() == null) { return "shallIndexCacheForCrawler: profile() is null !"; } // check profile - if (!profile().indexText() && !profile().indexMedia()) { + if (!this.profile().indexText() && !this.profile().indexMedia()) { return "indexing not allowed - indexText and indexMedia not set (for crawler = " + this.profile.collectionName() + ")"; } // -CGI access in request // CGI access makes the page very individual, and therefore not usable in caches - if (!profile().crawlingQ()) { - if (url().isPOST()) { return "Dynamic_(POST)"; } - if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(url().getFileName()))) { return "Dynamic_(CGI)"; } + if (!this.profile().crawlingQ()) { + if (this.url().isPOST()) { return "Dynamic_(POST)"; } + if (MultiProtocolURL.isCGI(MultiProtocolURL.getFileExtension(this.url().getFileName()))) { return "Dynamic_(CGI)"; } } // -authorization cases in request @@ -750,10 +750,10 @@ public class Response { * profile, the indexingDocumentProcessor can eventually index only URL metadata * using the generic parser for unsupported media types */ - if (this.responseHeader != null && !profile().isIndexNonParseableUrls()) { + if (this.responseHeader != null && !this.profile().isIndexNonParseableUrls()) { final String mimeType = this.responseHeader.getContentType(); final String parserError = TextParser.supportsMime(mimeType); - if (parserError != null && TextParser.supportsExtension(url()) != null) return "no parser available: " + parserError; + if (parserError != null && TextParser.supportsExtension(this.url()) != null) return "no parser available: " + parserError; } // -if-modified-since in request @@ -827,7 +827,7 @@ public class Response { } public boolean validResponseStatus() { - int status = this.responseHeader.getStatusCode(); + final int status = this.responseHeader.getStatusCode(); return status == 200 || status == 203; } @@ -854,13 +854,13 @@ public class Response { // 4) proxy-load (initiator is "------------") // 5) local prefetch/crawling (initiator is own seedHash) // 6) local fetching for global crawling (other known or unknown initiator) - // 7) local surrogates processing (can not be known here : crawl profile is required) + // 7) local packs processing (can not be known here : crawl profile is required) EventOrigin processCase = EventOrigin.UNKNOWN; // FIXME the equals seems to be incorrect: String.equals(boolean) - if (initiator() == null || initiator().length == 0 || ASCII.String(initiator()).equals("------------")) { + if (this.initiator() == null || this.initiator().length == 0 || ASCII.String(this.initiator()).equals("------------")) { // proxy-load processCase = EventOrigin.PROXY_LOAD; - } else if (UTF8.String(initiator()).equals(mySeedHash)) { + } else if (UTF8.String(this.initiator()).equals(mySeedHash)) { // normal crawling processCase = EventOrigin.LOCAL_CRAWLING; } else { @@ -871,15 +871,15 @@ public class Response { } public Document[] parse() throws Parser.Failure { - final String supportError = TextParser.supports(url(), this.responseHeader == null ? null : this.responseHeader.getContentType()); - if (supportError != null) throw new Parser.Failure("no parser support:" + supportError, url()); + final String supportError = TextParser.supports(this.url(), this.responseHeader == null ? null : this.responseHeader.getContentType()); + if (supportError != null) throw new Parser.Failure("no parser support:" + supportError, this.url()); try { return TextParser.parseSource( - url(), this.responseHeader == null ? null : this.responseHeader.getContentType(), + this.url(), this.responseHeader == null ? null : this.responseHeader.getContentType(), this.responseHeader == null ? StandardCharsets.UTF_8.name() : this.responseHeader.getCharacterEncoding(), - TagValency.EVAL, new HashSet<String>(), + TagValency.EVAL, new HashSet<>(), new VocabularyScraper(), this.request.timezoneOffset(), this.request.depth(), this.content); - } catch(Parser.Failure e) { + } catch(final Parser.Failure e) { throw e; } catch (final Exception e) { return null; diff --git a/source/net/yacy/document/Document.java b/source/net/yacy/document/Document.java index 34676836a..49993765c 100644 --- a/source/net/yacy/document/Document.java +++ b/source/net/yacy/document/Document.java @@ -87,10 +87,10 @@ public class Document { // text in image tags. private LinkedHashMap<AnchorURL, String> audiolinks, videolinks, applinks, hyperlinks; // TODO: check if redundant value (set to key.getNameProperty()) is needed private LinkedHashMap<DigestURL, String> inboundlinks, outboundlinks; - + /** links to icons that belongs to the document (mapped by absolute URL) */ private Map<DigestURL, IconEntry> icons; - + /** * URLs of linked data item types/classes referenced by the document (for example in * HTML with standard annotations such as RDFa, microdata, microformats or @@ -107,7 +107,7 @@ public class Document { private final Map<String, Set<String>> generic_facets; // a map from vocabulary names to the set of tags for that vocabulary which apply for this document private final Date lastModified; // creation or last modification date of the source document private int crawldepth; - + /** True when this document is the result of a partially parsed resource, for example due to resource content size exceeding a given limit */ private boolean partiallyParsed; @@ -129,17 +129,17 @@ public class Document { this.mimeType = (mimeType == null) ? "application/octet-stream" : mimeType; this.charset = charset; this.parserObject = parserObject; - this.keywords = new LinkedHashSet<String>(); + this.keywords = new LinkedHashSet<>(); if (keywords != null) { Collections.addAll(this.keywords, keywords); } - this.titles = (titles == null) ? new ArrayList<String>(1) : titles; + this.titles = (titles == null) ? new ArrayList<>(1) : titles; this.creator = (author == null) ? new StringBuilder(0) : new StringBuilder(author); - this.sections = new LinkedList<String>() ; + this.sections = new LinkedList<>() ; if (sections != null) { Collections.addAll(this.sections, sections); } - this.descriptions = (abstrcts == null) ? new ArrayList<String>() : abstrcts; + this.descriptions = (abstrcts == null) ? new ArrayList<>() : abstrcts; if (lat >= -90.0d && lat <= 90.0d && lon >= -180.0d && lon <= 180.0d) { this.lon = lon; this.lat = lat; @@ -148,9 +148,9 @@ public class Document { this.lon = 0.0d; this.lat = 0.0d; } - this.anchors = (anchors == null) ? new ArrayList<AnchorURL>(0) : anchors; - this.rss = (rss == null) ? new LinkedHashMap<DigestURL, String>(0) : rss; - this.images = (images == null) ? new LinkedHashMap<DigestURL, ImageEntry>() : images; + this.anchors = (anchors == null) ? new ArrayList<>(0) : anchors; + this.rss = (rss == null) ? new LinkedHashMap<>(0) : rss; + this.images = (images == null) ? new LinkedHashMap<>() : images; this.publisher = publisher; this.hyperlinks = null; this.audiolinks = null; @@ -164,7 +164,7 @@ public class Document { this.languages = languages; this.indexingDenied = indexingDenied; this.text = text == null ? "" : text; - this.generic_facets = new HashMap<String, Set<String>>(); + this.generic_facets = new HashMap<>(); this.lastModified = lastModified == null ? new Date() : lastModified; this.crawldepth = 999; // unknown yet this.scraperObject = null; // will be set by setScraperObject() @@ -177,7 +177,7 @@ public class Document { * @return the content domain which classifies the content type */ public ContentDomain getContentDomain() { - ContentDomain contentDomain = Classification.getContentDomainFromMime(this.mimeType); + final ContentDomain contentDomain = Classification.getContentDomainFromMime(this.mimeType); if (contentDomain != ContentDomain.ALL) return contentDomain; return this.dc_source().getContentDomainFromExt(); } @@ -202,9 +202,9 @@ public class Document { /** * Remember the scraper object used, to be able to access used scraper by * getScraperObject(). - * This is used for surrogate parsers to set a other source/scraper then ContentScraper + * This is used for pack parsers to set a other source/scraper then ContentScraper * used e.g. by htmlParser. - * @param scraper + * @param scraper */ public void setScraperObject(Object scraper) { if (this.scraperObject != null) { @@ -216,12 +216,12 @@ public class Document { } this.scraperObject = scraper; } - + public AnchorURL getCanonical() { final Object scraper = this.getScraperObject(); if (!(scraper instanceof ContentScraper)) return null; final ContentScraper html = (ContentScraper) scraper; - AnchorURL canonical = html.getCanonical(); + final AnchorURL canonical = html.getCanonical(); return canonical; } @@ -236,21 +236,21 @@ public class Document { public Map<String, Set<String>> getGenericFacets() { return this.generic_facets; } - + /** * @return true when this document is the result of a partially parsed resource, for example due to resource content size exceeding a given limit */ public boolean isPartiallyParsed() { return this.partiallyParsed; } - + /** * @param partiallyParsed set to true to indicates this document is the result of a partially parsed resource, for example due to resource content size exceeding a given limit */ public void setPartiallyParsed(final boolean partiallyParsed) { this.partiallyParsed = partiallyParsed; } - + /** * compute a set of languages that this document contains * the language is not computed using a statistical analysis of the content, only from given metadata that came with the document @@ -305,11 +305,11 @@ dc_rights * @param title */ public void setTitle(final String title) { - this.titles = new ArrayList<String>(); + this.titles = new ArrayList<>(); if (title != null) this.titles.add(title); } - + public String dc_creator() { return (this.creator == null) ? "" : this.creator.toString(); @@ -321,7 +321,7 @@ dc_rights * @param tags */ public void addTags(Set<String> tags) { - for (String s: tags) { + for (final String s: tags) { if (s != null && !s.isEmpty()) this.keywords.add(s); } } @@ -341,12 +341,12 @@ dc_rights * @return a map where the key is the navigator name and the value is the set of attributes names */ public static Map<String, Set<String>> computeGenericFacets(Map<String, Set<Tagging.Metatag>> tags) { - Map<String, Set<String>> gf = new HashMap<String, Set<String>>(); - for (Map.Entry<String, Set<Tagging.Metatag>> e: tags.entrySet()) { - Tagging vocabulary = LibraryProvider.autotagging.getVocabulary(e.getKey()); + final Map<String, Set<String>> gf = new HashMap<>(); + for (final Map.Entry<String, Set<Tagging.Metatag>> e: tags.entrySet()) { + final Tagging vocabulary = LibraryProvider.autotagging.getVocabulary(e.getKey()); if (vocabulary == null) continue; - Set<String> objects = new HashSet<String>(); - for (Tagging.Metatag s: e.getValue()) { + final Set<String> objects = new HashSet<>(); + for (final Tagging.Metatag s: e.getValue()) { objects.add(s.getObject()); } gf.put(vocabulary.getName(), objects); @@ -378,7 +378,7 @@ dc_rights } public String[] dc_description() { - if (descriptions == null) return new String[0]; + if (this.descriptions == null) return new String[0]; return this.descriptions.toArray(new String[this.descriptions.size()]); } @@ -408,13 +408,13 @@ dc_rights */ public void rewrite_dc_source(Pattern pattern, String replacement) { String u = this.source.toNormalform(false); - Matcher m = pattern.matcher(u); + final Matcher m = pattern.matcher(u); if (m.matches()) { u = m.replaceAll(replacement); try { - DigestURL du = new DigestURL(u); + final DigestURL du = new DigestURL(u); this.source = du; - } catch (MalformedURLException e) { + } catch (final MalformedURLException e) { } } } @@ -428,7 +428,7 @@ dc_rights public String[] getSectionTitles() { if (this.sections == null) { - return new String[] { dc_title() }; + return new String[] { this.dc_title() }; } return this.sections.toArray(new String[this.sections.size()]); } @@ -500,8 +500,8 @@ dc_rights } public List<StringBuilder> getSentences(final boolean pre) { - final SentenceReader sr = new SentenceReader(getTextString(), pre); - List<StringBuilder> sentences = new ArrayList<StringBuilder>(); + final SentenceReader sr = new SentenceReader(this.getTextString(), pre); + final List<StringBuilder> sentences = new ArrayList<>(); while (sr.hasNext()) { sentences.add(sr.next()); } @@ -535,29 +535,29 @@ dc_rights */ public Map<AnchorURL, String> getHyperlinks() { // this is a subset of the getAnchor-set: only links to other hyperrefs - if (!this.resorted) resortLinks(); + if (!this.resorted) this.resortLinks(); return this.hyperlinks; } public Map<AnchorURL, String> getAudiolinks() { - if (!this.resorted) resortLinks(); + if (!this.resorted) this.resortLinks(); return this.audiolinks; } public Map<AnchorURL, String> getVideolinks() { - if (!this.resorted) resortLinks(); + if (!this.resorted) this.resortLinks(); return this.videolinks; } public LinkedHashMap<DigestURL, ImageEntry> getImages() { // returns all links enbedded as pictures (visible in document) // this resturns a htmlFilterImageEntry collection - if (!this.resorted) resortLinks(); + if (!this.resorted) this.resortLinks(); return this.images; } public Map<AnchorURL, String> getApplinks() { - if (!this.resorted) resortLinks(); + if (!this.resorted) this.resortLinks(); return this.applinks; } @@ -589,13 +589,13 @@ dc_rights int extpos, qpos; String ext = null; final String thishost = this.source.getHost(); - this.inboundlinks = new LinkedHashMap<DigestURL, String>(); - this.outboundlinks = new LinkedHashMap<DigestURL, String>(); - this.hyperlinks = new LinkedHashMap<AnchorURL, String>(); - this.videolinks = new LinkedHashMap<AnchorURL, String>(); - this.audiolinks = new LinkedHashMap<AnchorURL, String>(); - this.applinks = new LinkedHashMap<AnchorURL, String>(); - final Map<AnchorURL, ImageEntry> collectedImages = new HashMap<AnchorURL, ImageEntry>(); // this is a set that is collected now and joined later to the imagelinks + this.inboundlinks = new LinkedHashMap<>(); + this.outboundlinks = new LinkedHashMap<>(); + this.hyperlinks = new LinkedHashMap<>(); + this.videolinks = new LinkedHashMap<>(); + this.audiolinks = new LinkedHashMap<>(); + this.applinks = new LinkedHashMap<>(); + final Map<AnchorURL, ImageEntry> collectedImages = new HashMap<>(); // this is a set that is collected now and joined later to the imagelinks for (final Map.Entry<DigestURL, ImageEntry> entry: this.images.entrySet()) { if (entry.getKey() != null && entry.getKey().getHost() != null && entry.getKey().getHost().equals(thishost)) this.inboundlinks.put(entry.getKey(), "image"); else this.outboundlinks.put(entry.getKey(), "image"); } @@ -644,7 +644,7 @@ dc_rights // expand the hyperlinks: // we add artificial hyperlinks to the hyperlink set // that can be calculated from given hyperlinks and imagelinks - + /* * Should we also include icons ? with * this.hyperlinks.putAll(allReflinks(this.icons.keySet())); It is @@ -672,7 +672,7 @@ dc_rights public static Map<MultiProtocolURL, String> allSubpaths(final Collection<?> links) { // links is either a Set of Strings (urls) or a Set of // htmlFilterImageEntries - final Set<String> h = new HashSet<String>(); + final Set<String> h = new HashSet<>(); Iterator<?> i = links.iterator(); Object o; MultiProtocolURL url; @@ -704,7 +704,7 @@ dc_rights } catch (final MalformedURLException e) { } // now convert the strings to yacyURLs i = h.iterator(); - final Map<MultiProtocolURL, String> v = new HashMap<MultiProtocolURL, String>(); + final Map<MultiProtocolURL, String> v = new HashMap<>(); while (i.hasNext()) { u = (String) i.next(); try { @@ -723,7 +723,7 @@ dc_rights * @return map with contained urls as key and "ref" as value */ private static Map<AnchorURL, String> allReflinks(final Collection<?> links) { - final Map<AnchorURL, String> v = new HashMap<AnchorURL, String>(); + final Map<AnchorURL, String> v = new HashMap<>(); final Iterator<?> i = links.iterator(); Object o; AnchorURL url = null; @@ -770,14 +770,14 @@ dc_rights v.put(url, "ref"); continue loop; } - + if ((pos = u.toLowerCase(Locale.ROOT).indexOf("/www.", 11)) > 0) { // 11 = skip protocol part + www of source url "http://www." i.remove(); u = url.getProtocol()+":/" + u.substring(pos); while ((pos = u.toLowerCase(Locale.ROOT).indexOf("/www.", 11)) > 0) u = url.getProtocol()+":/" + u.substring(pos); - AnchorURL addurl = new AnchorURL(u); + final AnchorURL addurl = new AnchorURL(u); if (!(v.containsKey(addurl))) v.put(addurl, "ref"); continue loop; @@ -805,12 +805,12 @@ dc_rights this.sections.addAll(doc.sections); this.titles.addAll(doc.titles()); this.keywords.addAll(doc.dc_subject()); - for (String d: doc.dc_description()) this.descriptions.add(d); + for (final String d: doc.dc_description()) this.descriptions.add(d); if (!(this.text instanceof ByteArrayOutputStream)) { this.text = new ByteArrayOutputStream(); } - InputStream textStream = doc.getTextStream(); + final InputStream textStream = doc.getTextStream(); try { FileUtils.copy(textStream, (ByteArrayOutputStream) this.text); } finally { @@ -819,7 +819,7 @@ dc_rights /* textStream can be a FileInputStream : we must close it to ensure releasing system resource */ textStream.close(); } - } catch(IOException e) { + } catch(final IOException e) { ConcurrentLog.warn("DOCUMENT", "Could not close text input stream"); } } @@ -829,14 +829,14 @@ dc_rights this.images.putAll(doc.getImages()); } } - + /** * @return links to icons that belongs to the document (mapped by absolute URL) */ public Map<DigestURL, IconEntry> getIcons() { - return icons; + return this.icons; } - + /** * Set links to icons that belongs to the document (mapped by absolute URL) * @param icons @@ -844,12 +844,12 @@ dc_rights public void setIcons(final Map<DigestURL, IconEntry> icons) { /* Better to ensure now icons property will not be null */ if(icons != null) { - this.icons = icons; + this.icons = icons; } else { this.icons = new HashMap<>(); } } - + /** * @return URLs of linked data item types/classes referenced by the document (for example in * HTML with standard annotations such as RDFa, microdata, microformats or @@ -858,7 +858,7 @@ dc_rights public Set<DigestURL> getLinkedDataTypes() { return this.linkedDataTypes; } - + /** * @return URLs of linked data item types/classes referenced by the document */ @@ -870,10 +870,10 @@ dc_rights this.linkedDataTypes.clear(); } } - + public int inboundLinkNofollowCount() { - if (this.inboundlinks == null) resortLinks(); + if (this.inboundlinks == null) this.resortLinks(); if (this.inboundlinks == null) return 0; int c = 0; for (final String tag: this.inboundlinks.values()) { @@ -883,7 +883,7 @@ dc_rights } public int outboundLinkNofollowCount() { - if (this.outboundlinks == null) resortLinks(); + if (this.outboundlinks == null) this.resortLinks(); if (this.outboundlinks == null) return 0; int c = 0; for (final String tag: this.outboundlinks.values()) { @@ -893,12 +893,12 @@ dc_rights } public LinkedHashMap<DigestURL, String> inboundLinks() { - if (this.inboundlinks == null) resortLinks(); + if (this.inboundlinks == null) this.resortLinks(); return (this.inboundlinks == null) ? null : this.inboundlinks; } public LinkedHashMap<DigestURL, String> outboundLinks() { - if (this.outboundlinks == null) resortLinks(); + if (this.outboundlinks == null) this.resortLinks(); return (this.outboundlinks == null) ? null : this.outboundlinks; } @@ -913,30 +913,30 @@ dc_rights public void setDepth(int depth) { this.crawldepth = depth; } - + public int getDepth() { return this.crawldepth; } - + public void writeXML(final Writer os) throws IOException { os.write("<record>\n"); - final String title = dc_title(); + final String title = this.dc_title(); if (title != null && title.length() > 0) os.write("<dc:title><![CDATA[" + title + "]]></dc:title>\n"); - os.write("<dc:identifier>" + dc_identifier() + "</dc:identifier>\n"); - final String creator = dc_creator(); + os.write("<dc:identifier>" + this.dc_identifier() + "</dc:identifier>\n"); + final String creator = this.dc_creator(); if (creator != null && creator.length() > 0) os.write("<dc:creator><![CDATA[" + creator + "]]></dc:creator>\n"); - final String publisher = dc_publisher(); + final String publisher = this.dc_publisher(); if (publisher != null && publisher.length() > 0) os.write("<dc:publisher><![CDATA[" + publisher + "]]></dc:publisher>\n"); final String subject = this.dc_subject(';'); if (subject != null && subject.length() > 0) os.write("<dc:subject><![CDATA[" + subject + "]]></dc:subject>\n"); if (this.text != null) { os.write("<dc:description><![CDATA["); - os.write(getTextString()); + os.write(this.getTextString()); os.write("]]></dc:description>\n"); } - final String language = dc_language(); - if (language != null && language.length() > 0) os.write("<dc:language>" + dc_language() + "</dc:language>\n"); - os.write("<dc:date>" + ISO8601Formatter.FORMATTER.format(getLastModified()) + "</dc:date>\n"); + final String language = this.dc_language(); + if (language != null && language.length() > 0) os.write("<dc:language>" + this.dc_language() + "</dc:language>\n"); + os.write("<dc:date>" + ISO8601Formatter.FORMATTER.format(this.getLastModified()) + "</dc:date>\n"); if (this.lon != 0.0 && this.lat != 0.0) os.write("<geo:Point><geo:long>" + this.lon +"</geo:long><geo:lat>" + this.lat + "</geo:lat></geo:Point>\n"); os.write("</record>\n"); } @@ -946,7 +946,7 @@ dc_rights final ByteArrayOutputStream baos = new ByteArrayOutputStream(); try { final Writer osw = new OutputStreamWriter(baos, StandardCharsets.UTF_8); - writeXML(osw); + this.writeXML(osw); osw.close(); return UTF8.String(baos.toByteArray()); } catch (final UnsupportedEncodingException e1) { @@ -976,7 +976,7 @@ dc_rights /** * merge documents: a helper method for all parsers that return multiple documents. - * Note : when docs contains more than one item, eventual icons in each docs are not merged in result doc, + * Note : when docs contains more than one item, eventual icons in each docs are not merged in result doc, * as their scope is limited to only one document. * @param location url of merged document * @param globalMime Mime type of merged document @@ -1036,7 +1036,7 @@ dc_rights if (doc.getTextLength() > 0) { if (docTextLength > 0) content.write('\n'); - InputStream textStream = doc.getTextStream(); + final InputStream textStream = doc.getTextStream(); try { docTextLength += FileUtils.copy(textStream, content); } catch (final IOException e) { @@ -1047,7 +1047,7 @@ dc_rights /* textStream can be a FileInputStream : we must close it to ensure releasing system resource */ textStream.close(); } - } catch (IOException e) { + } catch (final IOException e) { ConcurrentLog.warn("DOCUMENT", "Could not close text input stream"); } } @@ -1057,17 +1057,17 @@ dc_rights images.putAll(doc.getImages()); if (doc.lon() != 0.0 && doc.lat() != 0.0) { lon = doc.lon(); lat = doc.lat(); } if (date == null || doc.lastModified.after(date)) date = doc.lastModified; // set youngest lastModified date from doc collection - + if (doc.getDepth() < mindepth) mindepth = doc.getDepth(); if (doc.dc_language() != null) languages.add(doc.dc_language()); - + indexingDenied |= doc.indexingDenied; partiallyParsed |= doc.isPartiallyParsed(); } // clean up parser data for (final Document doc: docs) { - Object scraper = doc.getScraperObject(); + final Object scraper = doc.getScraperObject(); if (scraper instanceof ContentScraper) { final ContentScraper html = (ContentScraper) scraper; html.close(); @@ -1076,9 +1076,8 @@ dc_rights } // return consolidation - ArrayList<String> titlesa = new ArrayList<String>(); - titlesa.addAll(titles); - Document newDoc = new Document( + final ArrayList<String> titlesa = new ArrayList<>(titles); + final Document newDoc = new Document( location, globalMime, charset, @@ -1113,29 +1112,29 @@ dc_rights if (includeNofollow) { result.putAll(d.getHyperlinks()); } else { - for (Map.Entry<AnchorURL, String> entry: d.getHyperlinks().entrySet()) { + for (final Map.Entry<AnchorURL, String> entry: d.getHyperlinks().entrySet()) { if (!entry.getKey().attachedNofollow()) result.put(entry.getKey(), entry.getValue()); } } final Object scraper = d.getScraperObject(); if (scraper instanceof ContentScraper) { final ContentScraper html = (ContentScraper) scraper; - String refresh = html.getRefreshPath(); + final String refresh = html.getRefreshPath(); if (refresh != null && refresh.length() > 0) try {result.put(new AnchorURL(refresh), "refresh");} catch (final MalformedURLException e) {} - AnchorURL canonical = html.getCanonical(); + final AnchorURL canonical = html.getCanonical(); if (canonical != null) { result.put(canonical, CANONICAL_MARKER); } - for (AnchorURL u: html.getFrames()) result.put(u, FRAME_MARKER); - for (AnchorURL u: html.getIFrames()) result.put(u, IFRAME_MARKER); - for (AnchorURL u: html.getEmbeds().keySet()) result.put(u, EMBED_MARKER); + for (final AnchorURL u: html.getFrames()) result.put(u, FRAME_MARKER); + for (final AnchorURL u: html.getIFrames()) result.put(u, IFRAME_MARKER); + for (final AnchorURL u: html.getEmbeds().keySet()) result.put(u, EMBED_MARKER); } } return result; } public static Map<DigestURL, String> getImagelinks(final Document[] documents) { - final Map<DigestURL, String> result = new HashMap<DigestURL, String>(); + final Map<DigestURL, String> result = new HashMap<>(); for (final Document d: documents) { for (final ImageEntry imageReference : d.getImages().values()) { // construct a image name which contains the document title to enhance the search process for images @@ -1146,9 +1145,9 @@ dc_rights } public static Map<DigestURL, String> getAudiolinks(final Document[] documents) { - final Map<DigestURL, String> result = new HashMap<DigestURL, String>(); + final Map<DigestURL, String> result = new HashMap<>(); for (final Document d: documents) { - for (Map.Entry<AnchorURL, String> e: d.audiolinks.entrySet()) { + for (final Map.Entry<AnchorURL, String> e: d.audiolinks.entrySet()) { result.put(e.getKey(), description(d, e.getValue())); } } @@ -1156,9 +1155,9 @@ dc_rights } public static Map<DigestURL, String> getVideolinks(final Document[] documents) { - final Map<DigestURL, String> result = new HashMap<DigestURL, String>(); + final Map<DigestURL, String> result = new HashMap<>(); for (final Document d: documents) { - for (Map.Entry<AnchorURL, String> e: d.videolinks.entrySet()) { + for (final Map.Entry<AnchorURL, String> e: d.videolinks.entrySet()) { result.put(e.getKey(), description(d, e.getValue())); } } @@ -1166,9 +1165,9 @@ dc_rights } public static Map<DigestURL, String> getApplinks(final Document[] documents) { - final Map<DigestURL, String> result = new HashMap<DigestURL, String>(); + final Map<DigestURL, String> result = new HashMap<>(); for (final Document d: documents) { - for (Map.Entry<AnchorURL, String> e: d.applinks.entrySet()) { + for (final Map.Entry<AnchorURL, String> e: d.applinks.entrySet()) { result.put(e.getKey(), description(d, e.getValue())); } } @@ -1179,7 +1178,7 @@ dc_rights if (tagname == null || tagname.isEmpty()) { tagname = d.source.toTokens(); } - StringBuilder sb = new StringBuilder(60); + final StringBuilder sb = new StringBuilder(60); sb.append(d.dc_title()); if (d.dc_description().length > 0) { if (!d.dc_description()[0].equals(d.dc_title()) && sb.length() < Request.descrLength - tagname.length()) { diff --git a/source/net/yacy/document/content/DCEntry.java b/source/net/yacy/document/content/DCEntry.java index 728b75094..a326af31b 100644 --- a/source/net/yacy/document/content/DCEntry.java +++ b/source/net/yacy/document/content/DCEntry.java @@ -25,8 +25,6 @@ package net.yacy.document.content;
-import com.ibm.icu.util.ULocale;
-
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.net.MalformedURLException;
@@ -42,6 +40,8 @@ import java.util.TreeMap; import org.apache.solr.common.params.MultiMapSolrParams;
+import com.ibm.icu.util.ULocale;
+
import net.yacy.cora.date.ISO8601Formatter;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.lod.vocabulary.DublinCore;
@@ -65,7 +65,7 @@ public class DCEntry extends MultiMapSolrParams { public static final DCEntry poison = new DCEntry();
public DCEntry() {
- super(new TreeMap<String, String[]>((Collator) insensitiveCollator.clone()));
+ super(new TreeMap<>((Collator) insensitiveCollator.clone()));
}
public DCEntry(
@@ -77,7 +77,7 @@ public class DCEntry extends MultiMapSolrParams { double lat,
double lon
) {
- super(new TreeMap<String, String[]>((Collator) insensitiveCollator.clone()));
+ super(new TreeMap<>((Collator) insensitiveCollator.clone()));
this.getMap().put(DublinCore.Identifier.getURIref(), new String[]{url.toNormalform(true)});
this.getMap().put(DublinCore.Date.getURIref(), new String[]{ISO8601Formatter.FORMATTER.format(date)});
this.getMap().put(DublinCore.Title.getURIref(), new String[]{title});
@@ -124,7 +124,7 @@ public class DCEntry extends MultiMapSolrParams { }
/**
- * get Identifier (url) (so far only used for surrogate processing)
+ * get Identifier (url) (so far only used for pack processing)
* @param useRelationAsAlternative true = take relation if no identifier resolves to url
* @return this entry identifier url
*/
@@ -135,15 +135,15 @@ public class DCEntry extends MultiMapSolrParams { // <dc:identifier>10.1051/0004-6361/201117940</dc:identifier>
String u = this.get("url");
if (u == null) u = this.get("sku");
-
+
if (u == null) {
final String[] urls = this.getParams(DublinCore.Identifier.getURIref());
if (urls == null) {
- return useRelationAsAlternative ? getRelation() : null;
+ return useRelationAsAlternative ? this.getRelation() : null;
}
if (urls.length > 0) { // check best also with 1 in case it's not http urn
// select one that fits
- u = bestU(urls);
+ u = this.bestU(urls);
}
}
@@ -166,7 +166,7 @@ public class DCEntry extends MultiMapSolrParams { String[] urls = CommonPattern.SEMICOLON.split(u);
if (urls.length > 1) {
// select one that fits
- u = bestU(urls);
+ u = this.bestU(urls);
}
try {
return new DigestURL(u);
@@ -240,7 +240,7 @@ public class DCEntry extends MultiMapSolrParams { }
return l;
}
- if (l == null) l = getIdentifier(true).language(); // determine from identifier-url.TLD
+ if (l == null) l = this.getIdentifier(true).language(); // determine from identifier-url.TLD
if (l == null) return this.get("language");//from TLD
return l;
}
@@ -294,7 +294,7 @@ public class DCEntry extends MultiMapSolrParams { public List<String> getDescriptions() {
String[] t = this.getParams(DublinCore.Description.getURIref());
- List<String> descriptions = new ArrayList<String>();
+ List<String> descriptions = new ArrayList<>();
if (t == null) return descriptions;
for (String s: t) descriptions.add(stripCDATA(s));
return descriptions;
@@ -312,7 +312,7 @@ public class DCEntry extends MultiMapSolrParams { return CommonPattern.SEMICOLON.split(t);
}
tx = this.getParams(DublinCore.Subject.getURIref());
-
+
if (tx != null) {
for (int i = 0; i < tx.length; i++) {
tx[i] = stripCDATA(tx[i]);
@@ -345,39 +345,39 @@ public class DCEntry extends MultiMapSolrParams { }
public Document document() {
- HashSet<String> languages = new HashSet<String>();
- languages.add(getLanguage());
- List<String> t = new ArrayList<String>(1);
- t.add(getTitle());
-
+ HashSet<String> languages = new HashSet<>();
+ languages.add(this.getLanguage());
+ List<String> t = new ArrayList<>(1);
+ t.add(this.getTitle());
+
// for processing during indexing, embed entry as source scraperObject in a standard parserobj object
genericParser parserobj = new genericParser(); // init the simplest parser with DCEntry as source/scraperObject used during indexing
Document document = new Document(
- getIdentifier(true),
+ this.getIdentifier(true),
"text/html",
StandardCharsets.UTF_8.name(),
parserobj,
languages,
- getSubject(), // might be null
+ this.getSubject(), // might be null
t,
- getCreator(),
- getPublisher(),
+ this.getCreator(),
+ this.getPublisher(),
null,
- getDescriptions(),
- getLon(), getLat(),
- get(CollectionSchema.text_t.name(), ""),
+ this.getDescriptions(),
+ this.getLon(), this.getLat(),
+ this.get(CollectionSchema.text_t.name(), ""),
null,
null,
null,
false,
- getDate());
- document.setScraperObject(this); // TODO: used during indexing to access some possible but special YaCy meta tags in surrogate source ( <md:solrfilename>value ) -> optimize/find alternative
+ this.getDate());
+ document.setScraperObject(this); // TODO: used during indexing to access some possible but special YaCy meta tags in pack source ( <md:solrfilename>value ) -> optimize/find alternative
return document;
}
public void writeXML(OutputStreamWriter os) throws IOException {
- Document doc = document();
+ Document doc = this.document();
if (doc != null) {
doc.writeXML(os);
}
diff --git a/source/net/yacy/document/content/SurrogateReader.java b/source/net/yacy/document/content/XMLPackReader.java index 8f23ebca6..c6ed3a6bb 100644 --- a/source/net/yacy/document/content/SurrogateReader.java +++ b/source/net/yacy/document/content/XMLPackReader.java @@ -1,4 +1,4 @@ -// SurrogateReader.java
+// XMLPackReader.java
// (C) 2009 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany
// first published 15.04.2009 on http://yacy.net
//
@@ -7,7 +7,7 @@ // $LastChangedBy$
//
// LICENSE
-//
+//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
@@ -59,7 +59,7 @@ import net.yacy.crawler.CrawlStacker; import net.yacy.search.schema.CollectionConfiguration;
-public class SurrogateReader extends DefaultHandler implements Runnable {
+public class XMLPackReader extends DefaultHandler implements Runnable {
// definition of the surrogate main element
public final static String SURROGATES_MAIN_ELEMENT_NAME =
@@ -72,7 +72,7 @@ public class SurrogateReader extends DefaultHandler implements Runnable { public final static String SURROGATES_MAIN_ELEMENT_CLOSE =
"</" + SURROGATES_MAIN_ELEMENT_NAME + ">";
public final static SolrInputDocument POISON_DOCUMENT = new SolrInputDocument();
-
+
/** Maximum bytes number that can be unread on the underlying input stream */
private static final int PUSHBACK_SIZE = 1024;
@@ -89,7 +89,7 @@ public class SurrogateReader extends DefaultHandler implements Runnable { private final CollectionConfiguration configuration;
private final int concurrency;
- private static final ThreadLocal<SAXParser> tlSax = new ThreadLocal<SAXParser>();
+ private static final ThreadLocal<SAXParser> tlSax = new ThreadLocal<>();
private static SAXParser getParser() throws SAXException {
SAXParser parser = tlSax.get();
if (parser == null) {
@@ -103,11 +103,11 @@ public class SurrogateReader extends DefaultHandler implements Runnable { return parser;
}
- public SurrogateReader(final InputStream stream, int queueSize, CrawlStacker crawlStacker, CollectionConfiguration configuration, int concurrency) throws IOException {
+ public XMLPackReader(final InputStream stream, int queueSize, CrawlStacker crawlStacker, CollectionConfiguration configuration, int concurrency) throws IOException {
this(new PushbackInputStream(stream, PUSHBACK_SIZE), queueSize, crawlStacker, configuration, concurrency);
}
-
- public SurrogateReader(final PushbackInputStream stream, int queueSize, CrawlStacker crawlStacker, CollectionConfiguration configuration, int concurrency) throws IOException {
+
+ public XMLPackReader(final PushbackInputStream stream, int queueSize, CrawlStacker crawlStacker, CollectionConfiguration configuration, int concurrency) throws IOException {
this.crawlStacker = crawlStacker;
this.configuration = configuration;
this.concurrency = concurrency;
@@ -117,7 +117,7 @@ public class SurrogateReader extends DefaultHandler implements Runnable { this.elementName = null;
this.surrogates = new ArrayBlockingQueue<>(queueSize);
this.inputStream = stream;
-
+
try {
this.saxParser = getParser();
} catch (final SAXException e) {
@@ -125,18 +125,18 @@ public class SurrogateReader extends DefaultHandler implements Runnable { throw new IOException(e.getMessage());
}
}
-
+
@Override
public void run() {
// test the syntax of the stream by reading parts of the beginning
try {
BufferedReader br = new BufferedReader(new InputStreamReader(this.inputStream, StandardCharsets.UTF_8));
- if (isSolrDump()) {
+ if (this.isSolrDump()) {
String line;
while ((line = br.readLine()) != null) {
if (!line.startsWith("<doc>")) continue;
try {
- NamedList<Object> nl = new XMLResponseParser().processResponse(new StringReader("<result>" + line + "</result>")); //
+ NamedList<Object> nl = new XMLResponseParser().processResponse(new StringReader("<result>" + line + "</result>")); //
SolrDocument doc = (SolrDocument) nl.iterator().next().getValue();
// check if url is in accepted domain
@@ -182,7 +182,7 @@ public class SurrogateReader extends DefaultHandler implements Runnable { }
}
}
-
+
/**
* Check for format string in responseHeader "yacy.index.export.solr.xml"
* (introduced v1.92/9188 2017-04-30) or guess format by existing "<response>"
@@ -217,7 +217,7 @@ public class SurrogateReader extends DefaultHandler implements Runnable { }
return res;
}
-
+
@Override
public void startElement(final String uri, final String name, String tag, final Attributes atts) throws SAXException {
if (tag == null) return;
@@ -260,21 +260,21 @@ public class SurrogateReader extends DefaultHandler implements Runnable { this.buffer.setLength(0);
this.parsingValue = false;
} else if ("str".equals(tag) || "int".equals(tag) || "bool".equals(tag) || "long".equals(tag)){
- final String value = buffer.toString().trim();
+ final String value = this.buffer.toString().trim();
if (this.elementName != null) {
this.dcEntry.getMap().put(this.elementName, new String[]{value});
}
this.buffer.setLength(0);
this.parsingValue = false;
} else if ("value".equals(tag)) {
- final String value = buffer.toString().trim();
+ final String value = this.buffer.toString().trim();
if (this.elementName != null) {
this.dcEntry.getMap().put(this.elementName, new String[]{value});
}
this.buffer.setLength(0);
this.parsingValue = false;
} else if (tag.startsWith("dc:") || tag.startsWith("geo:") || tag.startsWith("md:")) {
- final String value = buffer.toString().trim();
+ final String value = this.buffer.toString().trim();
if (this.elementName != null && tag.equals(this.elementName)) {
Map<String,String[]> map = this.dcEntry.getMap();
String[] oldcontent = map.get(this.elementName);
@@ -294,8 +294,8 @@ public class SurrogateReader extends DefaultHandler implements Runnable { @Override
public void characters(final char ch[], final int start, final int length) {
- if (parsingValue) {
- buffer.append(ch, start, length);
+ if (this.parsingValue) {
+ this.buffer.append(ch, start, length);
}
}
diff --git a/source/net/yacy/document/content/dao/Dao.java b/source/net/yacy/document/content/dao/Dao.java index 62e43a1eb..c305cd143 100644 --- a/source/net/yacy/document/content/dao/Dao.java +++ b/source/net/yacy/document/content/dao/Dao.java @@ -89,7 +89,7 @@ public interface Dao { // export methods
- public int writeSurrogates(
+ public int writePacks(
BlockingQueue<DCEntry> queue,
File targetdir,
String versioninfo,
diff --git a/source/net/yacy/document/content/dao/PhpBB3Dao.java b/source/net/yacy/document/content/dao/PhpBB3Dao.java index 1c46d9524..9e759a61e 100644 --- a/source/net/yacy/document/content/dao/PhpBB3Dao.java +++ b/source/net/yacy/document/content/dao/PhpBB3Dao.java @@ -43,7 +43,7 @@ import java.util.concurrent.BlockingQueue; import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.document.content.DCEntry;
-import net.yacy.document.content.SurrogateReader;
+import net.yacy.document.content.XMLPackReader;
public class PhpBB3Dao implements Dao {
@@ -258,7 +258,7 @@ public class PhpBB3Dao implements Dao { }
@Override
- public int writeSurrogates(
+ public int writePacks(
BlockingQueue<DCEntry> queue,
File targetdir,
String versioninfo,
@@ -283,7 +283,7 @@ public class PhpBB3Dao implements Dao { if (outputfile.exists()) outputfile.delete();
outStream = new FileOutputStream(outputfiletmp);
osw = new OutputStreamWriter(new BufferedOutputStream(outStream), StandardCharsets.UTF_8);
- osw.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + SurrogateReader.SURROGATES_MAIN_ELEMENT_OPEN + "\n");
+ osw.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + XMLPackReader.SURROGATES_MAIN_ELEMENT_OPEN + "\n");
}
e.writeXML(osw);
c++;
@@ -298,7 +298,7 @@ public class PhpBB3Dao implements Dao { fc++;
}
}
- osw.write(SurrogateReader.SURROGATES_MAIN_ELEMENT_CLOSE + "\n");
+ osw.write(XMLPackReader.SURROGATES_MAIN_ELEMENT_CLOSE + "\n");
osw.close();
outStream.close();
osw = null;
@@ -353,7 +353,7 @@ public class PhpBB3Dao implements Dao { System.out.println("First entry : " + db.first());
System.out.println("Last entry : " + db.latest());
File targetdir = new File("x").getParentFile();
- db.writeSurrogates(db.query(0, -1, 100), targetdir, "id0-current", 3000);
+ db.writePacks(db.query(0, -1, 100), targetdir, "id0-current", 3000);
} catch (final Exception e) {
ConcurrentLog.logException(e);
}
diff --git a/source/net/yacy/document/importer/JsonListImporter.java b/source/net/yacy/document/importer/JsonListImporter.java index f82539837..928350c03 100644 --- a/source/net/yacy/document/importer/JsonListImporter.java +++ b/source/net/yacy/document/importer/JsonListImporter.java @@ -55,7 +55,7 @@ import net.yacy.document.Document; import net.yacy.document.LibraryProvider; import net.yacy.document.Tokenizer; import net.yacy.document.VocabularyScraper; -import net.yacy.document.content.SurrogateReader; +import net.yacy.document.content.XMLPackReader; import net.yacy.search.Switchboard; import net.yacy.search.schema.CollectionSchema; @@ -89,13 +89,13 @@ public class JsonListImporter extends Thread implements Importer { @Override public void run() { try { - this.processSurrogateJson(); + this.proceessPackJson(); } catch (final IOException e) { log.warn(e); } } - public void processSurrogateJson() throws IOException { + public void proceessPackJson() throws IOException { this.startTime = System.currentTimeMillis(); job = this; @@ -104,14 +104,14 @@ public class JsonListImporter extends Thread implements Importer { final BlockingQueue<SolrInputDocument> sidQueue = new ArrayBlockingQueue<>(concurrency * 2); final Thread[] indexer = new Thread[concurrency]; for (int t = 0; t < indexer.length; t++) { - indexer[t] = new Thread("Switchboard.processSurrogateJson-" + t) { + indexer[t] = new Thread("Switchboard.processPackJson-" + t) { @Override public void run() { final VocabularyScraper scraper = new VocabularyScraper(); SolrInputDocument sid; try { - while ((sid = sidQueue.take()) != SurrogateReader.POISON_DOCUMENT ) { - // enrich the surrogate + while ((sid = sidQueue.take()) != XMLPackReader.POISON_DOCUMENT ) { + // enrich the pack final String id = (String) sid.getFieldValue(CollectionSchema.id.getSolrFieldName()); final String text = (String) sid.getFieldValue(CollectionSchema.text_t.getSolrFieldName()); DigestURL rootURL; @@ -147,10 +147,10 @@ public class JsonListImporter extends Thread implements Importer { throw new IOException(e1.getMessage()); } if ((json.opt("index") != null && json.length() == 1) || json.length() == 0) continue; - final SolrInputDocument surrogate = new SolrInputDocument(); + final SolrInputDocument pack = new SolrInputDocument(); // set default values which act as constraints for a proper search - CollectionSchema.httpstatus_i.add(surrogate, 200); + CollectionSchema.httpstatus_i.add(pack, 200); // get fields for json object jsonreader: for (final String key: json.keySet()) { @@ -169,8 +169,8 @@ public class JsonListImporter extends Thread implements Importer { urlstub.add(b.urlstub(true, true)); protocol.add(b.getProtocol()); } - CollectionSchema.inboundlinks_urlstub_sxt.add(surrogate, urlstub); - CollectionSchema.inboundlinks_protocol_sxt.add(surrogate, protocol); + CollectionSchema.inboundlinks_urlstub_sxt.add(pack, urlstub); + CollectionSchema.inboundlinks_protocol_sxt.add(pack, protocol); continue jsonreader; } if (key.equals("outboundlinks_sxt")) { @@ -182,8 +182,8 @@ public class JsonListImporter extends Thread implements Importer { urlstub.add(b.urlstub(true, true)); protocol.add(b.getProtocol()); } - CollectionSchema.outboundlinks_urlstub_sxt.add(surrogate, urlstub); - CollectionSchema.outboundlinks_protocol_sxt.add(surrogate, protocol); + CollectionSchema.outboundlinks_urlstub_sxt.add(pack, urlstub); + CollectionSchema.outboundlinks_protocol_sxt.add(pack, protocol); continue jsonreader; } if (key.equals("images_sxt")) { @@ -195,8 +195,8 @@ public class JsonListImporter extends Thread implements Importer { urlstub.add(b.urlstub(true, true)); protocol.add(b.getProtocol()); } - CollectionSchema.images_urlstub_sxt.add(surrogate, urlstub); - CollectionSchema.images_protocol_sxt.add(surrogate, protocol); + CollectionSchema.images_urlstub_sxt.add(pack, urlstub); + CollectionSchema.images_protocol_sxt.add(pack, protocol); continue jsonreader; } @@ -208,31 +208,31 @@ public class JsonListImporter extends Thread implements Importer { } final List<Object> list = new ArrayList<>(); for (int i = 0; i < a.length(); i++) list.add(a.opt(i)); - ctype.add(surrogate, list); + ctype.add(pack, list); } else { // first handle exceptional keys / maybe patch for other systems + other names if (key.equals("url_s") || key.equals("sku")) { // patch yacy grid altered schema (yacy grid does not have IDs any more, but they can be re-computed here) final DigestURL durl = new DigestURL(o.toString()); final String id = ASCII.String(durl.hash()); - surrogate.setField(CollectionSchema.sku.getSolrFieldName(), durl.toNormalform(true)); - surrogate.setField(CollectionSchema.id.getSolrFieldName(), id); - surrogate.setField(CollectionSchema.host_s.getSolrFieldName(), durl.getHost()); - surrogate.setField(CollectionSchema.host_id_s.getSolrFieldName(), id.substring(6)); + pack.setField(CollectionSchema.sku.getSolrFieldName(), durl.toNormalform(true)); + pack.setField(CollectionSchema.id.getSolrFieldName(), id); + pack.setField(CollectionSchema.host_s.getSolrFieldName(), durl.getHost()); + pack.setField(CollectionSchema.host_id_s.getSolrFieldName(), id.substring(6)); continue jsonreader; } if (key.equals("description")) { // in YaCy descriptions are full-text indexed and also multi-value fields final List<Object> descriptions = new ArrayList<>(); descriptions.add(o.toString()); - CollectionSchema.description_txt.add(surrogate, descriptions); + CollectionSchema.description_txt.add(pack, descriptions); continue jsonreader; } if (key.equals("referrer_url_s")) { // same patch as for urls which require re-calculation of id's; in this case we store the id only! final DigestURL durl = new DigestURL(o.toString()); final String id = ASCII.String(durl.hash()); - surrogate.setField(CollectionSchema.referrer_id_s.getSolrFieldName(), id); + pack.setField(CollectionSchema.referrer_id_s.getSolrFieldName(), id); continue jsonreader; } @@ -246,26 +246,26 @@ public class JsonListImporter extends Thread implements Importer { // patch date into something that Solr can understand final String d = o.toString(); // i.e. Wed Apr 01 02:00:00 CEST 2020 final Date dd = d == null || d.length() == 0 ? null : AbstractFormatter.parseAny(d); - if (dd != null) surrogate.setField(ctype.getSolrFieldName(), ISO8601Formatter.FORMATTER.format(dd)); // solr dateTime is ISO8601 format + if (dd != null) pack.setField(ctype.getSolrFieldName(), ISO8601Formatter.FORMATTER.format(dd)); // solr dateTime is ISO8601 format continue jsonreader; } // check if required fields are still missing and compute them - if (!surrogate.containsKey(CollectionSchema.host_s.getSolrFieldName())) { - final String durls = (String) surrogate.getFieldValue(CollectionSchema.sku.getSolrFieldName()); + if (!pack.containsKey(CollectionSchema.host_s.getSolrFieldName())) { + final String durls = (String) pack.getFieldValue(CollectionSchema.sku.getSolrFieldName()); if (durls != null) { final DigestURL durl = new DigestURL(durls); - surrogate.setField(CollectionSchema.host_s.getSolrFieldName(), durl.getHost()); + pack.setField(CollectionSchema.host_s.getSolrFieldName(), durl.getHost()); } } // regular situation, just read content of field - surrogate.setField(key, o.toString()); + pack.setField(key, o.toString()); } } try { - sidQueue.put(surrogate); + sidQueue.put(pack); } catch (final InterruptedException e) { e.printStackTrace(); } @@ -277,7 +277,7 @@ public class JsonListImporter extends Thread implements Importer { // finish indexing threads by giving them poison for (int t = 0; t < indexer.length; t++) { - try {sidQueue.put(SurrogateReader.POISON_DOCUMENT);} catch (final InterruptedException e) {} + try {sidQueue.put(XMLPackReader.POISON_DOCUMENT);} catch (final InterruptedException e) {} } // wait until indexer threads are finished for (int t = 0; t < indexer.length; t++) { @@ -286,7 +286,7 @@ public class JsonListImporter extends Thread implements Importer { if (this.deletewhendone) this.inputFile.delete(); - log.info("finished processing json surrogate: " + ((System.currentTimeMillis() - this.startTime) / 1000) + " seconds"); + log.info("finished processing json pack: " + ((System.currentTimeMillis() - this.startTime) / 1000) + " seconds"); } public void quit() { diff --git a/source/net/yacy/document/importer/MediawikiImporter.java b/source/net/yacy/document/importer/MediawikiImporter.java index bf2260b97..069a28e54 100644 --- a/source/net/yacy/document/importer/MediawikiImporter.java +++ b/source/net/yacy/document/importer/MediawikiImporter.java @@ -52,6 +52,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.zip.GZIPInputStream; +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; + import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.document.id.AnchorURL; import net.yacy.cora.document.id.DigestURL; @@ -67,12 +69,10 @@ import net.yacy.document.Document; import net.yacy.document.Parser; import net.yacy.document.TextParser; import net.yacy.document.VocabularyScraper; -import net.yacy.document.content.SurrogateReader; +import net.yacy.document.content.XMLPackReader; import net.yacy.document.parser.html.TagValency; import net.yacy.kelondro.util.NamePrefixThreadFactory; -import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; - /* * this class provides data structures to read a mediawiki dump file in xml format @@ -138,7 +138,7 @@ public class MediawikiImporter extends Thread implements Importer { @Override public int speed() { if (this.count == 0) return 0; - return (int) (this.count / Math.max(1L, runningTime() )); + return (int) (this.count / Math.max(1L, this.runningTime() )); } /** @@ -146,7 +146,7 @@ public class MediawikiImporter extends Thread implements Importer { */ @Override public long remainingTime() { - return Math.max(0, this.approxdocs - this.count) / Math.max(1, speed() ); + return Math.max(0, this.approxdocs - this.count) / Math.max(1, this.speed() ); } @Override @@ -161,8 +161,8 @@ public class MediawikiImporter extends Thread implements Importer { final int threads = Math.max(2, Runtime.getRuntime().availableProcessors() - 1); // out keeps a outputfile open until poisened, to make sure underlaying thread gets the end condition // regardless of any exception (e.g. eof memory) a add(poison) is added to the most outer final block - final BlockingQueue<wikiparserrecord> out = new ArrayBlockingQueue<wikiparserrecord>(threads * 10); - final wikiparserrecord poison = newRecord(); + final BlockingQueue<wikiparserrecord> out = new ArrayBlockingQueue<>(threads * 10); + final wikiparserrecord poison = this.newRecord(); BufferedReader reader = null; try { String targetstub = this.sourcefile.getFileName(); @@ -179,7 +179,7 @@ public class MediawikiImporter extends Thread implements Importer { StringBuilder sb = new StringBuilder(); boolean page = false, text = false; String title = null; - final BlockingQueue<wikiparserrecord> in = new ArrayBlockingQueue<wikiparserrecord>(threads * 10); + final BlockingQueue<wikiparserrecord> in = new ArrayBlockingQueue<>(threads * 10); final ExecutorService service = Executors.newCachedThreadPool( new NamePrefixThreadFactory(MediawikiImporter.class.getSimpleName() + ".convertConsumer")); final convertConsumer[] consumers = new convertConsumer[threads]; @@ -222,7 +222,7 @@ public class MediawikiImporter extends Thread implements Importer { ConcurrentLog.info("WIKITRANSLATION", "ERROR: " + title + " has empty content"); continue; } - record = newRecord(this.hostport, this.urlStub, title, sb); + record = this.newRecord(this.hostport, this.urlStub, title, sb); try { in.put(record); this.count++; @@ -243,7 +243,7 @@ public class MediawikiImporter extends Thread implements Importer { ConcurrentLog.info("WIKITRANSLATION", "ERROR: " + title + " has empty content"); continue; } - record = newRecord(this.hostport, this.urlStub, title, sb); + record = this.newRecord(this.hostport, this.urlStub, title, sb); try { in.put(record); this.count++; @@ -309,7 +309,7 @@ public class MediawikiImporter extends Thread implements Importer { public static class indexMaker extends Thread { File mediawikixml; - + public indexMaker(final File mediawikixml) { super("MediawikiImporter.indexMaker " + mediawikixml != null ? mediawikixml.getName() : ""); this.mediawikixml = mediawikixml; @@ -380,7 +380,7 @@ public class MediawikiImporter extends Thread implements Importer { int count; public indexProducer(final int bufferCount, final File indexFile) throws IOException { - this.entries = new ArrayBlockingQueue<wikisourcerecord>(bufferCount); + this.entries = new ArrayBlockingQueue<>(bufferCount); this.out = new PrintWriter(new BufferedWriter(new FileWriter(indexFile))); this.count = 0; this.out.println("<index>"); @@ -430,7 +430,7 @@ public class MediawikiImporter extends Thread implements Importer { private int count; public wikiConsumer(final int bufferCount, final indexProducer producer) { - this.entries = new ArrayBlockingQueue<wikiraw>(bufferCount); + this.entries = new ArrayBlockingQueue<>(bufferCount); this.producer = producer; this.count = 0; } @@ -537,7 +537,7 @@ public class MediawikiImporter extends Thread implements Importer { public void genDocument() throws Parser.Failure { try { this.url = new AnchorURL(this.urlStub + this.title); - final Document[] parsed = TextParser.parseSource(this.url, "text/html", StandardCharsets.UTF_8.name(), TagValency.EVAL, new HashSet<String>(), new VocabularyScraper(), 0, 1, UTF8.getBytes(this.html)); + final Document[] parsed = TextParser.parseSource(this.url, "text/html", StandardCharsets.UTF_8.name(), TagValency.EVAL, new HashSet<>(), new VocabularyScraper(), 0, 1, UTF8.getBytes(this.html)); this.document = Document.mergeDocuments(this.url, "text/html", parsed); // the wiki parser is not able to find the proper title in the source text, so it must be set here this.document.setTitle(this.title); @@ -730,13 +730,13 @@ public class MediawikiImporter extends Thread implements Importer { // start writing a new file this.outputfilename = this.targetstub + "." + this.fc + ".xml.prt"; this.osw = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(new File(this.targetdir, this.outputfilename))), StandardCharsets.UTF_8); - this.osw.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + SurrogateReader.SURROGATES_MAIN_ELEMENT_OPEN + "\n"); + this.osw.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + XMLPackReader.SURROGATES_MAIN_ELEMENT_OPEN + "\n"); } ConcurrentLog.info("WIKITRANSLATION", "[CONSUME] Title: " + record.title); record.document.writeXML(this.osw); this.rc++; if (this.rc >= 10000) { - this.osw.write(SurrogateReader.SURROGATES_MAIN_ELEMENT_CLOSE + "\n"); + this.osw.write(XMLPackReader.SURROGATES_MAIN_ELEMENT_CLOSE + "\n"); this.osw.close(); final String finalfilename = this.targetstub + "." + this.fc + ".xml"; new File(this.targetdir, this.outputfilename).renameTo(new File(this.targetdir, finalfilename)); @@ -744,7 +744,7 @@ public class MediawikiImporter extends Thread implements Importer { this.fc++; this.outputfilename = this.targetstub + "." + this.fc + ".xml.prt"; this.osw = new OutputStreamWriter(new BufferedOutputStream(new FileOutputStream(new File(this.targetdir, this.outputfilename))), StandardCharsets.UTF_8); - this.osw.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + SurrogateReader.SURROGATES_MAIN_ELEMENT_OPEN + "\n"); + this.osw.write("<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + XMLPackReader.SURROGATES_MAIN_ELEMENT_OPEN + "\n"); } } } catch (final InterruptedException e) { @@ -757,8 +757,8 @@ public class MediawikiImporter extends Thread implements Importer { ConcurrentLog.logException(e); } finally { try { - if (osw != null) { // maybe null on poison (immediately) - this.osw.write(SurrogateReader.SURROGATES_MAIN_ELEMENT_CLOSE + "\n"); + if (this.osw != null) { // maybe null on poison (immediately) + this.osw.write(XMLPackReader.SURROGATES_MAIN_ELEMENT_CLOSE + "\n"); this.osw.close(); final String finalfilename = this.targetstub + "." + this.fc + ".xml"; new File(this.targetdir, this.outputfilename).renameTo(new File(this.targetdir, finalfilename)); @@ -789,7 +789,7 @@ public class MediawikiImporter extends Thread implements Importer { // java -Xmx2000m -cp classes:lib/bzip2.jar // de.anomic.tools.mediawikiIndex -convert // DATA/HTCACHE/dewiki-20090311-pages-articles.xml.bz2 - // DATA/SURROGATES/in/ http://de.wikipedia.org/wiki/ + // DATA/PACKS/load/ http://de.wikipedia.org/wiki/ if (s[0].equals("-convert")) { if(s.length < 3) { diff --git a/source/net/yacy/document/importer/OAIPMHImporter.java b/source/net/yacy/document/importer/OAIPMHImporter.java index eae37b0ae..840527fab 100644 --- a/source/net/yacy/document/importer/OAIPMHImporter.java +++ b/source/net/yacy/document/importer/OAIPMHImporter.java @@ -47,9 +47,9 @@ public class OAIPMHImporter extends Thread implements Importer, Comparable<OAIPM private static int importerCounter = Integer.MAX_VALUE; private static final Object N = new Object(); - public static ConcurrentHashMap<OAIPMHImporter, Object> startedJobs = new ConcurrentHashMap<OAIPMHImporter, Object>(); - public static ConcurrentHashMap<OAIPMHImporter, Object> runningJobs = new ConcurrentHashMap<OAIPMHImporter, Object>(); - public static ConcurrentHashMap<OAIPMHImporter, Object> finishedJobs = new ConcurrentHashMap<OAIPMHImporter, Object>(); + public static ConcurrentHashMap<OAIPMHImporter, Object> startedJobs = new ConcurrentHashMap<>(); + public static ConcurrentHashMap<OAIPMHImporter, Object> runningJobs = new ConcurrentHashMap<>(); + public static ConcurrentHashMap<OAIPMHImporter, Object> finishedJobs = new ConcurrentHashMap<>(); private final LoaderDispatcher loader; private DigestURL source; @@ -124,7 +124,7 @@ public class OAIPMHImporter extends Thread implements Importer, Comparable<OAIPM @Override public int speed() { - return (int) (1000L * (count()) / runningTime()); + return (int) (1000L * (this.count()) / this.runningTime()); } @Override @@ -137,7 +137,7 @@ public class OAIPMHImporter extends Thread implements Importer, Comparable<OAIPM this.message = "loading first part of records"; while (true) { try { - OAIPMHLoader oailoader = new OAIPMHLoader(this.loader, this.source, Switchboard.getSwitchboard().surrogatesInPath, this.agent); + final OAIPMHLoader oailoader = new OAIPMHLoader(this.loader, this.source, Switchboard.getSwitchboard().packsLoadPath, this.agent); this.completeListSize = Math.max(this.completeListSize, oailoader.getResumptionToken().getCompleteListSize()); this.chunkCount++; this.recordsCount += oailoader.getResumptionToken().getRecordCounter(); @@ -170,7 +170,7 @@ public class OAIPMHImporter extends Thread implements Importer, Comparable<OAIPM if (this == obj) return true; if (obj == null) return false; if (!(obj instanceof OAIPMHImporter)) return false; - OAIPMHImporter other = (OAIPMHImporter) obj; + final OAIPMHImporter other = (OAIPMHImporter) obj; return this.compareTo(other) == 0; } @@ -184,26 +184,26 @@ public class OAIPMHImporter extends Thread implements Importer, Comparable<OAIPM /** * get a map for already loaded oai-pmh servers and their latest access date - * @param surrogatesIn - * @param surrogatesOut + * @param packsLoad + * @param packsLoaded * @return a map where the key is the hostID of the servers and the value is the last access date */ @SuppressWarnings("unchecked") - public static Map<String, Date> getLoadedOAIServer(File surrogatesIn, File surrogatesOut) { - Map<String, Date> map = getLoadedOAIServer(surrogatesOut); - map.putAll((Map<? extends String, ? extends Date>) getLoadedOAIServer(surrogatesIn).entrySet()); + public static Map<String, Date> getLoadedOAIServer(File packsLoad, File packsLoaded) { + final Map<String, Date> map = getLoadedOAIServer(packsLoaded); + map.putAll((Map<? extends String, ? extends Date>) getLoadedOAIServer(packsLoad).entrySet()); return map; } - private static Map<String, Date> getLoadedOAIServer(File surrogates) { - HashMap<String, Date> map = new HashMap<String, Date>(); + private static Map<String, Date> getLoadedOAIServer(File packs) { + final HashMap<String, Date> map = new HashMap<>(); //oaipmh_opus.bsz-bw.de_20091102113118728.xml - for (String s: surrogates.list()) { + for (final String s: packs.list()) { if (s.startsWith(filenamePrefix) && s.endsWith(".xml") && s.charAt(s.length() - 22) == filenameSeparationChar) { try { - Date fd = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(s.substring(s.length() - 21, s.length() - 4), 0).getTime(); - String hostID = s.substring(7, s.length() - 22); - Date md = map.get(hostID); + final Date fd = GenericFormatter.SHORT_MILSEC_FORMATTER.parse(s.substring(s.length() - 21, s.length() - 4), 0).getTime(); + final String hostID = s.substring(7, s.length() - 22); + final Date md = map.get(hostID); if (md == null || fd.after(md)) map.put(hostID, fd); } catch (final ParseException e) { ConcurrentLog.logException(e); diff --git a/source/net/yacy/document/importer/WarcImporter.java b/source/net/yacy/document/importer/WarcImporter.java index d5b0af059..14ce06bec 100644 --- a/source/net/yacy/document/importer/WarcImporter.java +++ b/source/net/yacy/document/importer/WarcImporter.java @@ -66,10 +66,10 @@ import net.yacy.server.http.ChunkedInputStream; * wget "https://yacy.net" --mirror --warc-file=yacy.net
*
* The result is a compressed warc file named "yacy.net.warc.gz".
- * To index the content, it can be copied to the surrogate input path:
- * cp yacy.net.warc.gz DATA/SURROGATES/in/
+ * To index the content, it can be copied to the pack input path:
+ * cp yacy.net.warc.gz DATA/PACKS/load/
*
- * after processing, that warc file is moved to DATA/SURROGATES/out/
+ * after processing, that warc file is moved to DATA/PACKS/loaded/
*/
public class WarcImporter extends Thread implements Importer {
@@ -165,15 +165,15 @@ public class WarcImporter extends Thread implements Importer { requestHeader.referer() == null ? null : requestHeader.referer().hash(),
"warc",
responseHeader.lastModified(),
- Switchboard.getSwitchboard().crawler.defaultSurrogateProfile.handle(),
+ Switchboard.getSwitchboard().crawler.defaultPackProfile.handle(),
0,
- Switchboard.getSwitchboard().crawler.defaultSurrogateProfile.timezoneOffset());
+ Switchboard.getSwitchboard().crawler.defaultPackProfile.timezoneOffset());
final Response response = new Response(
request,
requestHeader,
responseHeader,
- Switchboard.getSwitchboard().crawler.defaultSurrogateProfile,
+ Switchboard.getSwitchboard().crawler.defaultPackProfile,
false,
content
);
@@ -239,7 +239,7 @@ public class WarcImporter extends Thread implements Importer { @Override
public int speed() {
if (this.recordCnt == 0) return 0;
- return (int) (this.recordCnt / Math.max(0L, runningTime() ));
+ return (int) (this.recordCnt / Math.max(0L, this.runningTime() ));
}
/**
@@ -261,7 +261,7 @@ public class WarcImporter extends Thread implements Importer { if (this.consumed == 0) {
return 0;
}
- long speed = this.consumed / runningTime();
+ long speed = this.consumed / this.runningTime();
return (this.sourceSize - this.consumed) / speed;
}
diff --git a/source/net/yacy/document/importer/ZimImporter.java b/source/net/yacy/document/importer/ZimImporter.java index 59706a745..20e946d27 100644 --- a/source/net/yacy/document/importer/ZimImporter.java +++ b/source/net/yacy/document/importer/ZimImporter.java @@ -144,15 +144,15 @@ public class ZimImporter extends Thread implements Importer { null, // referrerhash the hash of the referrer URL de.title, // name the name of the document to crawl null, // appdate the time when the url was first time appeared - sb.crawler.defaultSurrogateProfile.handle(), // profileHandle the name of the prefetch profile. This must not be null! + sb.crawler.defaultPackProfile.handle(), // profileHandle the name of the prefetch profile. This must not be null! 0, // depth the crawling depth of the entry - sb.crawler.defaultSurrogateProfile.timezoneOffset() // timezone offset + sb.crawler.defaultPackProfile.timezoneOffset() // timezone offset ); final Response response = new Response( request, requestHeader, responseHeader, - Switchboard.getSwitchboard().crawler.defaultSurrogateProfile, + Switchboard.getSwitchboard().crawler.defaultPackProfile, false, b ); diff --git a/source/net/yacy/htroot/ContentIntegrationPHPBB3_p.java b/source/net/yacy/htroot/ContentIntegrationPHPBB3_p.java index 0c4c52cd2..b3d284878 100644 --- a/source/net/yacy/htroot/ContentIntegrationPHPBB3_p.java +++ b/source/net/yacy/htroot/ContentIntegrationPHPBB3_p.java @@ -108,7 +108,7 @@ public class ContentIntegrationPHPBB3_p { dbpw
);
- final int files = db.writeSurrogates(db.query(0, -1, 100), sb.surrogatesInPath, "fullexport-" + GenericFormatter.SHORT_SECOND_FORMATTER.format(), ppf);
+ final int files = db.writePacks(db.query(0, -1, 100), sb.packsLoadPath, "fullexport-" + GenericFormatter.SHORT_SECOND_FORMATTER.format(), ppf);
prop.put("export", 1);
prop.put("export_files", files);
db.close();
diff --git a/source/net/yacy/htroot/CrawlResults.java b/source/net/yacy/htroot/CrawlResults.java index 2c10be06a..cf680f1fa 100644 --- a/source/net/yacy/htroot/CrawlResults.java +++ b/source/net/yacy/htroot/CrawlResults.java @@ -101,7 +101,7 @@ public class CrawlResults { tabletype == EventOrigin.LOCAL_CRAWLING &&
ResultURLs.getStackSize(EventOrigin.LOCAL_CRAWLING) == 0) {
// the main menu does a request to the local crawler page, but in case this table is empty, the overview page is shown
- tabletype = (ResultURLs.getStackSize(EventOrigin.SURROGATES) == 0) ? EventOrigin.UNKNOWN : EventOrigin.SURROGATES;
+ tabletype = (ResultURLs.getStackSize(EventOrigin.PACKS) == 0) ? EventOrigin.UNKNOWN : EventOrigin.PACKS;
}
// check if authorization is needed and/or given
@@ -144,7 +144,7 @@ public class CrawlResults { final String domain = post.get("domain", null);
if (domain != null) {
selectedblacklist = post.get("blacklistname");
- final Set<String> hostnames = new HashSet<String>();
+ final Set<String> hostnames = new HashSet<>();
hostnames.add(domain);
sb.index.fulltext().deleteStaleDomainNames(hostnames, null);
ResultURLs.deleteDomain(tabletype, domain);
diff --git a/source/net/yacy/htroot/IndexExportImportSolr_p.java b/source/net/yacy/htroot/IndexExportImportSolr_p.java new file mode 100644 index 000000000..ce5225d30 --- /dev/null +++ b/source/net/yacy/htroot/IndexExportImportSolr_p.java @@ -0,0 +1,103 @@ +// IndexExportImportSolr_p.java +// ----------------------- +// (C) 2004-2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 2004 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.htroot; + +import java.io.File; +import java.util.List; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; + +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; +import net.yacy.search.index.Segment; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; + +public class IndexExportImportSolr_p { + + public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + // return variable that accumulates replacements + final Switchboard sb = (Switchboard) env; + + final serverObjects prop = new serverObjects(); + + final Segment segment = sb.index; + // we have two counts of document: total number and such that are exportable with status code 200 + final long ucount = segment.fulltext().collectionSize(); + + // set default values + prop.put("indexdump", 0); + prop.put("indexRestore", 0); + prop.put("dumprestore", 1); + prop.put("dumprestore_dumpRestoreEnabled", sb.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, + SwitchboardConstants.CORE_SERVICE_FULLTEXT_DEFAULT)); + List<File> dumpFiles = segment.fulltext().dumpFiles(); + prop.put("dumprestore_dumpfile", dumpFiles.size() == 0 ? "" : dumpFiles.get(dumpFiles.size() - 1).getAbsolutePath()); + prop.put("dumprestore_optimizemax", 10); + prop.putNum("ucount", ucount); + + if (post == null || env == null) { + return prop; // nothing to do + } + + if (post.containsKey("indexdump")) { + try { + final File dump = segment.fulltext().dumpEmbeddedSolr(); + prop.put("indexdump", 1); + prop.put("indexdump_dumpfile", dump.getAbsolutePath()); + dumpFiles = segment.fulltext().dumpFiles(); + prop.put("dumprestore_dumpfile", dumpFiles.size() == 0 ? "" : dumpFiles.get(dumpFiles.size() - 1).getAbsolutePath()); + // sb.tables.recordAPICall(post, "IndexExport_p.html", WorkTables.TABLE_API_TYPE_STEERING, "solr dump generation"); + } catch(final SolrException e) { + if(ErrorCode.SERVICE_UNAVAILABLE.code == e.code()) { + prop.put("indexdump", 2); + } else { + prop.put("indexdump", 3); + } + } + } + + if (post.containsKey("indexrestore")) { + try { + final File dump = new File(post.get("dumpfile", "")); + segment.fulltext().restoreEmbeddedSolr(dump); + prop.put("indexRestore", 1); + } catch(final SolrException e) { + if(ErrorCode.SERVICE_UNAVAILABLE.code == e.code()) { + prop.put("indexRestore", 2); + } else { + prop.put("indexRestore", 3); + } + } + } + + // insert constants + prop.putNum("ucount", ucount); + // return rewrite properties + return prop; + } + +}
\ No newline at end of file diff --git a/source/net/yacy/htroot/IndexExport_p.java b/source/net/yacy/htroot/IndexExport_p.java index 667ba5711..0b5d44e0c 100644 --- a/source/net/yacy/htroot/IndexExport_p.java +++ b/source/net/yacy/htroot/IndexExport_p.java @@ -25,12 +25,15 @@ package net.yacy.htroot; import java.io.File; import java.io.IOException; -import java.util.List; +import java.util.Date; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import net.yacy.cora.date.GenericFormatter; +import net.yacy.cora.federate.solr.connector.AbstractSolrConnector; import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.util.ConcurrentLog; import net.yacy.data.WorkTables; import net.yacy.search.Switchboard; import net.yacy.search.SwitchboardConstants; @@ -57,18 +60,8 @@ public class IndexExport_p { } catch (final IOException e1) {} // set default values - prop.put("otherHosts", ""); prop.put("reload", 0); - prop.put("indexdump", 0); - prop.put("indexRestore", 0); prop.put("lurlexport", 0); - prop.put("reload", 0); - prop.put("dumprestore", 1); - prop.put("dumprestore_dumpRestoreEnabled", sb.getConfigBool(SwitchboardConstants.CORE_SERVICE_FULLTEXT, - SwitchboardConstants.CORE_SERVICE_FULLTEXT_DEFAULT)); - List<File> dumpFiles = segment.fulltext().dumpFiles(); - prop.put("dumprestore_dumpfile", dumpFiles.size() == 0 ? "" : dumpFiles.get(dumpFiles.size() - 1).getAbsolutePath()); - prop.put("dumprestore_optimizemax", 10); prop.putNum("ucount", ucount); prop.putNum("ucount200", ucount200); @@ -116,12 +109,9 @@ public class IndexExport_p { final boolean text = fname.startsWith("text"); if (fname.endsWith("text")) format = Fulltext.ExportFormat.text; if (fname.endsWith("html")) format = Fulltext.ExportFormat.html; - if (fname.endsWith("rss")) format = Fulltext.ExportFormat.rss; - if (fname.endsWith("solr")) format = Fulltext.ExportFormat.solr; - if (fname.endsWith("elasticsearch")) format = Fulltext.ExportFormat.elasticsearch; final String filter = post.get("exportfilter", ".*"); - final String query = post.get("exportquery", "*:*"); + String query = post.get("exportquery", "*:*"); final int maxseconds = post.getInt("exportmaxseconds", -1); long maxChunkSize = post.getLong("maxchunksize", Long.MAX_VALUE); if (maxChunkSize <= 0) maxChunkSize = Long.MAX_VALUE; @@ -133,7 +123,79 @@ public class IndexExport_p { // start the export try { - export = sb.index.fulltext().export(format, filter, query, maxseconds, new File(path), dom, text, maxChunkSize, minified); + File filepath = new File(path); + + // modify query according to maxseconds + final long now = System.currentTimeMillis(); + if (maxseconds > 0) { + final long from = now - maxseconds * 1000L; + final String nowstr = new Date(now).toInstant().toString(); + final String fromstr = new Date(from).toInstant().toString(); + final String dateq = CollectionSchema.load_date_dt.getSolrFieldName() + ":[" + fromstr + " TO " + nowstr + "]"; + query = query == null || AbstractSolrConnector.CATCHALL_QUERY.equals(query) ? dateq : query + " AND " + dateq; + } else { + query = query == null? AbstractSolrConnector.CATCHALL_QUERY : query; + } + + // check the oldest and latest entry in the index for this query + SolrDocumentList firstdoclist, lastdoclist; + Object firstdateobject, lastdateobject; + firstdoclist = sb.index.fulltext().getDefaultConnector().getDocumentListByQuery( + query, CollectionSchema.load_date_dt.getSolrFieldName() + " asc", 0, 1,CollectionSchema.load_date_dt.getSolrFieldName()); + lastdoclist = sb.index.fulltext().getDefaultConnector().getDocumentListByQuery( + query, CollectionSchema.load_date_dt.getSolrFieldName() + " desc", 0, 1,CollectionSchema.load_date_dt.getSolrFieldName()); + + final long doccount; + final Date firstdate, lastdate; + if (firstdoclist.size() == 0 || lastdoclist.size() == 0) { + /* Now check again the number of documents without sorting, for compatibility with old fields indexed without DocValues fields (prior to YaCy 1.90) + * When the local Solr index contains such old documents, requests with sort query return nothing and trace in logs + * "java.lang.IllegalStateException: unexpected docvalues type NONE for field..." */ + doccount = sb.index.fulltext().getDefaultConnector().getCountByQuery(query); + if(doccount == 0) { + /* Finally no document to export was found */ + throw new IOException("number of exported documents == 0"); + } + /* we use default date values just to generate a proper dump file path */ + firstdate = new Date(0); + lastdate = new Date(0); + + } else { + doccount = firstdoclist.getNumFound(); + + // create the export name + final SolrDocument firstdoc = firstdoclist.get(0); + final SolrDocument lastdoc = lastdoclist.get(0); + firstdateobject = firstdoc.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName()); + lastdateobject = lastdoc.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName()); + + /* When firstdate or lastdate is null, we use a default one just to generate a proper dump file path + * This should not happen because load_date_dt field is mandatory in the main Solr schema, + * but for some reason some documents might end up here with an empty load_date_dt field value */ + if(firstdateobject instanceof Date) { + firstdate = (Date) firstdateobject; + } else { + ConcurrentLog.warn("Fulltext", "The required field " + CollectionSchema.load_date_dt.getSolrFieldName() + " is empty on document with id : " + + firstdoc.getFieldValue(CollectionSchema.id.getSolrFieldName())); + firstdate = new Date(0); + } + if(lastdateobject instanceof Date) { + lastdate = (Date) lastdateobject; + } else { + ConcurrentLog.warn("Fulltext", "The required field " + CollectionSchema.load_date_dt.getSolrFieldName() + " is empty on document with id : " + + lastdoc.getFieldValue(CollectionSchema.id.getSolrFieldName())); + lastdate = new Date(0); + } + } + + final String filename = SwitchboardConstants.YACY_PACK_PREFIX + + "f" + GenericFormatter.SHORT_MINUTE_FORMATTER.format(firstdate) + "_" + + "l" + GenericFormatter.SHORT_MINUTE_FORMATTER.format(lastdate) + "_" + + "n" + GenericFormatter.SHORT_MINUTE_FORMATTER.format(new Date(now)) + "_" + + "c" + String.format("%1$012d", doccount)+ "_tc"; // the name ends with the transaction token ('c' = 'created') + + export = sb.index.fulltext().export(filepath, filename, format.getExt(), filter, query, format, dom, text, maxChunkSize, minified); + } catch (final IOException e) { prop.put("lurlexporterror", 1); prop.put("lurlexporterror_exportfile", "-no export-"); @@ -150,37 +212,6 @@ public class IndexExport_p { prop.put("reload", 1); } - if (post.containsKey("indexdump")) { - try { - final File dump = segment.fulltext().dumpEmbeddedSolr(); - prop.put("indexdump", 1); - prop.put("indexdump_dumpfile", dump.getAbsolutePath()); - dumpFiles = segment.fulltext().dumpFiles(); - prop.put("dumprestore_dumpfile", dumpFiles.size() == 0 ? "" : dumpFiles.get(dumpFiles.size() - 1).getAbsolutePath()); - // sb.tables.recordAPICall(post, "IndexExport_p.html", WorkTables.TABLE_API_TYPE_STEERING, "solr dump generation"); - } catch(final SolrException e) { - if(ErrorCode.SERVICE_UNAVAILABLE.code == e.code()) { - prop.put("indexdump", 2); - } else { - prop.put("indexdump", 3); - } - } - } - - if (post.containsKey("indexrestore")) { - try { - final File dump = new File(post.get("dumpfile", "")); - segment.fulltext().restoreEmbeddedSolr(dump); - prop.put("indexRestore", 1); - } catch(final SolrException e) { - if(ErrorCode.SERVICE_UNAVAILABLE.code == e.code()) { - prop.put("indexRestore", 2); - } else { - prop.put("indexRestore", 3); - } - } - } - // insert constants prop.putNum("ucount", ucount); // return rewrite properties diff --git a/source/net/yacy/htroot/IndexImportMediawiki_p.java b/source/net/yacy/htroot/IndexImportMediawiki_p.java index faf30ff00..9ea5f834d 100644 --- a/source/net/yacy/htroot/IndexImportMediawiki_p.java +++ b/source/net/yacy/htroot/IndexImportMediawiki_p.java @@ -166,7 +166,7 @@ public class IndexImportMediawiki_p { sb.tables.recordAPICall(post, "IndexImportMediawiki_p.html", WorkTables.TABLE_API_TYPE_DUMP, "MediaWiki Dump Import for " + sourceURL); } - MediawikiImporter.job = new MediawikiImporter(sourceURL, sb.surrogatesInPath); + MediawikiImporter.job = new MediawikiImporter(sourceURL, sb.packsLoadPath); MediawikiImporter.job.start(); prop.put("import_dump", MediawikiImporter.job.source()); prop.put("import_thread", 1); diff --git a/source/net/yacy/htroot/IndexImportOAIPMH_p.java b/source/net/yacy/htroot/IndexImportOAIPMH_p.java index df9f8a92c..a37648267 100644 --- a/source/net/yacy/htroot/IndexImportOAIPMH_p.java +++ b/source/net/yacy/htroot/IndexImportOAIPMH_p.java @@ -65,7 +65,7 @@ public class IndexImportOAIPMH_p { try {
url = new DigestURL(oaipmhurl);
final ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName));
- final OAIPMHLoader r = new OAIPMHLoader(sb.loader, url, sb.surrogatesInPath, agent);
+ final OAIPMHLoader r = new OAIPMHLoader(sb.loader, url, sb.packsLoadPath, agent);
final ResumptionToken rt = r.getResumptionToken();
prop.put("import-one", 1);
prop.put("import-one_count", (rt == null) ? "not available" : Integer.toString(rt.getRecordCounter()));
@@ -115,7 +115,7 @@ public class IndexImportOAIPMH_p { if (post.get("loadrows", "").length() > 0) {
// create a time-ordered list of events to execute
- final Set<String> sources = new TreeSet<String>();
+ final Set<String> sources = new TreeSet<>();
for (final Map.Entry<String, String> entry: post.entrySet()) {
if (entry.getValue().startsWith("mark_")) {
sources.add(entry.getValue().substring(5));
@@ -126,7 +126,7 @@ public class IndexImportOAIPMH_p { prop.put("iframetype", 1);
// prepare the set for random read from it (to protect the servers at the beginning of the list)
- final List<String> sourceList = new ArrayList<String>(sources.size());
+ final List<String> sourceList = new ArrayList<>(sources.size());
for (final String oaipmhurl: sources) sourceList.add(oaipmhurl);
final Random r = new Random(System.currentTimeMillis());
diff --git a/source/net/yacy/htroot/IndexPackGenerator_p.java b/source/net/yacy/htroot/IndexPackGenerator_p.java new file mode 100644 index 000000000..a67eb02e9 --- /dev/null +++ b/source/net/yacy/htroot/IndexPackGenerator_p.java @@ -0,0 +1,221 @@ +// IndexExport_p.java +// ----------------------- +// (C) 2004-2007 by Michael Peter Christen; mc@yacy.net, Frankfurt a. M., Germany +// first published 2004 on http://yacy.net +// +// This is a part of YaCy, a peer-to-peer based web search engine +// +// LICENSE +// +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; either version 2 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program; if not, write to the Free Software +// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +package net.yacy.htroot; + +import java.io.File; +import java.io.IOException; +import java.util.Date; + +import net.yacy.cora.date.GenericFormatter; +import net.yacy.cora.federate.solr.connector.AbstractSolrConnector; +import net.yacy.cora.protocol.RequestHeader; +import net.yacy.cora.sorting.ReversibleScoreMap; +import net.yacy.data.WorkTables; +import net.yacy.search.Switchboard; +import net.yacy.search.SwitchboardConstants; +import net.yacy.search.index.Fulltext; +import net.yacy.search.index.Segment; +import net.yacy.search.schema.CollectionSchema; +import net.yacy.server.serverObjects; +import net.yacy.server.serverSwitch; + +public class IndexPackGenerator_p { + + public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { + // return variable that accumulates replacements + final Switchboard sb = (Switchboard) env; + + final serverObjects prop = new serverObjects(); + + final Segment segment = sb.index; + // we have two counts of document: total number and such that are exportable with status code 200 + final long ucount = segment.fulltext().collectionSize(); + long ucount200 = ucount; + try { + ucount200 = segment.fulltext().getDefaultConnector().getCountByQuery(CollectionSchema.httpstatus_i.getSolrFieldName() + ":200"); + } catch (final IOException e1) {} + + // set default values + prop.put("reload", 0); + prop.put("lurlexport", 0); + prop.putNum("ucount", ucount); + prop.putNum("ucount200", ucount200); + + // show export messages + Fulltext.Export export = segment.fulltext().export(); + if ((export != null) && (export.isAlive())) { + // there is currently a running export + prop.put("lurlexport", 2); + prop.put("lurlexportfinished", 0); + prop.put("lurlexporterror", 0); + prop.put("lurlexport_exportfile", export.file().toString()); + prop.put("lurlexport_urlcount", export.docCount()); + prop.put("reload", 1); + } else { + prop.put("lurlexport", 1); + if (export == null) { + // there has never been an export + prop.put("lurlexportfinished", 0); + prop.put("lurlexporterror", 0); + } else { + // an export was running but has finished + prop.put("lurlexportfinished", 1); + prop.put("lurlexportfinished_exportfile", export.file().toString()); + prop.put("lurlexportfinished_urlcount", export.docCount()); + if (export.failed() == null) { + prop.put("lurlexporterror", 0); + } else { + prop.put("lurlexporterror", 1); + prop.put("lurlexporterror_exportfile", export.file().toString()); + prop.put("lurlexporterror_exportfailmsg", export.failed()); + } + } + + // get the collection facet + ReversibleScoreMap<String> collections = null; + prop.put("lurlexport_collections", 0); + try { + collections = sb.index.fulltext().getDefaultConnector().getFacets(AbstractSolrConnector.CATCHALL_QUERY, 1000, CollectionSchema.collection_sxt.getSolrFieldName()).get(CollectionSchema.collection_sxt.getSolrFieldName()); + if (collections != null) { + int i = 0; + for (final String collection: collections) { + if (collection.startsWith("robot_")) continue; + prop.put("lurlexport_collections_" + i + "_collection", collection); + prop.putNum("lurlexport_collections_" + i + "_count", collections.get(collection)); + prop.putNum("lurlexport_collections_" + i + "_selected", "user".equals(collection) ? 1 : 0); + i++; + } + prop.put("lurlexport_collections", i); + } + } catch (final IOException e) {} + } + + // show Pack folder contents + int i = 0; + boolean dark = true; + for (final String file: sb.packsInHold()) { + prop.put("packs_" + i + "_file", file); + prop.put("packs_" + i + "_type", "hold"); + prop.put("packs_" + i + "_size", new File(sb.packsHoldPath, file).length() / 1024); + prop.put("packs_" + i + "_dark", dark ? "1" : "0"); + i++; + dark = !dark; + } + for (final String file: sb.packsInLoaded()) { + prop.put("packs_" + i + "_file", file); + prop.put("packs_" + i + "_type", "loaded"); + prop.put("packs_" + i + "_size", new File(sb.packsLoadedPath, file).length() / 1024); + prop.put("packs_" + i + "_dark", dark ? "1" : "0"); + i++; + dark = !dark; + } + for (final String file: sb.packsInLive()) { + prop.put("packs_" + i + "_file", file); + prop.put("packs_" + i + "_type", "live"); + prop.put("packs_" + i + "_size", new File(sb.packsLivePath, file).length() / 1024); + prop.put("packs_" + i + "_dark", dark ? "1" : "0"); + i++; + dark = !dark; + } + prop.put("packs", i); + + if (post == null || env == null) { + return prop; // nothing to do + } + + if (post.containsKey("lurlexport")) { + try { + // parse format + Fulltext.ExportFormat format = Fulltext.ExportFormat.elasticsearch; + final String fname = post.get("format", "full-elasticsearch"); + final boolean dom = fname.startsWith("dom"); // if dom== false complete urls are exported, otherwise only the domain + final boolean text = fname.startsWith("text"); + if (fname.endsWith("rss")) format = Fulltext.ExportFormat.rss; + if (fname.endsWith("solr")) format = Fulltext.ExportFormat.solr; + if (fname.endsWith("elasticsearch")) format = Fulltext.ExportFormat.elasticsearch; + + final String filter = post.get("exportfilter", ".*"); + String query = post.get("exportquery", "*:*"); + final String collection = post.get("collection", "user"); + query += " AND " + CollectionSchema.collection_sxt.getSolrFieldName() + ":\"" + collection + "\""; + + // store this call as api call: we do this even if there is a chance that it fails because recurring calls may do not fail + sb.tables.recordAPICall(post, "IndexPackGenerator_p.html", WorkTables.TABLE_API_TYPE_DUMP, "PackGenerator, q=" + query); + + // start the export + /* + Tier Tags: + | Tier | Size | Notes | + |-----------|-----------|--------------------| + | common | ≤ 1 GB | IndexPackGenerator | + | uncommon | 1–5 GB | large web crawls | + | rare | 5–50 GB | custom parser | + | epic | 50–200 GB | special infra | + | legendary | any | human curation | + */ + final long now = System.currentTimeMillis(); + final long doccount = sb.index.fulltext().getDefaultConnector().getCountByQuery(query); + if (doccount == 0) throw new IOException("number of exported documents == 0"); + final String category = post.get("category", "scroll"); // core, scroll, codex, gem, fiction, map, echo, spirit, vault + final String tier = "common"; // common, uncommon, rare, epic, legendary, legendary + final String origin = "web"; // web, synth, + String slug = post.get("slug", "export").trim().replaceAll(" ", "-"); + if (slug.isEmpty()) slug = "export"; + + // if collection is not user, the slug is the collection name + if (!"user".equals(collection)) { + slug = collection.trim().replaceAll(" ", "-"); + } + // we can not construct the file name + final String filename = + SwitchboardConstants.YACY_PACK_PREFIX + + category + "-" + tier + "-" + origin + "_" + + slug + "_" + + GenericFormatter.SHORT_DAY_FORMATTER.format(new Date(now)); + // file name schema: YaCyPack_<category>-<tier>-<origin>_<slug>_<YYMMDD>.jsonlist + // possible storage paths are: hold, load, loaded, unload, live; we use hold here, loaded would also be correct + export = sb.index.fulltext().export(new File(sb.getDataPath() + "/DATA/PACKS/hold/"), filename, format.getExt(), filter, query, format, dom, text, -1, false); + } catch (final IOException e) { + prop.put("lurlexporterror", 1); + prop.put("lurlexporterror_exportfile", "-no export-"); + prop.put("lurlexporterror_exportfailmsg", e.getMessage()); + return prop; + } + + // show result + prop.put("lurlexport_exportfile", export.file().toString()); + prop.put("lurlexport_urlcount", export.docCount()); + if ((export != null) && (export.failed() == null)) { + prop.put("lurlexport", 2); + } + prop.put("reload", 1); + } + + // insert constants + prop.putNum("ucount", ucount); + // return rewrite properties + return prop; + } + +}
\ No newline at end of file diff --git a/source/net/yacy/htroot/api/share.java b/source/net/yacy/htroot/api/share.java index 74da096dd..104827438 100644 --- a/source/net/yacy/htroot/api/share.java +++ b/source/net/yacy/htroot/api/share.java @@ -31,7 +31,7 @@ import net.yacy.cora.document.encoding.UTF8; import net.yacy.cora.order.Base64Order; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.search.index.Fulltext; +import net.yacy.search.SwitchboardConstants; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; @@ -77,8 +77,8 @@ public class share { prop.put("mode_success_message", "file name is empty"); return prop; } - if (!filename.startsWith(Fulltext.yacy_dump_prefix) || !filename.endsWith(".xml.gz")) { - prop.put("mode_success_message", "no index dump file (" + Fulltext.yacy_dump_prefix + "*.xml.gz)"); + if (!filename.startsWith(SwitchboardConstants.YACY_PACK_PREFIX) || !filename.endsWith(".xml.gz")) { + prop.put("mode_success_message", "no index dump file (" + SwitchboardConstants.YACY_PACK_PREFIX + "*.xml.gz)"); return prop; } diff --git a/source/net/yacy/htroot/api/yacydoc.java b/source/net/yacy/htroot/api/yacydoc.java index ff3092773..f82c35c9f 100644 --- a/source/net/yacy/htroot/api/yacydoc.java +++ b/source/net/yacy/htroot/api/yacydoc.java @@ -103,7 +103,7 @@ public class yacydoc { prop.putXML("dc_title", entry.dc_title()); prop.putXML("dc_creator", entry.dc_creator()); - prop.putXML("dc_description", ""); // this is the fulltext part in the surrogate + prop.putXML("dc_description", ""); // this is the fulltext part in the pack prop.putXML("dc_subject", entry.dc_subject()); prop.putXML("dc_publisher", entry.dc_publisher()); prop.putXML("dc_contributor", ""); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 159dd2d29..a8ed948b6 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -173,7 +173,7 @@ import net.yacy.document.TextParser; import net.yacy.document.Tokenizer; import net.yacy.document.VocabularyScraper; import net.yacy.document.content.DCEntry; -import net.yacy.document.content.SurrogateReader; +import net.yacy.document.content.XMLPackReader; import net.yacy.document.importer.JsonListImporter; import net.yacy.document.importer.OAIListFriendsLoader; import net.yacy.document.importer.WarcImporter; @@ -268,8 +268,7 @@ public final class Switchboard extends serverSwitch { public File releasePath; public File networkRoot; public File queuesRoot; - public File surrogatesInPath; - public File surrogatesOutPath; + public File packsHoldPath, packsLivePath, packsLoadPath, packsLoadedPath, packsUnloadPath; public Segment index; public LoaderDispatcher loader; public CrawlSwitchboard crawler; @@ -548,11 +547,11 @@ public final class Switchboard extends serverSwitch { // define load limitation according to current number of cpu cores if (this.firstInit) { - float numberOfCores2 = 2.0f * (float) Runtime.getRuntime().availableProcessors(); + float numberOfCores2 = 2.0f * Runtime.getRuntime().availableProcessors(); sb.setConfig(SwitchboardConstants.CRAWLJOB_LOCAL_CRAWL_LOADPREREQ, numberOfCores2); - sb.setConfig(SwitchboardConstants.SURROGATES_LOADPREREQ, numberOfCores2); + sb.setConfig(SwitchboardConstants.PACKS_LOADPREREQ, numberOfCores2); } - + // define boosts Ranking.setMinTokenLen(this.getConfigInt(SwitchboardConstants.SEARCH_RANKING_SOLR_DOUBLEDETECTION_MINLENGTH, 3)); Ranking.setQuantRate(this.getConfigFloat(SwitchboardConstants.SEARCH_RANKING_SOLR_DOUBLEDETECTION_QUANTRATE, 0.5f)); @@ -791,19 +790,22 @@ public final class Switchboard extends serverSwitch { Transactions.init(transactiondir, this.getConfigLong(SwitchboardConstants.SNAPSHOTS_WKHTMLTOPDF_TIMEOUT, SwitchboardConstants.SNAPSHOTS_WKHTMLTOPDF_TIMEOUT_DEFAULT)); - // create the surrogates directories - this.surrogatesInPath = - this.getDataPath( - SwitchboardConstants.SURROGATES_IN_PATH, - SwitchboardConstants.SURROGATES_IN_PATH_DEFAULT); - this.log.info("surrogates.in Path = " + this.surrogatesInPath.getAbsolutePath()); - this.surrogatesInPath.mkdirs(); - this.surrogatesOutPath = - this.getDataPath( - SwitchboardConstants.SURROGATES_OUT_PATH, - SwitchboardConstants.SURROGATES_OUT_PATH_DEFAULT); - this.log.info("surrogates.out Path = " + this.surrogatesOutPath.getAbsolutePath()); - this.surrogatesOutPath.mkdirs(); + // create the packs directories + this.packsHoldPath = this.getDataPath(SwitchboardConstants.PACKS_HOLD_PATH, SwitchboardConstants.PACKS_HOLD_PATH_DEFAULT); + this.packsHoldPath.mkdirs(); + this.log.info("packs.hold Path = " + this.packsHoldPath.getAbsolutePath()); + this.packsLivePath = this.getDataPath(SwitchboardConstants.PACKS_LIVE_PATH, SwitchboardConstants.PACKS_LIVE_PATH_DEFAULT); + this.packsLivePath.mkdirs(); + this.log.info("packs.live Path = " + this.packsLivePath.getAbsolutePath()); + this.packsLoadPath = this.getDataPath(SwitchboardConstants.PACKS_LOAD_PATH, SwitchboardConstants.PACKS_LOAD_PATH_DEFAULT); + this.packsLoadPath.mkdirs(); + this.log.info("packs.load Path = " + this.packsLoadPath.getAbsolutePath()); + this.packsLoadedPath = this.getDataPath(SwitchboardConstants.PACKS_LOADED_PATH, SwitchboardConstants.PACKS_LOADED_PATH_DEFAULT); + this.packsLoadedPath.mkdirs(); + this.log.info("packs.loaded Path = " + this.packsLoadedPath.getAbsolutePath()); + this.packsUnloadPath = this.getDataPath(SwitchboardConstants.PACKS_UNLOAD_PATH, SwitchboardConstants.PACKS_UNLOAD_PATH_DEFAULT); + this.packsUnloadPath.mkdirs(); + this.log.info("packs.unload Path = " + this.packsUnloadPath.getAbsolutePath()); // copy opensearch heuristic config (if not exist) final File osdConfig = new File(this.getDataPath(), "DATA/SETTINGS/heuristicopensearch.conf"); @@ -926,8 +928,8 @@ public final class Switchboard extends serverSwitch { public void run() { final ClientIdentification.Agent agent = ClientIdentification.getAgent(ClientIdentification.yacyInternetCrawlerAgentName); try { - final Response documentResponse = Switchboard.this.loader.load(Switchboard.this.loader.request(new DigestURL(getConfig("donation.iframesource", "")), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent); - if (documentResponse != null) FileUtils.copy(documentResponse.getContent(), new File(Switchboard.this.htDocsPath, getConfig("donation.iframetarget", ""))); + final Response documentResponse = Switchboard.this.loader.load(Switchboard.this.loader.request(new DigestURL(Switchboard.this.getConfig("donation.iframesource", "")), false, true), CacheStrategy.NOCACHE, Integer.MAX_VALUE, null, agent); + if (documentResponse != null) FileUtils.copy(documentResponse.getContent(), new File(Switchboard.this.htDocsPath, Switchboard.this.getConfig("donation.iframetarget", ""))); } catch (final Exception e) {} } }.start(); @@ -1068,7 +1070,7 @@ public final class Switchboard extends serverSwitch { new String[] { "storeDocumentIndex" }, - in -> Switchboard.this.webStructureAnalysis(in), + Switchboard.this::webStructureAnalysis, WorkflowProcessor.availableCPU + 1, this.indexingStorageProcessor, WorkflowProcessor.availableCPU); @@ -1079,7 +1081,7 @@ public final class Switchboard extends serverSwitch { new String[] { "webStructureAnalysis" }, - in -> Switchboard.this.condenseDocument(in), + Switchboard.this::condenseDocument, WorkflowProcessor.availableCPU + 1, this.indexingAnalysisProcessor, WorkflowProcessor.availableCPU); @@ -1090,7 +1092,7 @@ public final class Switchboard extends serverSwitch { new String[] { "condenseDocument", "CrawlStacker" }, - in -> Switchboard.this.parseDocument(in), + Switchboard.this::parseDocument, Math.max(20, WorkflowProcessor.availableCPU * 2), // it may happen that this is filled with new files from the search process. That means there should be enough place for two result pages this.indexingCondensementProcessor, WorkflowProcessor.availableCPU); @@ -1146,24 +1148,24 @@ public final class Switchboard extends serverSwitch { 60000); // all 10 minutes, wait 1 minute until first run this.deployThread( - SwitchboardConstants.SURROGATES, - "Surrogates", - "A thread that polls the SURROGATES path and puts all Documents in one surroagte file into the indexing queue.", + SwitchboardConstants.PACKS, + "Packs", + "A thread that polls the PACKS path and puts all Documents in one surroagte file into the indexing queue.", null, - new InstantBusyThread("Switchboard.surrogateProcess", 20000, 0) { + new InstantBusyThread("Switchboard.packProcess", 20000, 0) { @Override public boolean jobImpl() throws Exception { - return Switchboard.this.surrogateProcess(); + return Switchboard.this.packProcess(); } @Override public int getJobCount() { - return Switchboard.this.surrogateQueueSize(); + return Switchboard.this.packsInLoad().size(); } @Override public void freememImpl() { - Switchboard.this.surrogateFreeMem(); + // do nothing } }, 10000); @@ -2116,12 +2118,12 @@ public final class Switchboard extends serverSwitch { return null; } - public boolean processSurrogate(final String s) { - final File infile = new File(this.surrogatesInPath, s); + public boolean processPack(final String s) { + final File infile = new File(this.packsLoadPath, s); if ( !infile.exists() || !infile.canWrite() || !infile.canRead() ) { return false; } - final File outfile = new File(this.surrogatesOutPath, s); + final File outfile = new File(this.packsLoadedPath, s); //if (outfile.exists()) return false; boolean moved = false; if ( s.endsWith("xml.zip") ) { @@ -2139,7 +2141,7 @@ public final class Switchboard extends serverSwitch { baos.write(buffer, 0, size); } baos.flush(); - this.processSurrogateXML(new ByteArrayInputStream(baos.toByteArray()), entry.getName()); + this.processXMLPack(new ByteArrayInputStream(baos.toByteArray()), entry.getName()); baos.close(); if (this.shallTerminate()) break; } @@ -2184,13 +2186,13 @@ public final class Switchboard extends serverSwitch { s.endsWith(".jsonl") || s.endsWith(".jsonl.gz") || s.endsWith(".jsonlist") || s.endsWith(".jsonlist.gz") || s.endsWith(".flatjson") || s.endsWith(".flatjson.gz")) { - return this.processSurrogateJson(infile, outfile); + return this.processPackJson(infile, outfile); } InputStream is = null; try { is = new BufferedInputStream(new FileInputStream(infile)); if (s.endsWith(".gz")) is = new GZIPInputStream(is, 65535); - this.processSurrogateXML(is, infile.getName()); + this.processXMLPack(is, infile.getName()); } catch (final IOException e ) { ConcurrentLog.logException(e); } finally { @@ -2217,7 +2219,7 @@ public final class Switchboard extends serverSwitch { ConcurrentLog.logException(e); } } - this.log.info("processed surrogate " + infile); + this.log.info("processed pack " + infile); } } if (is != null) try {is.close();} catch (final IOException e) { @@ -2227,10 +2229,10 @@ public final class Switchboard extends serverSwitch { return moved; } - private boolean processSurrogateJson(final File infile, final File outfile) { + private boolean processPackJson(final File infile, final File outfile) { // parse a file that can be generated with yacy_grid_parser // see https://github.com/yacy/yacy_grid_parser/blob/master/README.md - this.log.info("processing json surrogate " + infile); + this.log.info("processing json pack " + infile); try { final JsonListImporter importer = new JsonListImporter(infile, false, false); importer.run(); @@ -2242,11 +2244,11 @@ public final class Switchboard extends serverSwitch { return moved; } - private void processSurrogateXML(final InputStream is, final String name) throws IOException { + private void processXMLPack(final InputStream is, final String name) throws IOException { final int concurrency = Runtime.getRuntime().availableProcessors(); // start reader thread - final SurrogateReader reader = new SurrogateReader(is, 100, this.crawlStacker, this.index.fulltext().getDefaultConfiguration(), concurrency); + final XMLPackReader reader = new XMLPackReader(is, 100, this.crawlStacker, this.index.fulltext().getDefaultConfiguration(), concurrency); final Thread readerThread = new Thread(reader, name); readerThread.setPriority(Thread.MAX_PRIORITY); // we must have maximum prio here because this thread feeds the other threads. It must always be ahead of them. readerThread.start(); @@ -2255,27 +2257,27 @@ public final class Switchboard extends serverSwitch { assert this.crawlStacker != null; final Thread[] indexer = new Thread[concurrency]; for (int t = 0; t < concurrency; t++) { - indexer[t] = new Thread("Switchboard.processSurrogateXML-" + t) { + indexer[t] = new Thread("Switchboard.processXMLPack-" + t) { @Override public void run() { final VocabularyScraper scraper = new VocabularyScraper(); - Object surrogateObj; - while ((surrogateObj = reader.take()) != SurrogateReader.POISON_DOCUMENT ) { - assert surrogateObj != null; - /* When parsing a full-text Solr xml data dump Surrogate reader produces SolrInputDocument instances */ - if(surrogateObj instanceof SolrInputDocument) { - final SolrInputDocument surrogate = (SolrInputDocument)surrogateObj; + Object xmlPackObj; + while ((xmlPackObj = reader.take()) != XMLPackReader.POISON_DOCUMENT ) { + assert xmlPackObj != null; + /* When parsing a full-text Solr xml data dump Pack reader produces SolrInputDocument instances */ + if(xmlPackObj instanceof SolrInputDocument) { + final SolrInputDocument pack = (SolrInputDocument)xmlPackObj; try { - // enrich the surrogate - final String id = (String) surrogate.getFieldValue(CollectionSchema.id.getSolrFieldName()); - final String text = (String) surrogate.getFieldValue(CollectionSchema.text_t.getSolrFieldName()); - final DigestURL rootURL = new DigestURL((String) surrogate.getFieldValue(CollectionSchema.sku.getSolrFieldName()), ASCII.getBytes(id)); + // enrich the pack + final String id = (String) pack.getFieldValue(CollectionSchema.id.getSolrFieldName()); + final String text = (String) pack.getFieldValue(CollectionSchema.text_t.getSolrFieldName()); + final DigestURL rootURL = new DigestURL((String) pack.getFieldValue(CollectionSchema.sku.getSolrFieldName()), ASCII.getBytes(id)); if (text != null && text.length() > 0 && id != null ) { // run the tokenizer on the text to get vocabularies and synonyms final Tokenizer tokenizer = new Tokenizer(rootURL, text, LibraryProvider.dymLib, true, scraper); final Map<String, Set<String>> facets = Document.computeGenericFacets(tokenizer.tags()); // overwrite the given vocabularies and synonyms with new computed ones - Switchboard.this.index.fulltext().getDefaultConfiguration().enrich(surrogate, tokenizer.synonyms(), facets); + Switchboard.this.index.fulltext().getDefaultConfiguration().enrich(pack, tokenizer.synonyms(), facets); } /* Update the ResultURLS stack for monitoring */ @@ -2285,16 +2287,16 @@ public final class Switchboard extends serverSwitch { rootURL.getHost(), myPeerHash, myPeerHash, - EventOrigin.SURROGATES); + EventOrigin.PACKS); } catch (final MalformedURLException e) { ConcurrentLog.logException(e); } - // write the surrogate into the index - Switchboard.this.index.putDocument(surrogate); - } else if(surrogateObj instanceof DCEntry) { - /* When parsing a MediaWiki dump Surrogate reader produces DCEntry instances */ + // write the pack into the index + Switchboard.this.index.putDocument(pack); + } else if(xmlPackObj instanceof DCEntry) { + /* When parsing a MediaWiki dump pack reader produces DCEntry instances */ // create a queue entry - final DCEntry entry = (DCEntry)surrogateObj; + final DCEntry entry = (DCEntry)xmlPackObj; final Document document = entry.document(); final Request request = new Request( @@ -2303,10 +2305,10 @@ public final class Switchboard extends serverSwitch { null, "", entry.getDate(), - Switchboard.this.crawler.defaultSurrogateProfile.handle(), + Switchboard.this.crawler.defaultPackProfile.handle(), 0, - Switchboard.this.crawler.defaultSurrogateProfile.timezoneOffset()); - final Response response = new Response(request, null, null, Switchboard.this.crawler.defaultSurrogateProfile, false, null); + Switchboard.this.crawler.defaultPackProfile.timezoneOffset()); + final Response response = new Response(request, null, null, Switchboard.this.crawler.defaultPackProfile, false, null); final IndexingQueueEntry queueEntry = new IndexingQueueEntry(response, new Document[] {document}, null); @@ -2326,38 +2328,56 @@ public final class Switchboard extends serverSwitch { } } - public int surrogateQueueSize() { - // count surrogates - final String[] surrogatelist = this.surrogatesInPath.list(); - if ( surrogatelist.length > 100 ) { - return 100; - } - int count = 0; - for ( final String s : surrogatelist ) { - if ( s.endsWith(".xml") - || s.endsWith(".xml.gz") - || s.endsWith(".xml.zip") - || s.endsWith(".warc") - || s.endsWith(".warc.gz") - || s.endsWith(".jsonl") - || s.endsWith(".jsonl.gz") - || s.endsWith(".jsonlist") - || s.endsWith(".jsonlist.gz") - || s.endsWith(".flatjson") ) { - count++; - } - if ( count >= 100 ) { - break; - } - } - return count; + public boolean isPackFile(String s) { + return s.endsWith(".xml") + || s.endsWith(".xml.gz") + || s.endsWith(".xml.zip") + || s.endsWith(".warc") + || s.endsWith(".warc.gz") + || s.endsWith(".jsonl") + || s.endsWith(".jsonl.gz") + || s.endsWith(".jsonlist") + || s.endsWith(".jsonlist.gz") + || s.endsWith(".flatjson") + || s.endsWith(".flatjson.gz"); + } + + public List<String> packsInHold() { + final String[] packlist = this.packsHoldPath.list(); + ArrayList<String> list = new ArrayList<>(); + for (final String s: packlist) if (this.isPackFile(s)) list.add(s); + return list; + } + + public List<String> packsInLive() { + final String[] packlist = this.packsLivePath.list(); + ArrayList<String> list = new ArrayList<>(); + for (final String s: packlist) if (this.isPackFile(s)) list.add(s); + return list; + } + + public List<String> packsInLoad() { + final String[] packlist = this.packsLoadPath.list(); + ArrayList<String> list = new ArrayList<>(); + for (final String s: packlist) if (this.isPackFile(s)) list.add(s); + return list; + } + + public List<String> packsInLoaded() { + final String[] packlist = this.packsLoadedPath.list(); + ArrayList<String> list = new ArrayList<>(); + for (final String s: packlist) if (this.isPackFile(s)) list.add(s); + return list; } - public void surrogateFreeMem() { - // do nothing + public List<String> packsInUnload() { + final String[] packlist = this.packsUnloadPath.list(); + ArrayList<String> list = new ArrayList<>(); + for (final String s: packlist) if (this.isPackFile(s)) list.add(s); + return list; } - public boolean surrogateProcess() { + public boolean packProcess() { // work off fresh entries from the proxy or from the crawler final String cautionCause = this.onlineCaution(); if ( cautionCause != null ) { @@ -2370,27 +2390,18 @@ public final class Switchboard extends serverSwitch { } try { - // check surrogates - final String[] surrogatelist = this.surrogatesInPath.list(); - if ( surrogatelist != null && surrogatelist.length > 0 ) { + // check packs + final String[] packlist = this.packsLoadPath.list(); + if ( packlist != null && packlist.length > 0 ) { // look if the is any xml inside - for ( final String surrogate : surrogatelist ) { + for ( final String pack : packlist ) { // check for interruption this.checkInterruption(); - if ( surrogate.endsWith(".xml") - || surrogate.endsWith(".xml.gz") - || surrogate.endsWith(".xml.zip") - || surrogate.endsWith(".zim") - || surrogate.endsWith(".warc") - || surrogate.endsWith(".warc.gz") - || surrogate.endsWith(".jsonlist") - || surrogate.endsWith(".jsonlist.gz") - || surrogate.endsWith(".flatjson") - || surrogate.endsWith(".flatjson.gz") ) { - // read the surrogate file and store entry in index - if ( this.processSurrogate(surrogate) ) { + if (this.isPackFile(pack)) { + // read the pack file and store entry in index + if ( this.processPack(pack) ) { return true; } } @@ -2589,8 +2600,8 @@ public final class Switchboard extends serverSwitch { selentry.put(CrawlProfile.CrawlAttribute.RECRAWL_IF_OLDER.key, Long.toString(CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA_RECRAWL_CYCLE).getTime())); insert = true; } - if ( selentry.name().equals(CrawlSwitchboard.CRAWL_PROFILE_SURROGATE) ) { - selentry.put(CrawlProfile.CrawlAttribute.RECRAWL_IF_OLDER.key, Long.toString(CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_SURROGATE_RECRAWL_CYCLE).getTime())); + if ( selentry.name().equals(CrawlSwitchboard.CRAWL_PROFILE_PACKS) ) { + selentry.put(CrawlProfile.CrawlAttribute.RECRAWL_IF_OLDER.key, Long.toString(CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_PACK_RECRAWL_CYCLE).getTime())); insert = true; } if ( insert ) { @@ -3368,8 +3379,8 @@ public final class Switchboard extends serverSwitch { /* This entry may have been locally created by the MediaWiki dump reader : * we can distinguish the case here from a regular local crawl with the crawl profile used */ - if(this.crawler != null && queueEntry.profile() == this.crawler.defaultSurrogateProfile) { - processCase = EventOrigin.SURROGATES; + if(this.crawler != null && queueEntry.profile() == this.crawler.defaultPackProfile) { + processCase = EventOrigin.PACKS; } final CrawlProfile profile = queueEntry.profile(); diff --git a/source/net/yacy/search/SwitchboardConstants.java b/source/net/yacy/search/SwitchboardConstants.java index 3a168a269..91cf24484 100644 --- a/source/net/yacy/search/SwitchboardConstants.java +++ b/source/net/yacy/search/SwitchboardConstants.java @@ -145,16 +145,16 @@ public final class SwitchboardConstants { public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL = "62_remotetriggeredcrawl"; public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_IDLESLEEP = "62_remotetriggeredcrawl_idlesleep"; public static final String CRAWLJOB_REMOTE_TRIGGERED_CRAWL_BUSYSLEEP = "62_remotetriggeredcrawl_busysleep"; - // 70_surrogates + // 70_packs /** - * <p><code>public static final String <strong>SURROGATES</strong> = "70_surrogates"</code></p> - * <p>A thread that polls the SURROGATES path and puts all Documents in one surrogate file into the indexing queue.</p> + * <p><code>public static final String <strong>PACKS</strong> = "70_packs"</code></p> + * <p>A thread that polls the PACKS path and puts all Documents in one pack file into the indexing queue.</p> */ - public static final String SURROGATES = "70_surrogates"; - public static final String SURROGATES_MEMPREREQ = "70_surrogates_memprereq"; - public static final String SURROGATES_LOADPREREQ = "70_surrogates_loadprereq"; - public static final String SURROGATES_IDLESLEEP = "70_surrogates_idlesleep"; - public static final String SURROGATES_BUSYSLEEP = "70_surrogates_busysleep"; + public static final String PACKS = "70_packs"; + public static final String PACKS_MEMPREREQ = "70_packs_memprereq"; + public static final String PACKS_LOADPREREQ = "70_packs_loadprereq"; + public static final String PACKS_IDLESLEEP = "70_packs_idlesleep"; + public static final String PACKS_BUSYSLEEP = "70_packs_busysleep"; // 85_scheduler /** * <p><code>public static final String <strong>SCHEDULER</strong> = "85_scheduler"</code></p> @@ -434,10 +434,16 @@ public final class SwitchboardConstants { public static final String RELEASE_PATH = "releases"; public static final String RELEASE_PATH_DEFAULT = "DATA/RELEASE"; - public static final String SURROGATES_IN_PATH = "surrogates.in"; - public static final String SURROGATES_IN_PATH_DEFAULT = "DATA/SURROGATES/in"; - public static final String SURROGATES_OUT_PATH = "surrogates.out"; - public static final String SURROGATES_OUT_PATH_DEFAULT = "DATA/SURROGATES/out"; + public static final String PACKS_HOLD_PATH = "packs.hold"; + public static final String PACKS_HOLD_PATH_DEFAULT = "DATA/PACKS/hold"; + public static final String PACKS_LIVE_PATH = "packs.live"; + public static final String PACKS_LIVE_PATH_DEFAULT = "DATA/PACKS/live"; + public static final String PACKS_LOAD_PATH = "packs.load"; + public static final String PACKS_LOAD_PATH_DEFAULT = "DATA/PACKS/load"; + public static final String PACKS_LOADED_PATH = "packs.loaded"; + public static final String PACKS_LOADED_PATH_DEFAULT = "DATA/PACKS/loaded"; + public static final String PACKS_UNLOAD_PATH = "packs.unload"; + public static final String PACKS_UNLOAD_PATH_DEFAULT = "DATA/PACKS/unload"; public static final String DICTIONARY_SOURCE_PATH = "dictionaries"; public static final String DICTIONARY_SOURCE_PATH_DEFAULT = "DATA/DICTIONARIES"; @@ -572,6 +578,7 @@ public final class SwitchboardConstants { * Some constants */ public static final String STR_REMOTECRAWLTRIGGER = "REMOTECRAWLTRIGGER: REMOTE CRAWL TO PEER "; + public final static String YACY_PACK_PREFIX = "YaCy-Pack_"; /** * network properties diff --git a/source/net/yacy/search/index/Fulltext.java b/source/net/yacy/search/index/Fulltext.java index 0fd173399..ef824d880 100644 --- a/source/net/yacy/search/index/Fulltext.java +++ b/source/net/yacy/search/index/Fulltext.java @@ -47,7 +47,6 @@ import java.util.zip.Deflater; import java.util.zip.GZIPOutputStream; import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.SolrInputDocument; @@ -298,7 +297,7 @@ public final class Fulltext { public long collectionSize() { final long t = System.currentTimeMillis(); if (t - this.collectionSizeLastAccess < 1000) return this.collectionSizeLastValue; - final SolrConnector sc = getDefaultConnector(); + final SolrConnector sc = this.getDefaultConnector(); if (sc == null) return 0; final long size = sc.getSize(); this.collectionSizeLastAccess = t; @@ -326,8 +325,8 @@ public final class Fulltext { final long t = System.currentTimeMillis(); if (this.lastCommit + 10000 > t) return; this.lastCommit = t; - getDefaultConnector().commit(softCommit); - if (this.writeWebgraph) getWebgraphConnector().commit(softCommit); + this.getDefaultConnector().commit(softCommit); + if (this.writeWebgraph) this.getWebgraphConnector().commit(softCommit); } /** @@ -346,13 +345,13 @@ public final class Fulltext { final WordReferenceVars wre = element.getElement(); if (wre == null) return null; // all time was already wasted in takeRWI to get another element final long score = element.getWeight(); - final URIMetadataNode node = getMetadata(wre.urlhash(), wre, score); + final URIMetadataNode node = this.getMetadata(wre.urlhash(), wre, score); return node; } public URIMetadataNode getMetadata(final byte[] urlHash) { if (urlHash == null) return null; - return getMetadata(urlHash, null, 0L); + return this.getMetadata(urlHash, null, 0L); } private URIMetadataNode getMetadata(final byte[] urlHash, final WordReferenceVars wre, final long score) { @@ -383,7 +382,7 @@ public final class Fulltext { } catch (final SolrException e) { throw new IOException(e.getMessage(), e); } - if (MemoryControl.shortStatus()) clearCaches(); + if (MemoryControl.shortStatus()) this.clearCaches(); } public void putEdges(final Collection<SolrInputDocument> edges) throws IOException { @@ -394,7 +393,7 @@ public final class Fulltext { } catch (final SolrException e) { throw new IOException(e.getMessage(), e); } - if (MemoryControl.shortStatus()) clearCaches(); + if (MemoryControl.shortStatus()) this.clearCaches(); } /** @@ -408,19 +407,19 @@ public final class Fulltext { final SolrDocument doc = this.getDefaultConnector().getDocumentById(id, CollectionSchema.collection_sxt.getSolrFieldName()); if (doc == null || !doc.containsKey(CollectionSchema.collection_sxt.getSolrFieldName())) { // document does not exist - putDocument(getDefaultConfiguration().metadata2solr(entry)); + this.putDocument(this.getDefaultConfiguration().metadata2solr(entry)); } else { final Collection<Object> collections = doc.getFieldValues(CollectionSchema.collection_sxt.getSolrFieldName()); // collection dht is used to identify metadata from full crawled documents (if "dht" exists don't overwrite rich crawldata with metadata if (!collections.contains("dht")) return; // passed all checks, overwrite document - putDocument(getDefaultConfiguration().metadata2solr(entry)); + this.putDocument(this.getDefaultConfiguration().metadata2solr(entry)); } } catch (final SolrException e) { throw new IOException(e.getMessage(), e); } - if (MemoryControl.shortStatus()) clearCaches(); + if (MemoryControl.shortStatus()) this.clearCaches(); } /** @@ -517,7 +516,7 @@ public final class Fulltext { count.incrementAndGet(); } } - remove(deleteIDs); + this.remove(deleteIDs); if (count.get() > 0) Fulltext.this.commit(true); } catch (final InterruptedException e) {} return count.get(); @@ -670,8 +669,8 @@ public final class Fulltext { */ public void optimize(final int size) { if (size < 1) return; - getDefaultConnector().optimize(size); - if (this.writeWebgraph) getWebgraphConnector().optimize(size); + this.getDefaultConnector().optimize(size); + if (this.writeWebgraph) this.getWebgraphConnector().optimize(size); } /** @@ -695,90 +694,12 @@ public final class Fulltext { } public static enum ExportFormat { - text("txt"), html("html"), rss("rss"), solr("xml"), elasticsearch("flatjson"); + text("txt"), html("html"), rss("rss"), solr("xml"), elasticsearch("jsonl"); private final String ext; private ExportFormat(final String ext) {this.ext = ext;} public String getExt() {return this.ext;} } - public final static String yacy_dump_prefix = "yacy_dump_"; - public Export export( - final Fulltext.ExportFormat format, final String filter, String query, - final int maxseconds, final File path, final boolean dom, final boolean text, - final long maxChunkSize, final boolean minified) throws IOException { - - // modify query according to maxseconds - final long now = System.currentTimeMillis(); - if (maxseconds > 0) { - final long from = now - maxseconds * 1000L; - final String nowstr = new Date(now).toInstant().toString(); - final String fromstr = new Date(from).toInstant().toString(); - final String dateq = CollectionSchema.load_date_dt.getSolrFieldName() + ":[" + fromstr + " TO " + nowstr + "]"; - query = query == null || AbstractSolrConnector.CATCHALL_QUERY.equals(query) ? dateq : query + " AND " + dateq; - } else { - query = query == null? AbstractSolrConnector.CATCHALL_QUERY : query; - } - - // check the oldest and latest entry in the index for this query - SolrDocumentList firstdoclist, lastdoclist; - Object firstdateobject, lastdateobject; - firstdoclist = this.getDefaultConnector().getDocumentListByQuery( - query, CollectionSchema.load_date_dt.getSolrFieldName() + " asc", 0, 1,CollectionSchema.load_date_dt.getSolrFieldName()); - lastdoclist = this.getDefaultConnector().getDocumentListByQuery( - query, CollectionSchema.load_date_dt.getSolrFieldName() + " desc", 0, 1,CollectionSchema.load_date_dt.getSolrFieldName()); - - final long doccount; - final Date firstdate, lastdate; - if (firstdoclist.size() == 0 || lastdoclist.size() == 0) { - /* Now check again the number of documents without sorting, for compatibility with old fields indexed without DocValues fields (prior to YaCy 1.90) - * When the local Solr index contains such old documents, requests with sort query return nothing and trace in logs - * "java.lang.IllegalStateException: unexpected docvalues type NONE for field..." */ - doccount = this.getDefaultConnector().getCountByQuery(query); - if(doccount == 0) { - /* Finally no document to export was found */ - throw new IOException("number of exported documents == 0"); - } - /* we use default date values just to generate a proper dump file path */ - firstdate = new Date(0); - lastdate = new Date(0); - - } else { - doccount = firstdoclist.getNumFound(); - - // create the export name - final SolrDocument firstdoc = firstdoclist.get(0); - final SolrDocument lastdoc = lastdoclist.get(0); - firstdateobject = firstdoc.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName()); - lastdateobject = lastdoc.getFieldValue(CollectionSchema.load_date_dt.getSolrFieldName()); - - /* When firstdate or lastdate is null, we use a default one just to generate a proper dump file path - * This should not happen because load_date_dt field is mandatory in the main Solr schema, - * but for some reason some documents might end up here with an empty load_date_dt field value */ - if(firstdateobject instanceof Date) { - firstdate = (Date) firstdateobject; - } else { - ConcurrentLog.warn("Fulltext", "The required field " + CollectionSchema.load_date_dt.getSolrFieldName() + " is empty on document with id : " - + firstdoc.getFieldValue(CollectionSchema.id.getSolrFieldName())); - firstdate = new Date(0); - } - if(lastdateobject instanceof Date) { - lastdate = (Date) lastdateobject; - } else { - ConcurrentLog.warn("Fulltext", "The required field " + CollectionSchema.load_date_dt.getSolrFieldName() + " is empty on document with id : " - + lastdoc.getFieldValue(CollectionSchema.id.getSolrFieldName())); - lastdate = new Date(0); - } - } - - final String filename = yacy_dump_prefix + - "f" + GenericFormatter.SHORT_MINUTE_FORMATTER.format(firstdate) + "_" + - "l" + GenericFormatter.SHORT_MINUTE_FORMATTER.format(lastdate) + "_" + - "n" + GenericFormatter.SHORT_MINUTE_FORMATTER.format(new Date(now)) + "_" + - "c" + String.format("%1$012d", doccount)+ "_tc"; // the name ends with the transaction token ('c' = 'created') - - return export(path, filename, format.getExt(), filter, query, format, dom, text, maxChunkSize, minified); - } - // export methods public Export export( final File path, final String filename, @@ -905,8 +826,8 @@ public final class Fulltext { this.docCount = 0; this.chunkSize = 0; this.chunkCount = 0; - PrintWriter pw = getWriter(); - printHead(pw); + PrintWriter pw = this.getWriter(); + this.printHead(pw); if (this.dom) { final Map<String, ReversibleScoreMap<String>> scores = Fulltext.this.getDefaultConnector().getFacets(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", 100000000, CollectionSchema.host_s.getSolrFieldName()); final ReversibleScoreMap<String> stats = scores.get(CollectionSchema.host_s.getSolrFieldName()); @@ -921,7 +842,7 @@ public final class Fulltext { final BlockingQueue<SolrDocument> docs = Fulltext.this.getDefaultConnector().concurrentDocumentsByQuery(this.query + " AND " + CollectionSchema.httpstatus_i.getSolrFieldName() + ":200", null, 0, 100000000, Long.MAX_VALUE, 100, 1, true); SolrDocument doc; while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) { - final String url = getStringFrom(doc.getFieldValue(CollectionSchema.sku.getSolrFieldName())); + final String url = this.getStringFrom(doc.getFieldValue(CollectionSchema.sku.getSolrFieldName())); if (this.pattern != null && !this.pattern.matcher(url).matches()) continue; if (this.minified) { final Iterator<Entry<String, Object>> i = doc.iterator(); @@ -938,12 +859,12 @@ public final class Fulltext { final String d = sw.toString(); pw.println(d); this.docCount++; this.chunkSize++; - if (this.chunkSize >= this.maxChunkSize) { - printTail(pw); + if (this.maxChunkSize > 0 && this.chunkSize >= this.maxChunkSize) { + this.printTail(pw); pw.close(); this.chunkCount++; - pw = getWriter(); - printHead(pw); + pw = this.getWriter(); + this.printHead(pw); this.chunkSize = 0; } } @@ -956,11 +877,11 @@ public final class Fulltext { Integer size; Date date; while ((doc = docs.take()) != AbstractSolrConnector.POISON_DOCUMENT) { - hash = getStringFrom(doc.getFieldValue(CollectionSchema.id.getSolrFieldName())); - url = getStringFrom(doc.getFieldValue(CollectionSchema.sku.getSolrFieldName())); - title = getStringFrom(doc.getFieldValue(CollectionSchema.title.getSolrFieldName())); - author = getStringFrom(doc.getFieldValue(CollectionSchema.author.getSolrFieldName())); - description = getStringFrom(doc.getFieldValue(CollectionSchema.description_txt.getSolrFieldName())); + hash = this.getStringFrom(doc.getFieldValue(CollectionSchema.id.getSolrFieldName())); + url = this.getStringFrom(doc.getFieldValue(CollectionSchema.sku.getSolrFieldName())); + title = this.getStringFrom(doc.getFieldValue(CollectionSchema.title.getSolrFieldName())); + author = this.getStringFrom(doc.getFieldValue(CollectionSchema.author.getSolrFieldName())); + description = this.getStringFrom(doc.getFieldValue(CollectionSchema.description_txt.getSolrFieldName())); size = (Integer) doc.getFieldValue(CollectionSchema.size_i.getSolrFieldName()); date = (Date) doc.getFieldValue(CollectionSchema.last_modified.getSolrFieldName()); if (this.pattern != null && !this.pattern.matcher(url).matches()) continue; @@ -982,18 +903,18 @@ public final class Fulltext { pw.println("</item>"); } this.docCount++; this.chunkSize++; - if (this.chunkSize >= this.maxChunkSize) { - printTail(pw); + if (this.maxChunkSize > 0 && this.chunkSize >= this.maxChunkSize) { + this.printTail(pw); pw.close(); this.chunkCount++; - pw = getWriter(); - printHead(pw); + pw = this.getWriter(); + this.printHead(pw); this.chunkSize = 0; } } } } - printTail(pw); + this.printTail(pw); pw.close(); } catch (final Exception e) { /* Catch but log any IO exception that can occur on copy, automatic closing or streams creation */ @@ -1004,12 +925,17 @@ public final class Fulltext { } public File file() { - final File f = new File(this.path, this.filename + "_" + chunkcount(this.chunkCount) + "." + this.fileext); + if (this.maxChunkSize <= 0 || this.maxChunkSize == Long.MAX_VALUE) { + // no chunking, just one file + final File f = new File(this.path, this.filename + "." + this.fileext); + return f; + } + final File f = new File(this.path, this.filename + "_" + this.chunkcount(this.chunkCount) + "." + this.fileext); return f; } private PrintWriter getWriter() throws IOException { - final File f = file(); + final File f = this.file(); final OutputStream os = new FileOutputStream(this.format == ExportFormat.solr ? new File(f.getAbsolutePath() + ".gz") : f); final PrintWriter pw = new PrintWriter(new BufferedOutputStream(((this.format == ExportFormat.solr)) ? new GZIPOutputStream(os, 65536){{this.def.setLevel(Deflater.BEST_COMPRESSION);}} : os)); return pw; diff --git a/source/net/yacy/search/navigator/NavigatorPlugins.java b/source/net/yacy/search/navigator/NavigatorPlugins.java index 884c552bc..019c6bbac 100644 --- a/source/net/yacy/search/navigator/NavigatorPlugins.java +++ b/source/net/yacy/search/navigator/NavigatorPlugins.java @@ -188,7 +188,7 @@ public class NavigatorPlugins { tmpnav.addForbidden("robot_" + CrawlSwitchboard.CRAWL_PROFILE_GREEDY_LEARNING_TEXT);
tmpnav.addForbidden("robot_" + CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA);
tmpnav.addForbidden("robot_" + CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA);
- tmpnav.addForbidden("robot_" + CrawlSwitchboard.CRAWL_PROFILE_SURROGATE);
+ tmpnav.addForbidden("robot_" + CrawlSwitchboard.CRAWL_PROFILE_PACKS);
navigatorPlugins.put("collections", tmpnav);
} else if ("filetype".equals(navName)) {
navigatorPlugins.put("filetype", new FileTypeNavigator("Filetype", CollectionSchema.url_file_ext_s,
diff --git a/source/net/yacy/search/schema/CollectionConfiguration.java b/source/net/yacy/search/schema/CollectionConfiguration.java index 1e5c5b2f6..9f3787673 100644 --- a/source/net/yacy/search/schema/CollectionConfiguration.java +++ b/source/net/yacy/search/schema/CollectionConfiguration.java @@ -893,7 +893,7 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri }
if (scraper instanceof DCEntry) {
- // the document was created with a surrogate parsing; overwrite all md: -entries to Solr
+ // the document was created with a pack parsing; overwrite all md: -entries to Solr
final DCEntry dcentry = (DCEntry) scraper;
for (final Map.Entry<String, String[]> entry: dcentry.getMap().entrySet()) {
final String tag = entry.getKey();
|