diff options
69 files changed, 105 insertions, 7383 deletions
diff --git a/.classpath b/.classpath index 2132cdb45..b4fbbf155 100644 --- a/.classpath +++ b/.classpath @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <classpath> - <classpathentry excluding="api/|env/|processing/domaingraph/applet/|yacy/|api/bookmarks/|api/ymarks/|api/bookmarks/posts/|api/bookmarks/tags/|api/bookmarks/xbel/|solr/|gsa/|solr/collection1/|api/blacklists/|proxymsg/|p2p/" kind="src" path="htroot"/> - <classpathentry excluding="bookmarks/|ymarks/|bookmarks/posts/|bookmarks/tags/|bookmarks/xbel/|blacklists/" kind="src" path="htroot/api"/> + <classpathentry excluding="api/|env/|processing/domaingraph/applet/|yacy/|api/bookmarks/|api/bookmarks/posts/|api/bookmarks/tags/|api/bookmarks/xbel/|solr/|gsa/|solr/collection1/|api/blacklists/|proxymsg/|p2p/" kind="src" path="htroot"/> + <classpathentry excluding="bookmarks/|bookmarks/posts/|bookmarks/tags/|bookmarks/xbel/|blacklists/" kind="src" path="htroot/api"/> <classpathentry excluding="posts/|tags/|xbel/" kind="src" path="htroot/api/bookmarks"/> <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/> <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"> @@ -120,7 +120,6 @@ <classpathentry kind="src" path="htroot/api/bookmarks/posts"/> <classpathentry kind="src" path="htroot/api/bookmarks/tags"/> <classpathentry kind="src" path="htroot/api/bookmarks/xbel"/> - <classpathentry kind="src" path="htroot/api/ymarks"/> <classpathentry kind="src" path="htroot/env"/> <classpathentry kind="src" path="htroot/p2p"/> <classpathentry kind="src" path="htroot/proxymsg"/> diff --git a/.settings/org.eclipse.wst.validation.prefs b/.settings/org.eclipse.wst.validation.prefs new file mode 100644 index 000000000..7983aae5c --- /dev/null +++ b/.settings/org.eclipse.wst.validation.prefs @@ -0,0 +1,11 @@ +DELEGATES_PREFERENCE=delegateValidatorList +USER_BUILD_PREFERENCE=enabledBuildValidatorList +USER_MANUAL_PREFERENCE=enabledManualValidatorList +USER_PREFERENCE=overrideGlobalPreferencestruedisableAllValidationfalseversion1.2.800.v201904082137 +eclipse.preferences.version=1 +override=true +suspend=false +vals/org.eclipse.wst.dtd.core.dtdDTDValidator/global=FF01 +vals/org.eclipse.wst.xml.core.xml/global=FF03 +vals/org.eclipse.wst.xsd.core.xsd/global=FF02162org.eclipse.wst.xsd.core.internal.validation.eclipse.Validator +vf.version=3 diff --git a/htroot/Table_YMark_p.html b/htroot/Table_YMark_p.html deleted file mode 100644 index 6d520fb70..000000000 --- a/htroot/Table_YMark_p.html +++ /dev/null @@ -1,212 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml"> - <head> - <title>YaCy '#[clientname]#': Table Viewer</title> - #(showtable)#:: - <link rel="alternate" type="application/xml" title="Tables" href="Tables.rss?table=#[table]#" /> - #(/showtable)# - #%env/templates/metas.template%# - <script src="yacy/ui/js/jquery-1.3.2.min.js" type="text/javascript"></script> - <script src="yacy/ui/js/jquery.dimensions.min.js" type="text/javascript"></script> - <script type="text/javascript" src="js/sorttable.js"></script> - <script type="text/javascript"> - <!-- - function setall(name, check){ - var selectForm = document.forms.namedItem(name); - var count = selectForm.elements["num"].value; - if (check) for(i = 0; i < count; i++) { - if (selectForm.elements["item_" + i].checked) { - check = false; - break; - } - } - for(i = 0; i < count; i++) { - selectForm.elements["item_" + i].checked = check; - } - }; - $(function() { - $(window).resize(function(){ - var width = document.documentElement.clientWidth; - var height = document.documentElement.clientHeight; - width = width - 490; - height = height - 250; - $("#table").width(width); - $("#table").height(height); - $("#content_area").width(width); - }); - $(window).trigger("resize"); - }); - --> - </script> - </head> - <body id="YMark"> - #%env/templates/header.template%# - #%env/templates/submenuConfig.template%# - <h2>YMark Table Administration</h2> - <div id="content_area" style="float: left;"> - #(showtable)#:: - <form action="Table_YMark_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8" id="tablelisting" name="tablelisting"> - <fieldset> - <legend><label for="tablelisting">Table Editor: showing table '#[table]#' containing #[bmksize]# bookmarks with #[tagsize]# tags in #[foldersize]# folders - </label></legend> - <p> - <input type="hidden" name="table" value="#[table]#" /> - <input type="hidden" name="num" value="#[num]#" /> - <input type="submit" name="editrow" value="Edit Selected Row" /> - <input type="submit" name="addrow" value="Add a new Row" disabled="disabled"/> - <input type="submit" name="deleterows" value="Delete Selected Rows" /> - <input type="submit" name="deletetable" value="Delete Table" /> - <input type="submit" name="rebuildindex" value="Rebuild Index" /> - </p> - <div id="table" style="float:left; overflow:auto;"> - <table class="sortable" border="0"> - <tr class="TableHeader" valign="bottom"> - <td><input type="checkbox" name="allswitch" onclick="setall(this.form.name, this.value)" /></td> - <td>PK</td> - #{columns}# - <td>#[header]#</td> - #{/columns}# - </tr> - #{list}# - <tr class="TableCell#(dark)#Light::Dark::Summary#(/dark)#"> - <td align="left"><input type="checkbox" name="item_#[count]#" value="mark_#[pk]#" /></td> - <td>#[pk]#</td> - #{columns}# - <td align="left">#[cell]#</td> - #{/columns}# - </tr> - #{/list}# - </table> - </div> - </fieldset> - </form> - #(/showtable)# - #(showedit)#:: - <div> - <form action="Table_YMark_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8"> - <fieldset><legend>Row Editor</legend> - <dl> - <dt class="TableCellLight">Primary Key</dt> - <dd>#[pk]#</dd> - #{list}# - <dt class="TableCellLight">#[key]#</dt> - <dd> - <input type="text" name="col_#[key]#" value="#[value]#" size="65"/> - </dd> - #{/list}# - <dt class="TableCellLight"></dt> - <dd> - <input type="hidden" name="table" value="#[table]#" /> - <input type="hidden" name="pk" value="#[pk]#" /> - <input type="submit" name="commitrow" value="Commit" /> - </dd> - </dl> - </fieldset> - </form> - </div> - #(/showedit)# - </div> - #(showselection)#:: - <div style="width:310px; float:right;"> - <form action="Table_YMark_p.html" method="get" accept-charset="UTF-8"> - <fieldset><legend>Table Selection</legend> - <dl> - <dt> - <label>Select Table:</label> - </dt> - <dd> - <select name="table" onchange='this.form.submit()'> - #{tables}# - <option value="#[name]#" #(selected)#::selected="selected"#(/selected)#>#[name]#</option> - #{/tables}# - </select> - </dd> - <dt> - <label for="count">show max. entries</label> - </dt> - <dd> - <select name="count" id="count" onchange='this.form.submit()'> - <option value="10">10</option> - <option value="100">100</option> - <option value="1000">1000</option> - <option value="all" selected="selected">all</option> - </select> - </dd> - <dt> - <label>Display columns:</label> - </dt> - <dd> - #{columns}# - <input type="checkbox" name="col_#[col]#" #(checked)#::checked="checked"#(/checked)# />#[col]#<br/> - #{/columns}# - </dd> - </dl> - <input type="submit" value="load" /> - </fieldset> - <fieldset> - <legend>Search/Filter Table</legend> - <dl> - <dt> - <label for="search">search rows for</label> - </dt> - <dd> - <input type="text" name="search" value="#[pattern]#" id="search" /> - <input type="submit" name="edittable" value="Search" /> - </dd> - <dt> - <label for="tags">Tags</label> - </dt> - <dd> - <select name="tags" style="width: 180px" onchange='this.form.submit()' id="tags"> - <option value="" selected="selected">select a tag</option> - #{tags}# - <option value="#[tagName]#">#[tagName]# [#[tagCount]#]</option> - #{/tags}# - </select> - </dd> - <dt> - <label for="folders">Folders</label> - </dt> - <dd> - <select name="folders" style="width: 180px" onchange='this.form.submit()' id="folders"> - <option value="" selected="selected">select a folder</option> - #{folders}# - <option value="#[folderName]#">#[folderName]# [#[folderCount]#]</option> - #{/folders}# - </select> - </dd> - </dl> - </fieldset> - </form> - <fieldset> - <legend>Import Bookmarks</legend> - <form action="api/ymarks/import_ymark.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8"> - <fieldset> - <input type="hidden" name="table" value="#[table]#" /> - <input type="hidden" name="redirect" value="Table_YMark_p.html?table=#[table]#&count=100" /> - <dl> - <dt> - <label for="importer">Importer:</label> - </dt> - <dd> - <select name="importer" id="importer"> - <option value="xbel">XBEL Importer</option> - <option value="html">Netscape HTML Importer</option> - </select> - </dd> - <dt> - <label for="bmkfile">File:</label> - </dt> - <dd> - <input type="file" name="bmkfile" id="bmkfile" size="10"/> - </dd> - </dl> - <input type="submit" name="importbookmarks" value="import" /> - </fieldset> - </form> - </fieldset> - </div> - #(/showselection)# - #%env/templates/footer.template%# - </body> -</html> diff --git a/htroot/Table_YMark_p.java b/htroot/Table_YMark_p.java deleted file mode 100644 index 43cb1d991..000000000 --- a/htroot/Table_YMark_p.java +++ /dev/null @@ -1,296 +0,0 @@ -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.regex.Pattern; - -import net.yacy.cora.document.encoding.UTF8; -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.cora.util.SpaceExceededException; -import net.yacy.data.ymark.YMarkEntry; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.data.ymark.YMarkUtil; -import net.yacy.kelondro.blob.Tables; -import net.yacy.search.Switchboard; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - - -public class Table_YMark_p { - public static serverObjects respond(@SuppressWarnings("unused") final RequestHeader header, final serverObjects post, final serverSwitch env) { - final Switchboard sb = (Switchboard) env; - final serverObjects prop = new serverObjects(); - - prop.put("showtable", 0); - prop.put("showedit", 0); - prop.put("showselection", 0); - - String table = (post == null) ? "admin_bookmarks" : post.get("table", "admin_bookmarks"); - if (table != null && !sb.tables.hasHeap(table)) table = null; - - // get the user name for the selected table - String bmk_user = null; - if (table != null) - bmk_user = table.substring(0,table.indexOf('_',0)); - - // currently selected table - prop.put("showselection_table", table); - - // show table selection - int count = 0; - final Iterator<String> ti = sb.tables.iterator(); - String tablename; - prop.put("showselection", 1); - while (ti.hasNext()) { - tablename = ti.next(); - if(tablename.endsWith(YMarkTables.TABLES.BOOKMARKS.basename())) { - prop.put("showselection_tables_" + count + "_name", tablename); - prop.put("showselection_tables_" + count + "_selected", (table != null && table.equals(tablename)) ? 1 : 0); - count++; - } - } - prop.put("showselection_tables", count); - prop.put("showselection_pattern", ""); - - if (post == null) return prop; // return rewrite properties - - // get available tags and folders - count = 0; - /* - byte[] key; - String name; - try { - Iterator<byte[]> iter = sb.tables.keys(YMarkTables.TABLES.TAGS.tablename(bmk_user)); - while(iter.hasNext()) { - key = iter.next(); - name = sb.tables.bookmarks.tags.getKeyname(bmk_user, key); - prop.put("showselection_tags_" + count + "_tagHash", UTF8.String(key)); - prop.put("showselection_tags_" + count + "_tagName", name); - prop.put("showselection_tags_" + count + "_tagCount", sb.tables.bookmarks.tags.getBookmarkIds(bmk_user, name).size()); - count++; - } - prop.put("showselection_tags", count); - count = 0; - iter = sb.tables.keys(YMarkTables.TABLES.FOLDERS.tablename(bmk_user)); - while(iter.hasNext()) { - key = iter.next(); - name = sb.tables.bookmarks.folders.getKeyname(bmk_user, key); - prop.put("showselection_folders_" + count + "_folderHash", UTF8.String(key)); - prop.put("showselection_folders_" + count + "_folderName", name); - prop.put("showselection_folders_" + count + "_folderCount", sb.tables.bookmarks.folders.getBookmarkIds(bmk_user, name).size()); - count++; - } - prop.put("showselection_folders", count); - } catch (final IOException e) { - Log.logException(e); - } catch (final RowSpaceExceededException e) { - Log.logException(e); - } - */ - - final String counts = post.get("count", null); - int maxcount = (counts == null || counts.equals("all")) ? Integer.MAX_VALUE : post.getInt("count", 10); - final String pattern = post.get("search", ""); - final Pattern matcher = (pattern.isEmpty() || pattern.equals(".*")) ? null : Pattern.compile(".*" + pattern + ".*"); - prop.put("pattern", pattern); - - List<String> columns = new ArrayList<String>(); - for (final Map.Entry<String, String> entry: post.entrySet()) { - if (entry.getKey().startsWith("col_")) { - columns.add(entry.getKey().substring(4)); - } - } - if (columns.isEmpty() && table != null) try { - columns = sb.tables.columns(table); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - - count = 0; - if (table != null) { - Iterator<String> cit; - String col; - try { - cit = sb.tables.columns(table).iterator(); - while(cit.hasNext()) { - col = cit.next(); - prop.put("showselection_columns_" + count + "_col", col); - prop.put("showselection_columns_" + count + "_checked", columns.contains(col) ? 1 : 0); - count++; - } - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - } - prop.put("showselection_columns", count); - - // apply deletion requests - if (!post.get("deletetable", "").isEmpty()) { - sb.tables.clear(table); - sb.tables.clear(YMarkTables.TABLES.FOLDERS.tablename(bmk_user)); - sb.tables.clear(YMarkTables.TABLES.TAGS.tablename(bmk_user)); - } - - - // apply rebuildIndex request - /* - if (!post.get("rebuildindex", "").isEmpty()) try { - sb.tables.bookmarks.folders.rebuildIndex(bmk_user); - sb.tables.bookmarks.tags.rebuildIndex(bmk_user); - } catch (final IOException e) { - Log.logException(e); - } - */ - - if (!post.get("deleterows", "").isEmpty()) { - for (final Map.Entry<String, String> entry: post.entrySet()) { - if (entry.getValue().startsWith("mark_")) try { - sb.tables.bookmarks.deleteBookmark(bmk_user, entry.getValue().substring(5).getBytes()); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final SpaceExceededException e) { - ConcurrentLog.logException(e); - } - } - } - - if (!post.get("commitrow", "").isEmpty()) { - final YMarkEntry bmk = new YMarkEntry(); - for (final Map.Entry<String, String> entry: post.entrySet()) { - if (entry.getKey().startsWith("col_")) { - bmk.put(entry.getKey().substring(4), entry.getValue()); - } - } - try { - sb.tables.bookmarks.addBookmark(bmk_user, bmk, false, false); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - } - - // generate table - prop.put("showtable", 0); - prop.put("showedit", 0); - - if (table != null) { - - if (post.containsKey("editrow")) { - // check if we can find a key - String pk = null; - for (final Map.Entry<String, String> entry: post.entrySet()) { - if (entry.getValue().startsWith("mark_")) { - pk = entry.getValue().substring(5); - break; - } - } - try { - if (pk != null && sb.tables.has(table, pk.getBytes())) { - setEdit(sb, prop, table, pk, columns); - } - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final SpaceExceededException e) { - ConcurrentLog.logException(e); - } - } else if (post.containsKey("addrow")) try { - // get a new key - final String pk = UTF8.String(sb.tables.createRow(table)); - setEdit(sb, prop, table, pk, columns); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final SpaceExceededException e) { - ConcurrentLog.logException(e); - } else { - prop.put("showtable", 1); - prop.put("showtable_table", table); - - - try { - prop.put("showtable_bmksize", sb.tables.size(table)); - prop.put("showtable_tagsize", sb.tables.size(YMarkTables.TABLES.TAGS.tablename(bmk_user))); - prop.put("showtable_foldersize", sb.tables.size(YMarkTables.TABLES.FOLDERS.tablename(bmk_user))); - } catch (final IOException e) { - ConcurrentLog.logException(e); - prop.put("showtable_bmksize", 0); - prop.put("showtable_tagsize", 0); - prop.put("showtable_foldersize", 0); - } - - // insert the columns - - for (int i = 0; i < columns.size(); i++) { - prop.putHTML("showtable_columns_" + i + "_header", columns.get(i)); - } - prop.put("showtable_columns", columns.size()); - - // insert all rows - try { - maxcount = Math.min(maxcount, sb.tables.size(table)); - } catch (final IOException e) { - ConcurrentLog.logException(e); - maxcount = 0; - } - count = 0; - try { - Iterator<Tables.Row> mapIterator; - if (post.containsKey("folders") && !post.get("folders").isEmpty()) { - // mapIterator = sb.tables.orderByPK(sb.tables.bookmarks.folders.getBookmarks(bmk_user, post.get("folders")), maxcount).iterator(); - mapIterator = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, post.get("folders")); - } else if(post.containsKey("tags") && !post.get("tags").isEmpty()) { - // mapIterator = sb.tables.orderByPK(sb.tables.bookmarks.tags.getBookmarks(bmk_user, post.get("tags")), maxcount).iterator(); - final String tagsString = YMarkUtil.cleanTagsString(post.get(YMarkEntry.BOOKMARK.TAGS.key())); - mapIterator = sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagsString); - } else { - mapIterator = sb.tables.iterator(table, matcher, true); - } - - Tables.Row row; - boolean dark = true; - byte[] cell; - while (mapIterator.hasNext() && count < maxcount) { - row = mapIterator.next(); - if (row == null) continue; - - // write table content - prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark; - prop.put("showtable_list_" + count + "_pk", UTF8.String(row.getPK())); - prop.put("showtable_list_" + count + "_count", count); - for (int i = 0; i < columns.size(); i++) { - cell = row.get(columns.get(i)); - prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cell == null ? "" : UTF8.String(cell)); - } - prop.put("showtable_list_" + count + "_columns", columns.size()); - count++; - } - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - prop.put("showtable_list", count); - prop.put("showtable_num", count); - } - - } - - // return rewrite properties - return prop; - } - - private static void setEdit(final Switchboard sb, final serverObjects prop, final String table, final String pk, final List<String> columns) throws IOException, SpaceExceededException { - prop.put("showedit", 1); - prop.put("showedit_table", table); - prop.put("showedit_pk", pk); - final Tables.Row row = sb.tables.select(table, pk.getBytes()); - if (row == null) return; - int count = 0; - byte[] cell; - for (final String col: columns) { - cell = row.get(col); - prop.put("showedit_list_" + count + "_key", col); - prop.put("showedit_list_" + count + "_value", cell == null ? "" : UTF8.String(cell)); - count++; - } - prop.put("showedit_list", count); - } -} diff --git a/htroot/YMarks.html b/htroot/YMarks.html deleted file mode 100644 index e05d0ba3c..000000000 --- a/htroot/YMarks.html +++ /dev/null @@ -1,436 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml"> - <head> - <title>YaCy Bookmarks</title> - - <!-- the metas.template can not be used here while some JS components (such as flexigrid or treeview) - are not compatible with the main embedded JQuery version (1.12.4) --> - - <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> - <link rel="shortcut icon" type="image/x-icon" href="#[relativeBase]#favicon.ico" /> - <meta name="Content-Language" content="English, Englisch" /> - <meta name="keywords" content="YaCy HTTP search engine spider indexer java network open free download Mac Windows Linux Software development" /> - <meta name="description" content="Software HTTP Freeware Home Page" /> - <meta name="copyright" content="Michael Christen et al." /> - <meta name="referrer" content="#[referrer.meta.policy]#"/> - - <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries --> - <!--[if lt IE 9]> - <script src="#[relativeBase]#env/bootstrap/js/html5shiv.min.js"></script> - <script src="#[relativeBase]#env/bootstrap/js/respond.min.js"></script> - <![endif]--> - - <!-- old css styles --> - <link rel="stylesheet" type="text/css" media="all" href="#[relativeBase]#env/base.css" /> - <link rel="stylesheet" type="text/css" media="screen" href="#[relativeBase]#env/style.css" /> - <!--[if lt IE 6]> - <link rel="stylesheet" type="text/css" media="screen" href="#[relativeBase]#env/oldie.css" /> - <![endif]--> - <!--[if lte IE 6.0]> - <link rel="stylesheet" type="text/css" media="screen" href="#[relativeBase]#env/ie6.css" /> - <![endif]--> - <!--[if lte IE 7.0]> - <link rel="stylesheet" type="text/css" media="screen" href="#[relativeBase]#env/ie7.css" /> - <![endif]--> - <!-- (C), Architecture: Michael Peter Christen; Contact: mc <at> yacy.net --> - - - <script src="jquery/js/jquery-1.7.min.js" type="text/javascript"></script> - - <link media="screen" type="text/css" href="jquery/themes/start/jquery-ui-1.8.16.custom.css" rel="stylesheet" /> - <script src="jquery/js/jquery-ui-1.8.16.custom.min.js" type="text/javascript"></script> - - <script src="jquery/js/jquery.form-2.73.js" type="text/javascript"></script> - <script src="jquery/js/jquery.field-0.9.2.min.js" type="text/javascript"></script> - - <link media="screen" type="text/css" href="jquery/flexigrid/css/flexigrid.pack.css" rel="stylesheet" /> - <script src="jquery/flexigrid/js/flexigrid.pack.js" type="text/javascript"></script> - - <link media="screen" type="text/css" href="jquery/css/jquery.multiselect.css" rel="stylesheet" /> - <script src="jquery/js/jquery.multiselect.min.js" type="text/javascript"></script> - - <link media="screen" type="text/css" href="jquery/css/jquery.tagsinput.css" rel="stylesheet" /> - <script src="jquery/js/jquery.tagsinput.min.js" type="text/javascript"></script> - - <link media="screen" type="text/css" href="jquery/css/jquery.multiselect.filter.css" rel="stylesheet" /> - <script src="jquery/js/jquery.multiselect.filter.min.js" type="text/javascript"></script> - - <link media="screen" type="text/css" href="yacy/ui/css/jquery.treeview.css" rel="stylesheet" /> - <script src="yacy/ui/js/jquery.treeview.min.js" type="text/javascript"></script> - <script src="yacy/ui/js/jquery.treeview.async.js" type="text/javascript"></script> - - <link media="screen" type="text/css" href="env/yacy-ymarks.css" rel="stylesheet" /> - <script src="js/yacy-ymarks.js" type="text/javascript"></script> - <script src="js/yacy-ymarks-bookmark-actions.js" type="text/javascript"></script> - <script src="js/yacy-ymarks-tag-actions.js" type="text/javascript"></script> - - </head> - <body id="ymarks_body"> - - - #%env/templates/embeddedheader.template%# - -<div id="api"> -<a href="api/table_p.xml?table=admin_bookmarks&count=100&search=" id="apilink"><img src="env/grafics/api.png" width="60" height="40" alt="API"/></a> -<span>The information that is presented on this page can also be retrieved as XML. -Click the API icon to see the XML. -To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de/wiki/index.php/Dev:API" target="_blank">API wiki page</a>.</span> -</div> - - <div class="SubMenu"> - <h3>Bookmarks (user: #[user]# size: #[size]#)</h3> - <!-- - <ul class="SubMenu"> - #(login)#<li><a href="YMarks.html" class="MenuItemLink">Login</a></li>::#(/login)# - <li><a href="api/ymarks/test_treeview.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">TreeView</a></li> - <li><a href="api/ymarks/test_import.html" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Import Bookmarks</a></li> - <li><a href="api/ymarks/get_xbel.xml" class="MenuItemLink #(authorized)#lock::unlock#(/authorized)#">Bookmarks (XBEL)</a></li> - </ul> - <p/> - --> - </div> - <div id="ymarks_sidebar" class="ymark-sidebar" style="overflow: auto;"> - <ul> - <li><a href="#ymarks_nav_tab" title="navigation"><span>Explorer</span></a></li> - <li><a href="#ymarks_tags_tab" title="tags"><span>Tag Manager</span></a></li> - <li><a href="#ymarks_import_tab" title="import"><span>Import</span></a></li> - <!-- <li><a href="#ymarks_export_tab" title="export"><span>Export</span></a></li> --> - </ul> - <div id="ymarks_nav_tab"> - <!-- - <div style="margin-top:2px; background:url('yacy/ui/img/flexigrid/bg.gif') repeat-x scroll center top #FAFAFA; - border-color:#CCCCCC #CCCCCC -moz-use-text-color; border-style:solid solid none; border-width:1px 1px 0; padding:5px; - margin-top:-10px; margin-left:-15px; margin-right:-15px; margin-bottom: 15px;"> - <img src="yacy/ui/img-2/plus.png" alt="Add" style="padding-right:4px; padding-left:5px;"/>Add - <img src="yacy/ui/img-2/cancel.png" alt="Delete" style="padding-right:4px; padding-left:5px;"/>Delete - <img src="yacy/ui/img-2/pencil.png" alt="Edit" style="padding-right:4px; padding-left:5px;"/>Edit - <img src="yacy/ui/img-2/question_blue.png" alt="Help" style="padding-right:4px; padding-left:25px;"/>Help - </div> - --> - <div class="tagSelect"> - <p> - <select id="tag_include" name="tag_include" multiple="multiple"></select> - </p> - </div> - <div> - <ul id="ymarks_treeview" class="filetree"></ul> - </div> - </div> - <div id="ymarks_tags_tab"> - <!-- - <div style="margin-top:2px; background:url('yacy/ui/img/flexigrid/bg.gif') repeat-x scroll center top #FAFAFA; - border-color:#CCCCCC #CCCCCC -moz-use-text-color; border-style:solid solid none; border-width:1px 1px 0; padding:5px; - margin-top:-10px; margin-left:-15px; margin-right:-15px; margin-bottom: 15px;"> - <img src="yacy/ui/img-2/plus.png" alt="Add" style="padding-right:4px; padding-left:5px;"/>Add - <img src="yacy/ui/img-2/cancel.png" alt="Delete" style="padding-right:4px; padding-left:5px;"/>Delete - <img src="yacy/ui/img-2/pencil.png" alt="Edit" style="padding-right:4px; padding-left:5px;"/>Edit - <img src="yacy/ui/img-2/question_blue.png" alt="Help" style="padding-right:4px; padding-left:25px;"/>Help - </div> - --> - <div> - <p id="ymarks_tagcloud"></p> - </div> - <hr /> - <div class="tagSelect"> - <form id="ymarks_tagmanager" action="YMarks.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8"> - <h4>Tag Manager<img title="help" alt="help" class="help" src="yacy/ui/img-2/question_blue.png" /></h4> - All tag actions are applied to the sub-set of bookmarks defined by this query. - <p> - <small>Query</small> - <input class="input" type="text" name="query" id="query" /> - <br /> - <small>Query Type</small> - <select id= "ymarks_qtype" name="qtype"> - <option value="_tags" selected="selected">Tags (comma seperated)</option> - <option value="tags">Tags (regexp)</option> - <option value="_folder">Folders (comma seperated)</option> - <option value="folders">Folders (regexp)</option> - <option value="title">Title (regexp)</option> - <option value="desc">Description (regexp)</option> - </select> - </p> - <hr /> - <p> - <select id="tag_select" name="tags" multiple="multiple"></select><br /> - </p> - <p> - Enter tags to add (<i>replace with</i>) - <br /><small>(comma separated tags)</small> - <input class="input" type="text" name="replace" id="source" value="" /> - </p> - <hr /> - <p style="text-align: right"> - <input type="submit" name="action" value="Replace" /> - </p> - <!-- - <hr /> - <h5>Automatic tagging</h5> - <p> - <input type="radio" name="autotag" value="empty" checked="checked"/> Only for empty/unsorted tags - <br /> - <input type="radio" name="autotag" value="overwrite" /> Overwriting existing tags - <br /> - <input type="radio" name="autotag" value="merge" /> Merging with existing tags - </p> - <p style="text-align: right"> - <input type="submit" name="tagmanager" value="Run" /> - </p> - --> - </form> - </div> - </div> - <div id="ymarks_import_tab"> - <form action="/api/ymarks/import_ymark.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8"> - <input type="hidden" value="/YMarks.html" name="redirect"> - <h4>Bookmark Importer<img alt="help" title="If you put in your bookmarks here, you can access them anywhere where you have access to your YaCy peer. Think of it as your 'personal cloud' for bookmarking." class="help" src="yacy/ui/img-2/question_blue.png"></h4> - <p> - <table cellspacing="3"> - <tr> - <td> <input type="radio" name="importer" value="html" checked="checked" /> </td> - <td>Netscape HTML</td> - </tr> - <tr> - <td><input type="radio" name="importer" value="json" /></td> - <td>Firefox JSON</td> - </tr> - <tr> - <td><input type="radio" name="importer" value="xbel" /></td> - <td>XBEL</td> - </tr> - <tr> - <td><input type="radio" name="importer" value="surro" /></td> - <td>Surrogate XML</td> - </tr> - <tr> - <td><input type="radio" name="importer" value="dmoz" /></td> - <td>DMOZ XML</td> - </tr> - <tr> - <td><input type="radio" name="importer" value="list" disabled="disabled"/></td> - <td>YaCy White/Black List</td> - </tr> - <tr> - <td><input type="radio" name="importer" value="bmks" /></td> - <td>YaCy bookmarks.db (admin)</td> - </tr> - <tr> - <td><input type="radio" name="importer" value="crawls" /></td> - <td>YaCy Crawl Starts (admin)</td> - </tr> - </table> - </p> - <p> - <small>Bookmark file</small> - <input class="input" type="file" name="bmkfile" id="bmkfile" size="8" /><br /> - </p> - <hr /> - <h4>Folder settings<img alt="help" title="A folder structure is helpful to organize your bookmarks in a hierarchical way." class="help" src="yacy/ui/img-2/question_blue.png"></h4> - <p> - <small>Source folder</small> - <input class="input" type="text" name="source" id="source" value="" disabled="disabled" /> - </p> - <p> - <small>Target folder</small> - <input class="input" type="text" name="root" id="root" value="/Imported Bookmarks" /> - </p> - <hr /> - <h4>Automatic tagging<img alt="help" title="Tags are words that are attached to documents as metadata. It is possible to read all the documents and find the attached tags automatically." class="help" src="yacy/ui/img-2/question_blue.png"></h4> - <p> - <table cellspacing="5"> - <tr> - <td><input type="radio" name="autotag" value="off" checked="checked" /></td> - <td>Off</td> - </tr> - <tr> - <td><input type="radio" name="autotag" value="empty" /></td> - <td>Only for empty tags</td> - </tr> - <tr> - <td><input type="radio" name="autotag" value="overwrite" /></td> - <td>Overwriting existing tags</td> - </tr> - <tr> - <td><input type="radio" name="autotag" value="merge" /></td> - <td>Merging with existing tags</td> - </tr> - </table> - </p> - <hr /> - <h4>Automatic Indexing<img alt="help" title="While doing the bookmark import, YaCy can push all URLs to the indexing process" class="help" src="yacy/ui/img-2/question_blue.png"></h4> - <p> - <table cellspacing="5"> - <tr> - <td><input type="radio" name="indexing" value="off" checked="checked" /></td> - <td>No indexing</td> - </tr> - <tr> - <td><input type="radio" name="indexing" value="single" /></td> - <td>Index every bookmark entry</td> - </tr> - <tr> - <td><input type="radio" name="indexing" value="onelink" /></td> - <td>Index every bookmark entry plus all directly linked pages</td> - </tr> - <tr> - <td><input type="radio" name="indexing" value="fulldomain" /></td> - <td>Index all domains from all bookmarks completely</td> - </tr> - <tr> - <td><input type="checkbox" name="medialink" /></td> - <td><br />include all media (image/movie/document) links</td> - </tr> - </table> - </p> - <hr /> - <p style="text-align: right"> - <input type="submit" name="importbookmarks" value="Import" /> - </p> - </form> - </div> - <!-- - <div id="ymarks_export_tab"> - </div> - --> - </div> - <!-- Display Bookmarks Table --> - <div class ="main"> - <table id="ymarks_flexigrid" summary="YaCy Bookmarks"> - <tbody> - </tbody> - </table> - </div> - - <!-- Bookmarks Edit Dialog --> - <div id="ymarks_add_dialog" class="bm_dialog" title="Add & Edit Bookmark"> - <form id="bmaddform" method="post" accept-charset="UTF-8" action="jQuery"> - <table> - <tr> - <td> - <img id="bmaddimg" src="yacy/ui/img-1/Star.png" /> - </td> - <td> - <label for="bm_public">Public:</label> - <br /> - <select name="public" id="bm_public" class="bm_select"> - <option value="true">yes</option> - <option value="false">no</option> - </select> - </td> - </tr> - </table> - <div> - <label for="bm_url">URL:</label> - <br /> - <input type="text" name="url" id="bm_url" class="bm_input" size="80" /> - <br /> - <label for="bm_title">Title:</label> - <br /> - <input type="text" name="title" id="bm_title" class="bm_input" size="80" /> - <br /> - <label for="bm_desc">Description:</label> - <br /> - <textarea name="desc" id="bm_desc" class="bm_input" rows="3" cols="20"> - </textarea> - <br /> - <label for="bm_path">Folder (/folder/subfolder):</label> - <br /> - <input type="text" name="folders" id="bm_path" class="bm_input" size="80" /> - <br /> - <label for="bm_tags">Tags (comma separated):</label> - <br /> - <input type="text" name="tags" id="bm_tags" class="bm_input" size="80" /> - </div> - </form> - </div> - <div id="ymarks_crawlstart" class="bm_dialog" title="Craw Start"> - - <form id="ymarks_crawler" method="post" action="Crawler_p.html" enctype="multipart/form-data" accept-charset="UTF-8"> - <table cellspacing="10" cellpadding="20"> - <tr id="ymarks_crawlstart_msg"></tr> - <tr> - <td>Bookmark</td> - <td> - <input type="hidden" name="crawlingMode" id="crawlingMode" value="url" /> - <input name="crawlingURL" id="crawlingURL" type="text" size="35" maxlength="256" /> - - </td> - </tr> - <tr> - <td>Filter</td> - <td> - <input type="radio" name="range" id="rangeWide" value="wide" >No filter<br /> - <input type="hidden" id="mustmatch" type="text" value=".*" maxlength="100" size="60" name="mustmatch"> - - - <input type="radio" name="range" id="rangeDomain" value="domain" checked="checked"/>Restrict to start domain<br /> - <input type="radio" name="range" id="rangeSubpath" value="subpath" />Restrict to sub-path of given url - - <input type="hidden" name="mustnotmatch" id="mustnotmatch" value="" /> - <input type="hidden" name="crawlingDomFilterCheck" id="crawlingDomFilterCheck" value="off" /> - <input type="hidden" name="crawlingDomFilterDepth" id="crawlingDomFilterDepth" value="1" /> - </td> - <tr> - <tr> - <td>Crawling Depth</td> - <td> - <select name="crawlingDepth"> - <option value="0">bookmark only (0)</option> - <option value="4" selected="selected">shallow crawl (4)</option> - <option value="8">deep crawl (8)</option> - <option value="16">deeper crawl (16)</option> - <option value="99">indefinite (99)</option> - </select> - </td> - </tr> - <tr> - <td>Limitations</td> - <td> - <input type="checkbox" name="crawlingDomMaxCheck" id="crawlingDomMaxCheck" /> not more than - <input name="crawlingDomMaxPages" id="crawlingDomMaxPages" type="text" size="6" maxlength="6" value="10000" /> - documents - </td> - <tr> - <tr> - <td>Dynamic URLs</td> - <td> - <input type="checkbox" name="crawlingQ" id="crawlingQ" /> allow <a href="http://en.wikipedia.org/wiki/Query_string" target="_blank"> - query-strings</a> (urls with a '?' in the path) - <input type="hidden" name="storeHTCache" id="storeHTCache" value="on" /> - <input type="hidden" name="cachePolicy" id="cachePolicy" value="iffresh" /> - <input type="hidden" name="indexText" id="indexText" value="on" /> - <input type="hidden" name="indexMedia" id="indexMedia" value="on" /> - <input type="hidden" name="intention" id="intention" value="" /> - <input type="hidden" name="xsstopw" id="xsstopw" value="off" /> - <input type="hidden" name="xdstopw" id="xdstopw" value="off" /> - <input type="hidden" name="xpstopw" id="xpstopw" value="off" /> - </td> - <tr> - <tr> - <td>Scheduler</td> - <td> - <input type="radio" name="recrawl" value="nodoubles" checked="checked"/>run this crawl once<br/> - <input type="radio" name="recrawl" value="scheduler"/>scheduled, look every - <select name="repeat_time"> - <option value="1">1</option><option value="2">2</option><option value="3">3</option> - <option value="4">4</option><option value="5">5</option><option value="6">6</option> - <option value="7" selected="selected">7</option> - <option value="8">8</option><option value="9">9</option><option value="10">10</option> - <option value="12">12</option><option value="14">14</option><option value="21">21</option> - <option value="28">28</option><option value="30">30</option> - </select> - <select name="repeat_unit"> - <option value="selminutes">minutes</option> - <option value="selhours">hours</option> - <option value="seldays" selected="selected">days</option> - </select> - <br /> - for new documents automatically. - </td> - </tr> - </table> - <input type="hidden" value="Start New Crawl" name="crawlingstart"> - </form> - </div> - #%env/templates/embeddedfooter.template%# - </body> -</html>
\ No newline at end of file diff --git a/htroot/YMarks.java b/htroot/YMarks.java deleted file mode 100644 index c826b6a1c..000000000 --- a/htroot/YMarks.java +++ /dev/null @@ -1,36 +0,0 @@ -import java.io.IOException; - -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.data.UserDB; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.search.Switchboard; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - -public class YMarks { - public static serverObjects respond(final RequestHeader header, @SuppressWarnings("unused") final serverObjects post, final serverSwitch env) { - final Switchboard sb = (Switchboard) env; - final serverObjects prop = new serverObjects(); - final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header)); - final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - - if(isAdmin || isAuthUser) { - prop.put("login", 1); - final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - prop.putHTML("user", bmk_user.substring(0,1).toUpperCase() + bmk_user.substring(1)); - int size; - try { - size = sb.tables.bookmarks.getSize(bmk_user); - } catch (final IOException e) { - ConcurrentLog.logException(e); - size = 0; - } - prop.put("size", size); - } else { - prop.put("login", 0); - } - return prop; - } -}
\ No newline at end of file diff --git a/htroot/api/ymarks/add_ymark.html b/htroot/api/ymarks/add_ymark.html deleted file mode 100644 index ec376281a..000000000 --- a/htroot/api/ymarks/add_ymark.html +++ /dev/null @@ -1,14 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml"> - <head> - <title>Import Bookmarks</title> - #(redirect)#::<meta HTTP-EQUIV="REFRESH" content="0; url=#[url]#" />#(/redirect)# - </head> - <body> - #(status)# - <p>Status: error</p> - :: - <p>Status: ok</p> - #(/status)# - </body> -</html>
\ No newline at end of file diff --git a/htroot/api/ymarks/add_ymark.java b/htroot/api/ymarks/add_ymark.java deleted file mode 100644 index b079ae4f0..000000000 --- a/htroot/api/ymarks/add_ymark.java +++ /dev/null @@ -1,88 +0,0 @@ -import java.io.IOException; -import java.util.Locale; - -import net.yacy.cora.document.id.DigestURL; -import net.yacy.cora.protocol.ClientIdentification; -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.data.UserDB; -import net.yacy.data.ymark.YMarkEntry; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.data.ymark.YMarkUtil; -import net.yacy.document.Parser.Failure; -import net.yacy.search.Switchboard; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - - -public class add_ymark { - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - final Switchboard sb = (Switchboard) env; - final serverObjects prop = new serverObjects(); - - final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header)); - final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - - if (isAdmin || isAuthUser) { - final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - - if (post.containsKey("redirect") && post.get("redirect").length() > 0) { - prop.put("redirect_url", post.get("redirect")); - prop.put("redirect", "1"); - } - - if (post.containsKey("urlHash")) { - final String urlHash = post.get("urlHash",YMarkUtil.EMPTY_STRING); - try { - final DigestURL url = sb.index.fulltext().getURL(urlHash); - final String folders = post.get(YMarkEntry.BOOKMARK.FOLDERS.key(),YMarkEntry.BOOKMARK.FOLDERS.deflt()); - final String tags = post.get(YMarkEntry.BOOKMARK.TAGS.key(),YMarkUtil.EMPTY_STRING); - ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName)); - sb.tables.bookmarks.createBookmark(sb.loader, url, agent, bmk_user, true, tags, folders); - prop.put("status", "1"); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final Failure e) { - ConcurrentLog.logException(e); - } - - } else if (post.containsKey(YMarkEntry.BOOKMARK.URL.key())) { - String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt()); - boolean hasProtocol = false; - for (final YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) { - if(url.toLowerCase(Locale.ROOT).startsWith(p.protocol())) { - hasProtocol = true; - break; - } - } - if (!hasProtocol) { - url=YMarkTables.PROTOCOLS.HTTP.protocol(url); - } - - final YMarkEntry bmk = new YMarkEntry(); - - bmk.put(YMarkEntry.BOOKMARK.URL.key(), url); - bmk.put(YMarkEntry.BOOKMARK.TITLE.key(), post.get(YMarkEntry.BOOKMARK.TITLE.key(),YMarkEntry.BOOKMARK.TITLE.deflt())); - bmk.put(YMarkEntry.BOOKMARK.DESC.key(), post.get(YMarkEntry.BOOKMARK.DESC.key(),YMarkEntry.BOOKMARK.DESC.deflt())); - bmk.put(YMarkEntry.BOOKMARK.PUBLIC.key(), post.get(YMarkEntry.BOOKMARK.PUBLIC.key(),YMarkEntry.BOOKMARK.PUBLIC.deflt())); - bmk.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkUtil.cleanTagsString(post.get(YMarkEntry.BOOKMARK.TAGS.key(),YMarkEntry.BOOKMARK.TAGS.deflt()),YMarkEntry.BOOKMARK.TAGS.deflt())); - bmk.put(YMarkEntry.BOOKMARK.FOLDERS.key(), YMarkUtil.cleanFoldersString(post.get(YMarkEntry.BOOKMARK.FOLDERS.key(),YMarkEntry.BOOKMARK.FOLDERS.deflt()),YMarkEntry.BOOKMARK.FOLDERS.deflt())); - - try { - sb.tables.bookmarks.addBookmark(bmk_user, bmk, false, false); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - prop.put("status", "1"); - } else { - prop.put("status", "0"); - } - } else { - prop.put(serverObjects.ACTION_AUTHENTICATE, YMarkTables.USER_AUTHENTICATE_MSG); - } - // return rewrite properties - return prop; - } -} diff --git a/htroot/api/ymarks/add_ymark.xml b/htroot/api/ymarks/add_ymark.xml deleted file mode 100644 index a72c699cc..000000000 --- a/htroot/api/ymarks/add_ymark.xml +++ /dev/null @@ -1,2 +0,0 @@ -<?xml version='1.0' encoding="UTF-8" standalone='yes'?> -<status code="#(status)#error::ok#(/status)#" />
\ No newline at end of file diff --git a/htroot/api/ymarks/delete_ymark.java b/htroot/api/ymarks/delete_ymark.java deleted file mode 100644 index fdc8c7e1d..000000000 --- a/htroot/api/ymarks/delete_ymark.java +++ /dev/null @@ -1,52 +0,0 @@ -import java.io.IOException; - -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.cora.util.SpaceExceededException; -import net.yacy.data.UserDB; -import net.yacy.data.ymark.YMarkEntry; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.data.ymark.YMarkUtil; -import net.yacy.search.Switchboard; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - - -public class delete_ymark { - - private static Switchboard sb = null; - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - sb = (Switchboard) env; - final serverObjects prop = new serverObjects(); - - final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header)); - final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - - if(isAdmin || isAuthUser) { - final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - byte[] urlHash = null; - try { - if(post.containsKey(YMarkEntry.BOOKMARKS_ID)) { - urlHash = post.get(YMarkEntry.BOOKMARKS_ID).getBytes(); - } else if(post.containsKey(YMarkEntry.BOOKMARK.URL.key())) { - urlHash = YMarkUtil.getBookmarkId(post.get(YMarkEntry.BOOKMARK.URL.key())); - } else { - prop.put("result", "0"); - return prop; - } - sb.tables.bookmarks.deleteBookmark(bmk_user, urlHash); - prop.put("result", "1"); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final SpaceExceededException e) { - ConcurrentLog.logException(e); - } - } else { - prop.put(serverObjects.ACTION_AUTHENTICATE, YMarkTables.USER_AUTHENTICATE_MSG); - } - // return rewrite properties - return prop; - } -} diff --git a/htroot/api/ymarks/delete_ymark.xml b/htroot/api/ymarks/delete_ymark.xml deleted file mode 100644 index d6880fb21..000000000 --- a/htroot/api/ymarks/delete_ymark.xml +++ /dev/null @@ -1,2 +0,0 @@ -<?xml version='1.0' encoding="UTF-8" standalone='yes'?> -<result code="#(result)#something went wrong::done#(/result)#" />
\ No newline at end of file diff --git a/htroot/api/ymarks/get_metadata.java b/htroot/api/ymarks/get_metadata.java deleted file mode 100644 index 6bff178bc..000000000 --- a/htroot/api/ymarks/get_metadata.java +++ /dev/null @@ -1,108 +0,0 @@ -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.EnumMap; -import java.util.Iterator; -import java.util.Locale; - -import net.yacy.cora.document.id.DigestURL; -import net.yacy.cora.protocol.ClientIdentification; -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.data.UserDB; -import net.yacy.data.ymark.YMarkAutoTagger; -import net.yacy.data.ymark.YMarkCrawlStart; -import net.yacy.data.ymark.YMarkEntry; -import net.yacy.data.ymark.YMarkMetadata; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.data.ymark.YMarkUtil; -import net.yacy.document.Document; -import net.yacy.document.Parser.Failure; -import net.yacy.search.Switchboard; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - -public class get_metadata { - - static serverObjects prop; - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - final Switchboard sb = (Switchboard) env; - prop = new serverObjects(); - final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header)); - final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - - if(isAdmin || isAuthUser) { - - final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - - String url = post.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt()); - boolean hasProtocol = false; - for (final YMarkTables.PROTOCOLS p : YMarkTables.PROTOCOLS.values()) { - if(url.toLowerCase(Locale.ROOT).startsWith(p.protocol())) { - hasProtocol = true; - break; - } - } - if (!hasProtocol) { - url=YMarkTables.PROTOCOLS.HTTP.protocol(url); - } - - try { - final YMarkMetadata meta = new YMarkMetadata(new DigestURL(url), sb.index); - ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName)); - final Document document = meta.loadDocument(sb.loader, agent); - final EnumMap<YMarkMetadata.METADATA, String> metadata = meta.loadMetadata(); - - prop.putXML("title", metadata.get(YMarkMetadata.METADATA.TITLE)); - prop.putXML("desc", metadata.get(YMarkMetadata.METADATA.DESCRIPTION)); - prop.put("keywords", putTags(document.dc_subject(','), "keywords")); - prop.put("autotags", putTags(YMarkAutoTagger.autoTag(document, 5, sb.tables.bookmarks.getTags(bmk_user)), "autotags")); - - final YMarkCrawlStart crawlStart = new YMarkCrawlStart(sb.tables, url); - int count = 0; - if(!crawlStart.isEmpty()) { - final Iterator<String> iter = crawlStart.keySet().iterator(); - String key; - while(iter.hasNext()) { - key = iter.next(); - prop.putXML("crawlstart_"+count+"_key",key.toLowerCase()); - prop.putXML("crawlstart_"+count+"_value",crawlStart.get(key)); - count++; - } - } - prop.put("crawlstart", count); - - } catch (final MalformedURLException e) { - // TODO Auto-generated catch block - ConcurrentLog.logException(e); - prop.put("status", "error"); - } catch (final IOException e) { - // TODO Auto-generated catch block - ConcurrentLog.logException(e); - prop.put("status", "error"); - } catch (final Failure e) { - // TODO Auto-generated catch block - ConcurrentLog.logException(e); - prop.put("status", "error"); - } - } else { - prop.put(serverObjects.ACTION_AUTHENTICATE, YMarkTables.USER_AUTHENTICATE_MSG); - } - // return rewrite properties - return prop; - } - - public static int putTags(final String tagString, final String var) { - final String list[] = YMarkUtil.TAGS_SEPARATOR_PATTERN.split(tagString, 0); - int count = 0; - for (final String element : list) { - final String tag = element; - if (!tag.equals("")) { - prop.putXML(var+"_"+count+"_tag", tag); - count++; - } - } - return count; - } -}
\ No newline at end of file diff --git a/htroot/api/ymarks/get_metadata.xml b/htroot/api/ymarks/get_metadata.xml deleted file mode 100644 index dc72df1ef..000000000 --- a/htroot/api/ymarks/get_metadata.xml +++ /dev/null @@ -1,19 +0,0 @@ -<?xml version='1.0' encoding="UTF-8" standalone='yes'?> -<info status="#[status]#"> - <title>#[title]#</title> - <desc>#[desc]#</desc> - <keywords> - #{keywords}#<tag name="#[tag]#" /> - #{/keywords}# - </keywords> - <autotags> - #{autotags}#<tag name="#[tag]#" /> - #{/autotags}# - </autotags> - <crawlstart - #{crawlstart}##[key]#="#[value]#" - #{/crawlstart}# - > - </crawlstart> -</info> - diff --git a/htroot/api/ymarks/get_tags.java b/htroot/api/ymarks/get_tags.java deleted file mode 100644 index 6d4676c9b..000000000 --- a/htroot/api/ymarks/get_tags.java +++ /dev/null @@ -1,98 +0,0 @@ - -import java.io.IOException; -import java.text.CollationKey; -import java.text.Collator; -import java.util.Iterator; -import java.util.TreeMap; -import java.util.TreeSet; - -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.data.UserDB; -import net.yacy.data.ymark.YMarkEntry; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.data.ymark.YMarkTag; -import net.yacy.data.ymark.YMarkUtil; -import net.yacy.search.Switchboard; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - -public class get_tags { - - final static String TAG = "tag"; - final static String TOP = "top"; - final static String SORT = "sort"; - final static String SIZE = "size"; - final static String ALPHA = "alpha"; - - - private static Switchboard sb = null; - private static serverObjects prop = null; - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - sb = (Switchboard) env; - prop = new serverObjects(); - - final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header)); - final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - - if(isAdmin || isAuthUser) { - final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - Integer top = Integer.MAX_VALUE; - Boolean sortAlpha = true; - Iterator<YMarkTag> tit = null; - TreeSet<YMarkTag> tags = null; - int count = 0; - YMarkTag t; - - if (post != null && post.containsKey(TAG) && !post.get(TAG).isEmpty()) { - final String tagsString = YMarkUtil.cleanTagsString(post.get(TAG)); - tags = new TreeSet<YMarkTag>(sb.tables.bookmarks.getTags(sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagsString)).values()); - } else { - try { - tags = new TreeSet<YMarkTag>(sb.tables.bookmarks.getTags(bmk_user).values()); - } catch (final IOException e) { - return prop; - } - } - - if (post != null && post.containsKey(TOP)) { - top = post.getInt(TOP, Integer.MAX_VALUE); - } - - if (post != null && post.containsKey(SORT)) { - if (SIZE.equals(post.get(SORT))) { - sortAlpha = false; - } - } - - if(sortAlpha) { - final TreeMap<CollationKey, YMarkTag> sort = new TreeMap<CollationKey, YMarkTag>(); - final Collator collator = Collator.getInstance(); - collator.setStrength(Collator.SECONDARY); - tit = tags.iterator(); - while(tit.hasNext() && count < top) { - t = tit.next(); - sort.put(collator.getCollationKey(t.name()), t); - count++; - } - tit = sort.values().iterator(); - } else { - tit = tags.iterator(); - } - - count = 0; - while (tit.hasNext() && count < top) { - t = tit.next(); - if(!t.name().equals(YMarkEntry.BOOKMARK.TAGS.deflt())) { - prop.putXML("tags_" + count + "_name", t.name()); - prop.put("tags_" + count + "_count", t.size()); - count++; - } - } - - prop.put("tags", count); - } - return prop; - } -} diff --git a/htroot/api/ymarks/get_tags.xml b/htroot/api/ymarks/get_tags.xml deleted file mode 100644 index 6c58ea8fb..000000000 --- a/htroot/api/ymarks/get_tags.xml +++ /dev/null @@ -1,5 +0,0 @@ -<?xml version='1.0' encoding="UTF-8" standalone='yes'?> -<tags> -#{tags}#<tag count="#[count]#" tag="#[name]#"/> -#{/tags}# -</tags>
\ No newline at end of file diff --git a/htroot/api/ymarks/get_treeview.java b/htroot/api/ymarks/get_treeview.java deleted file mode 100644 index 1d988ade7..000000000 --- a/htroot/api/ymarks/get_treeview.java +++ /dev/null @@ -1,281 +0,0 @@ -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.Date; -import java.util.EnumMap; -import java.util.Iterator; -import java.util.TreeMap; - -import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.encoding.UTF8; -import net.yacy.cora.document.id.DigestURL; -import net.yacy.cora.protocol.ClientIdentification; -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.cora.util.SpaceExceededException; -import net.yacy.data.UserDB; -import net.yacy.data.ymark.YMarkAutoTagger; -import net.yacy.data.ymark.YMarkCrawlStart; -import net.yacy.data.ymark.YMarkEntry; -import net.yacy.data.ymark.YMarkMetadata; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.data.ymark.YMarkTag; -import net.yacy.data.ymark.YMarkUtil; -import net.yacy.document.Document; -import net.yacy.document.Parser.Failure; -import net.yacy.kelondro.blob.Tables; -import net.yacy.search.Switchboard; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - -public class get_treeview { - - public static final String ROOT = "root"; - public static final String SOURCE = "source"; - - static serverObjects prop; - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - final Switchboard sb = (Switchboard) env; - prop = new serverObjects(); - final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header)); - final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - - - if(isAdmin || isAuthUser) { - final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - - String root = YMarkTables.FOLDERS_ROOT; - String[] foldername = null; - boolean isFolder = true; - boolean isBookmark = false; - boolean isMetadata = false; - boolean isURLdb = false; - boolean isCrawlStart = false; - boolean isAutoTagger = false; - boolean displayBmk = false; - - if (post != null){ - if(post.containsKey("display") && post.get("display").equals("bmk")) { - displayBmk = true; - } - - if (post.containsKey(ROOT)) { - if (post.get(ROOT).equals(SOURCE) || post.get(ROOT).equals(YMarkTables.FOLDERS_ROOT)) { - root = ""; - } else if (post.get(ROOT).startsWith(YMarkTables.FOLDERS_ROOT)) { - root = post.get(ROOT); - } else if (post.get(ROOT).startsWith("b:")) { - isBookmark = true; - isFolder = false; - } else if (post.get(ROOT).startsWith("m:")) { - isMetadata = true; - isFolder = false; - } else if (post.get(ROOT).startsWith("u:")) { - isURLdb = true; - isFolder = false; - } else if (post.get(ROOT).startsWith("w:")) { - isAutoTagger = true; - isFolder = false; - } else if (post.get(ROOT).startsWith("c:")) { - isCrawlStart = true; - isFolder = false; - } - } - } - - Iterator<String> it = null; - Iterator<Tables.Row> bit = null; - Tables.Row bmk_row = null; - int count = 0; - - if(isFolder) { - // loop through folderList - try { - // it = sb.tables.bookmarks.folders.getFolders(bmk_user, root); - it = sb.tables.bookmarks.getFolders(bmk_user, root).iterator(); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - int n = YMarkUtil.FOLDERS_SEPARATOR_PATTERN.split(root, 0).length; - if (n == 0) n = 1; - while (it.hasNext()) { - final String folder = it.next(); - foldername = YMarkUtil.FOLDERS_SEPARATOR_PATTERN.split(folder); - if (foldername.length == n+1) { - prop.put("folders_"+count+"_foldername", foldername[n]); - prop.put("folders_"+count+"_expanded", "false"); - if(foldername[n].equals("IOExceptions")) - prop.put("folders_"+count+"_type", "err"); - else if(foldername[n].equals("unsorted")) - prop.put("folders_"+count+"_type", "question"); - else if(foldername[n].equals("Crawl Start")) - prop.put("folders_"+count+"_type", "crawl"); - else - prop.put("folders_"+count+"_type", "folder"); - prop.put("folders_"+count+"_hash", folder); //TODO: switch from pathString to folderHash - prop.put("folders_"+count+"_url", ""); //TODO: insert folder url - prop.put("folders_"+count+"_hasChildren", "true"); //TODO: determine if folder has children - prop.put("folders_"+count+"_comma", ","); - count++; - } - } - if(displayBmk && !root.isEmpty()) { - bit = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, root); - while (bit.hasNext()) { - bmk_row = bit.next(); - if(bmk_row != null) { - final String url = UTF8.String(bmk_row.get(YMarkEntry.BOOKMARK.URL.key())); - final String title = bmk_row.get(YMarkEntry.BOOKMARK.TITLE.key(), YMarkEntry.BOOKMARK.TITLE.deflt()); - - // TODO: get_treeview - get rid of bmtype - if (post.containsKey("bmtype")) { - if (post.get("bmtype").equals("title")) { - prop.putJSON("folders_"+count+"_foldername", title); - } else if (post.get("bmtype").equals("href")) { - prop.putJSON("folders_"+count+"_foldername", "<a href='"+url+"' target='_blank'>"+title+"</a>"); - } - } else { - prop.putJSON("folders_"+count+"_foldername", url); - } - prop.put("folders_"+count+"_expanded", "false"); - prop.put("folders_"+count+"_url", url); - prop.put("folders_"+count+"_type", "file"); - prop.put("folders_"+count+"_hash", "b:"+new String(bmk_row.getPK())); - prop.put("folders_"+count+"_hasChildren", "true"); - prop.put("folders_"+count+"_comma", ","); - count++; - } - } - } - count--; - prop.put("folders_"+count+"_comma", ""); - count++; - prop.put("folders", count); - } else if(displayBmk && isBookmark) { - try { - final String urlHash = post.get(ROOT).substring(2); - String url = ""; - bmk_row = sb.tables.select(YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user), urlHash.getBytes()); - if(bmk_row != null) { - it = bmk_row.keySet().iterator(); - while(it.hasNext()) { - final String key = it.next(); - if(key.startsWith("date")) { - final String d = UTF8.String(bmk_row.get(key)); - if(!d.isEmpty()) { - final String date = ISO8601Formatter.FORMATTER.format(new Date(Long.parseLong(d))); - prop.put("folders_"+count+"_foldername","<small><b>"+key+":</b> " + date + "</small>"); - putProp(count, "date"); - count++; - } - } else { - final String value = UTF8.String(bmk_row.get(key)); - if (key.equals("url")) - url = value; - prop.put("folders_"+count+"_foldername","<small><b>"+key+":</b> " + value + "</small>"); - if(YMarkEntry.BOOKMARK.contains(key)) - putProp(count, YMarkEntry.BOOKMARK.get(key).type()); - else - putProp(count, "meta"); - count++; - } - } - prop.put("folders_"+count+"_foldername","<small><b>MetaData</b></small>"); - putProp(count, "meta"); - prop.put("folders_"+count+"_hash", "m:"+url); - prop.put("folders_"+count+"_hasChildren", "true"); - count++; - prop.put("folders_"+count+"_foldername","<small><b>URLdb</b></small>"); - putProp(count, "meta"); - prop.put("folders_"+count+"_hash", "u:"+url); - prop.put("folders_"+count+"_hasChildren", "true"); - count++; - prop.put("folders_"+count+"_foldername","<small><b>CrawlStart</b></small>"); - putProp(count, "meta"); - prop.put("folders_"+count+"_hash", "c:"+url); - prop.put("folders_"+count+"_hasChildren", "true"); - count++; - prop.put("folders_"+count+"_foldername","<small><b>AutoTagger</b></small>"); - putProp(count, "meta"); - prop.put("folders_"+count+"_hash", "w:"+url); - prop.put("folders_"+count+"_hasChildren", "true"); - prop.put("folders_"+count+"_comma", ""); - count++; - prop.put("folders", count); - } - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final SpaceExceededException e) { - ConcurrentLog.logException(e); - } - } else if (isAutoTagger || isMetadata || isURLdb || isCrawlStart) { - try { - final YMarkMetadata meta = new YMarkMetadata(new DigestURL(post.get(ROOT).substring(2)), sb.index); - ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName)); - final Document document = meta.loadDocument(sb.loader, agent); - final TreeMap<String, YMarkTag> tags = sb.tables.bookmarks.getTags(bmk_user); - if(isAutoTagger) { - prop.put("folders_"+count+"_foldername","<small><b>meta-"+YMarkMetadata.METADATA.KEYWORDS.name().toLowerCase()+":</b> " + meta.loadMetadata().get(YMarkMetadata.METADATA.KEYWORDS) + "</small>"); - putProp(count, "meta"); - count++; - prop.put("folders_"+count+"_foldername","<small><b>with preference: </b>"+YMarkAutoTagger.autoTag(document, 4, tags)+"</small>"); - putProp(count, "meta"); - count++; - prop.put("folders_"+count+"_foldername","<small><b>without preference: </b>"+YMarkAutoTagger.autoTag(document, 4, new TreeMap<String, YMarkTag>())+"</small>"); - putProp(count, "meta"); - count++; - prop.put("folders", count); - } else if(isMetadata) { - count = putMeta(count, meta.loadMetadata()); - } else if(isURLdb) { - count = putMeta(count, meta.getMetadata()); - } else if(isCrawlStart) { - ConcurrentLog.info("YMark", "I am looking for CrawlStart: "+post.get(ROOT).substring(2)); - final YMarkCrawlStart crawlStart = new YMarkCrawlStart(sb.tables, post.get(ROOT).substring(2)); - final Iterator<String> iter = crawlStart.keySet().iterator(); - String key; - while(iter.hasNext()) { - key = iter.next(); - prop.put("folders_"+count+"_foldername","<small><b>"+key.toLowerCase()+":</b> " + crawlStart.get(key) + "</small>"); - putProp(count, "meta"); - count++; - } - prop.put("folders", count); - } - - } catch (final MalformedURLException e) { - ConcurrentLog.logException(e); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final Failure e) { - ConcurrentLog.logException(e); - } - } - } else { - prop.put(serverObjects.ACTION_AUTHENTICATE, YMarkTables.USER_AUTHENTICATE_MSG); - } - // return rewrite properties - return prop; - } - public static void putProp(final int count, final String type) { - prop.put("folders_"+count+"_expanded", "false"); - prop.put("folders_"+count+"_url", ""); - prop.put("folders_"+count+"_type", type); - prop.put("folders_"+count+"_hash", ""); - prop.put("folders_"+count+"_hasChildren", "false"); - prop.put("folders_"+count+"_comma", ","); - } - public static int putMeta(int count, final EnumMap<YMarkMetadata.METADATA, String> metadata) { - final Iterator<YMarkMetadata.METADATA> iter = metadata.keySet().iterator(); - while (iter.hasNext()) { - final YMarkMetadata.METADATA key = iter.next(); - final String value = metadata.get(key); - prop.put("folders_"+count+"_foldername","<small><b>"+key.toString().toLowerCase()+":</b> " + value + "</small>"); - putProp(count, "meta"); - count++; - } - prop.put("folders", count); - return count; - } -} diff --git a/htroot/api/ymarks/get_treeview.json b/htroot/api/ymarks/get_treeview.json deleted file mode 100644 index 7cf91fa05..000000000 --- a/htroot/api/ymarks/get_treeview.json +++ /dev/null @@ -1,9 +0,0 @@ -[ -#{folders}#{ -"text": "#[foldername]#", -"expanded": #[expanded]#, -"classes": "#[type]#", -"id": "#[hash]#", -"hasChildren": #[hasChildren]# -}#[comma]##{/folders}# -]
\ No newline at end of file diff --git a/htroot/api/ymarks/get_treeview.xml b/htroot/api/ymarks/get_treeview.xml deleted file mode 100644 index 3527108b7..000000000 --- a/htroot/api/ymarks/get_treeview.xml +++ /dev/null @@ -1,11 +0,0 @@ -<?xml version='1.0' encoding="UTF-8" standalone='yes'?> -<folders> -#{folders}#<folder - id="#[hash]#" - type="#[type]#" - name="#[foldername]#" - url="#[url]#" - hasChildren="#[hasChildren]#" -/> -#{/folders}# -</folders>
\ No newline at end of file diff --git a/htroot/api/ymarks/get_xbel.java b/htroot/api/ymarks/get_xbel.java deleted file mode 100644 index cb4c6c619..000000000 --- a/htroot/api/ymarks/get_xbel.java +++ /dev/null @@ -1,180 +0,0 @@ -import java.io.IOException; -import java.util.HashSet; -import java.util.Iterator; - -import net.yacy.cora.document.encoding.UTF8; -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.data.UserDB; -import net.yacy.data.ymark.YMarkDate; -import net.yacy.data.ymark.YMarkEntry; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.data.ymark.YMarkUtil; -import net.yacy.data.ymark.YMarkXBELImporter; -import net.yacy.document.parser.html.CharacterCoding; -import net.yacy.kelondro.blob.Tables; -import net.yacy.search.Switchboard; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - -public class get_xbel { - public static final String ROOT = "root"; - public static final String SOURCE = "source"; - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - final Switchboard sb = (Switchboard) env; - final serverObjects prop = new serverObjects(); - final HashSet<String> alias = new HashSet<String>(); - final StringBuilder buffer = new StringBuilder(250); - final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header)); - final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - final String bmk_user; - - if(isAdmin || isAuthUser) { - bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - - String root = YMarkTables.FOLDERS_ROOT; - String[] foldername = null; - - // TODO: better handling of query - if (post != null){ - if (post.containsKey(ROOT)) { - if (post.get(ROOT).equals(SOURCE) || post.get(ROOT).equals(YMarkTables.FOLDERS_ROOT)) { - root = ""; - } else if (post.get(ROOT).startsWith(YMarkTables.FOLDERS_ROOT)) { - root = post.get(ROOT); - } else { - root = ""; - } - } - } else { - root = ""; - } - - final int root_depth = YMarkUtil.FOLDERS_SEPARATOR_PATTERN.split(root).length - 1; - // Log.logInfo(YMarkTables.BOOKMARKS_LOG, "root: "+root+" root_depth: "+root_depth); - Iterator<String> fit = null; - Iterator<Tables.Row> bit = null; - int count = 0; - int n = root_depth; - - try { - fit = sb.tables.bookmarks.getFolders(bmk_user, root).iterator(); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - - while (fit.hasNext()) { - final String folder = fit.next(); - foldername = YMarkUtil.FOLDERS_SEPARATOR_PATTERN.split(folder); - final int len = foldername.length -1; - if(n > root_depth) { - for (; len <= n; n--) { - // Log.logInfo(YMarkTables.BOOKMARKS_LOG, "</folder> n: "+n); - prop.put("xbel_"+count+"_elements", "</folder>"); - count++; - } - } - if (len >= n) { - n = len; - if(n > root_depth) { - // Log.logInfo(YMarkTables.BOOKMARKS_LOG, "<folder>: "+folder+" n: "+n); - prop.put("xbel_"+count+"_elements", "<folder id=\"f:"+UTF8.String(YMarkUtil.getKeyId(foldername[n]))+"\">"); - count++; - prop.put("xbel_"+count+"_elements", "<title>" + CharacterCoding.unicode2xml(foldername[n], true) + "</title>"); - count++; - } - bit = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, folder); - Tables.Row bmk_row = null; - String urlHash; - final YMarkDate date = new YMarkDate(); - while(bit.hasNext()){ - bmk_row = bit.next(); - urlHash = new String(bmk_row.getPK()); - - if(alias.contains(urlHash)) { - buffer.setLength(0); - buffer.append(YMarkXBELImporter.XBEL.ALIAS.startTag(true)); - buffer.append(" ref=\"b:"); - buffer.append(urlHash); - buffer.append("\"/>"); - prop.put("xbel_"+count+"_elements", buffer.toString()); - count++; - } else { - alias.add(urlHash); - if(bmk_row != null) { - buffer.setLength(0); - - buffer.append(YMarkXBELImporter.XBEL.BOOKMARK.startTag(true)); - buffer.append(" id=\"b:"); - buffer.append(urlHash); - - buffer.append(YMarkEntry.BOOKMARK.URL.xbel()); - buffer.append(CharacterCoding.unicode2xml(bmk_row.get(YMarkEntry.BOOKMARK.URL.key(), YMarkEntry.BOOKMARK.URL.deflt()), true)); - - buffer.append(YMarkEntry.BOOKMARK.DATE_ADDED.xbel()); - date.set(bmk_row.get(YMarkEntry.BOOKMARK.DATE_ADDED.key())); - buffer.append(CharacterCoding.unicode2xml(date.toISO8601(), true)); - - buffer.append(YMarkEntry.BOOKMARK.DATE_MODIFIED.xbel()); - date.set(bmk_row.get(YMarkEntry.BOOKMARK.DATE_MODIFIED.key())); - buffer.append(CharacterCoding.unicode2xml(date.toISO8601(), true)); - - buffer.append(YMarkEntry.BOOKMARK.DATE_VISITED.xbel()); - date.set(bmk_row.get(YMarkEntry.BOOKMARK.DATE_VISITED.key())); - buffer.append(CharacterCoding.unicode2xml(date.toISO8601(), true)); - - buffer.append(YMarkEntry.BOOKMARK.TAGS.xbel()); - buffer.append(CharacterCoding.unicode2xml(bmk_row.get(YMarkEntry.BOOKMARK.TAGS.key(), YMarkEntry.BOOKMARK.TAGS.deflt()),true)); - - buffer.append(YMarkEntry.BOOKMARK.PUBLIC.xbel()); - buffer.append(bmk_row.get(YMarkEntry.BOOKMARK.PUBLIC.key(), YMarkEntry.BOOKMARK.PUBLIC.deflt())); - - buffer.append(YMarkEntry.BOOKMARK.VISITS.xbel()); - buffer.append(bmk_row.get(YMarkEntry.BOOKMARK.VISITS.key(), YMarkEntry.BOOKMARK.VISITS.deflt())); - - buffer.append("\"\n>"); - prop.put("xbel_"+count+"_elements", buffer.toString()); - count++; - - buffer.setLength(0); - buffer.append(YMarkXBELImporter.XBEL.TITLE.startTag(false)); - buffer.append(CharacterCoding.unicode2xml(bmk_row.get(YMarkEntry.BOOKMARK.TITLE.key(), YMarkEntry.BOOKMARK.TITLE.deflt()), true)); - buffer.append(YMarkXBELImporter.XBEL.TITLE.endTag(false)); - prop.put("xbel_"+count+"_elements", buffer.toString()); - count++; - - buffer.setLength(0); - buffer.append(YMarkXBELImporter.XBEL.DESC.startTag(false)); - buffer.append(CharacterCoding.unicode2xml(bmk_row.get(YMarkEntry.BOOKMARK.DESC.key(), YMarkEntry.BOOKMARK.DESC.deflt()), true)); - buffer.append(YMarkXBELImporter.XBEL.DESC.endTag(false)); - prop.put("xbel_"+count+"_elements", buffer.toString()); - count++; - - prop.put("xbel_"+count+"_elements", YMarkXBELImporter.XBEL.BOOKMARK.endTag(false)); - count++; - } - } - } - } - } - while(n > root_depth) { - // Log.logInfo(YMarkTables.BOOKMARKS_LOG, "</folder> n: "+n); - prop.put("xbel_"+count+"_elements", YMarkXBELImporter.XBEL.FOLDER.endTag(false)); - count++; - n--; - } - prop.put("root", root); - prop.put("user", bmk_user.substring(0,1).toUpperCase() + bmk_user.substring(1)); - prop.put("xbel", count); - - } else { - prop.put(serverObjects.ACTION_AUTHENTICATE, YMarkTables.USER_AUTHENTICATE_MSG); - } - // return rewrite properties - return prop; - } -} - - diff --git a/htroot/api/ymarks/get_xbel.xml b/htroot/api/ymarks/get_xbel.xml deleted file mode 100644 index 9dc946133..000000000 --- a/htroot/api/ymarks/get_xbel.xml +++ /dev/null @@ -1,14 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE xbel PUBLIC "+//IDN python.org//DTD XML Bookmark Exchange Language 1.0//EN//XML" "http://pyxml.sourceforge.net/topics/dtds/xbel-1.0.dtd" -[ - <!ENTITY % local.url.att "yacy:public CDATA #IMPLIED yacy:tags CDATA #IMPLIED yacy:visits CDATA #IMPLIED"> -]> -<xbel version="1.0" xmlns:yacy="http://www.yacy.net"> -<title>#[user]# YaCy Bookmarks: #[root]#</title> -<info> -<metadata owner="http://www.yacy.net" /> -</info> -#{xbel}# -#[elements]# -#{/xbel}# -</xbel>
\ No newline at end of file diff --git a/htroot/api/ymarks/get_ymark.java b/htroot/api/ymarks/get_ymark.java deleted file mode 100644 index e2d2956cb..000000000 --- a/htroot/api/ymarks/get_ymark.java +++ /dev/null @@ -1,168 +0,0 @@ -import java.io.IOException; -import java.util.Collection; -import java.util.Iterator; -import java.util.regex.Pattern; - -import net.yacy.cora.document.encoding.UTF8; -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.data.UserDB; -import net.yacy.data.ymark.YMarkCrawlStart; -import net.yacy.data.ymark.YMarkDate; -import net.yacy.data.ymark.YMarkEntry; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.data.ymark.YMarkUtil; -import net.yacy.data.ymark.YMarkTables.TABLES; -import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.blob.Tables.Row; -import net.yacy.search.Switchboard; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - - -public class get_ymark { - - private static Switchboard sb = null; - private static serverObjects prop = null; - final static String FOLDER_IMG = "<img src=\"/yacy/ui/img/treeview/folder-closed.gif\" />"; - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - sb = (Switchboard) env; - prop = new serverObjects(); - - int rp; // items per page - int page; // page - int total; - String sortorder; - String sortname; - String qtype; - String query; - - final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header)); - final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - Iterator<Tables.Row> bookmarks = null; - - if(isAdmin || isAuthUser) { - final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - - query = ".*"; - qtype = YMarkEntry.BOOKMARK.TITLE.key(); - page = 1; - rp = 100; - total = 0; - sortname = YMarkEntry.BOOKMARK.TITLE.key(); - sortorder = "asc"; - - if(post != null) { - rp = (post.containsKey("rp")) ? post.getInt("rp", 10) : 10; - page = (post.containsKey("page")) ? post.getInt("page", 1): 1; - query = (post.containsKey("query")) ? post.get("query", ".*") : ".*"; - qtype = (post.containsKey("qtype")) ? post.get("qtype", YMarkEntry.BOOKMARK.TAGS.key()) : YMarkEntry.BOOKMARK.TAGS.key(); - sortname = (post.containsKey("sortname")) ? post.get("sortname", YMarkEntry.BOOKMARK.TITLE.key()) : YMarkEntry.BOOKMARK.TITLE.key(); - sortorder = (post.containsKey("sortorder")) ? post.get("sortorder", "asc") : "asc"; - } - try { - final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); - final Collection<Row> result; - if(!query.isEmpty()) { - if(!qtype.isEmpty()) { - if(qtype.equals("_tags")) { - final String tags = YMarkUtil.cleanTagsString(query); - result = sb.tables.bookmarks.orderBookmarksBy(sb.tables.bookmarks.getBookmarksByTag(bmk_user, tags), sortname, sortorder); - } else if(qtype.equals("_folder")) { - result = sb.tables.bookmarks.orderBookmarksBy(sb.tables.bookmarks.getBookmarksByFolder(bmk_user, query), sortname, sortorder); - } else { - result = sb.tables.bookmarks.orderBookmarksBy(sb.tables.iterator(bmk_table, qtype, Pattern.compile(query)), sortname, sortorder); - } - } else { - result = sb.tables.bookmarks.orderBookmarksBy(sb.tables.iterator(bmk_table, Pattern.compile(query)), sortname, sortorder); - } - } else { - result = sb.tables.bookmarks.orderBookmarksBy(sb.tables.iterator(bmk_table), sortname, sortorder); - } - total = result.size(); - bookmarks = result.iterator(); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - prop.put("page", page); - prop.put("total", total); - putProp(bookmarks, rp, page); - } else { - prop.put(serverObjects.ACTION_AUTHENTICATE, YMarkTables.USER_AUTHENTICATE_MSG); - } - // return rewrite properties - return prop; - } - - private static void putProp(final Iterator<Tables.Row> bit, final int rp, final int page) { - Tables.Row bmk_row; - int count = 0; - int offset = 0; - if (page > 1) { - offset = ((page - 1) * rp) + 1; - } - while(count < offset && bit.hasNext()) { - bmk_row = bit.next(); - count++; - } - count = 0; - while (count < rp && bit.hasNext()) { - bmk_row = bit.next(); - if (bmk_row != null) { - - // put JSON - prop.put("json_"+count+"_id", count); - prop.put("json_"+count+"_hash", UTF8.String(bmk_row.getPK())); - for (final YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) { - if(bmk == YMarkEntry.BOOKMARK.PUBLIC) - prop.put("json_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt()).equals("false") ? 1 : 0); - else if(bmk == YMarkEntry.BOOKMARK.TAGS) - prop.putJSON("json_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt()).replaceAll(YMarkUtil.TAGS_SEPARATOR, ", ")); - else if(bmk == YMarkEntry.BOOKMARK.FOLDERS) - prop.putJSON("json_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt()).replaceAll(YMarkUtil.TAGS_SEPARATOR, "<br />"+FOLDER_IMG)); - else if(bmk == YMarkEntry.BOOKMARK.DATE_ADDED || bmk == YMarkEntry.BOOKMARK.DATE_MODIFIED || bmk == YMarkEntry.BOOKMARK.DATE_VISITED) - prop.putJSON("json_"+count+"_"+bmk.key(), (new YMarkDate(bmk_row.get(bmk.key()))).toISO8601().replaceAll("T", "<br />")); - else - prop.putJSON("json_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt())); - } - - final YMarkCrawlStart crawlstart = new YMarkCrawlStart(sb.tables, bmk_row.get(YMarkEntry.BOOKMARK.URL.key(),YMarkEntry.BOOKMARK.URL.deflt())); - int crawl = 0; - if (!crawlstart.isEmpty()) { - crawl = 1; - prop.put("json_"+count+"_crawlstart_info", "Crawl last executed: "+YMarkDate.ISO8601(crawlstart.date_last_exec())); - } - if (crawlstart.hasSchedule()) { - crawl = 2; - prop.put("json_"+count+"_crawlstart_info", "Crawl scheduled: "+YMarkDate.ISO8601(crawlstart.date_next_exec())); - } - if (crawlstart.isRunning(sb.crawler)) { - crawl = 3; - prop.put("json_"+count+"_crawlstart_info", "Crawl is running ..."); - } - prop.put("json_"+count+"_crawlstart", crawl); - - prop.put("json_"+count+"_apicall_pk", crawlstart.getPK()); - prop.put("json_"+count+"_date_recording", YMarkDate.ISO8601(crawlstart.date_recording()).replaceAll("T", "<br />")); - prop.put("json_"+count+"_date_next_exec", YMarkDate.ISO8601(crawlstart.date_next_exec()).replaceAll("T", "<br />")); - prop.put("json_"+count+"_date_last_exec", YMarkDate.ISO8601(crawlstart.date_last_exec()).replaceAll("T", "<br />")); - - prop.put("json_"+count+"_comma", ","); - - // put XML - prop.putXML("xml_"+count+"_id", UTF8.String(bmk_row.getPK())); - for (final YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) { - prop.putXML("xml_"+count+"_"+bmk.key(), bmk_row.get(bmk.key(),bmk.deflt())); - } - - count++; - } - } - // eliminate the trailing comma for Json output - prop.put("json_" + (count - 1) + "_comma", ""); - prop.put("json", count); - prop.put("xml", count); - } -} diff --git a/htroot/api/ymarks/get_ymark.json b/htroot/api/ymarks/get_ymark.json deleted file mode 100644 index 65b013b18..000000000 --- a/htroot/api/ymarks/get_ymark.json +++ /dev/null @@ -1,34 +0,0 @@ -{ -"page" : #[page]#, -"total": #[total]#, -"rows" : [ -#{json}# -{"id":"#[id]#","cell":[ -"#[hash]#", -#(public)# -"<img src='/yacy/ui/img-2/world2.png' alt='true' title='public bookmark'/>" -:: -"<img src='/yacy/ui/img-2/lock.png' alt='false' title='private bookmark'/>"#(/public)#, -#(crawlstart)# -"" -:: -"<img src='/yacy/ui/img-2/accept.png' alt='#[info]#' title='#[info]#'/>" -:: -"<img src='/yacy/ui/img-2/calendar.png' alt='#[info]#' title='#[info]#'/>" -:: -"<img src='/yacy/ui/img/crawl.png' alt='#[info]#' title='#[info]#'/>" -#(/crawlstart)#, -"<h3 class='linktitle'>#[title]#</h3><p class='desc'>#[desc]#</p><p class='url'><a href='#[url]#'>#[url]#</a>", -"<p class='tags'>#[tags]#</p>", -"<p class='folders'><img src='/yacy/ui/img/treeview/folder-closed.gif' />#[folders]#</p>", -"<p class='date'>#[date_added]#</p>", -"<p class='date'>#[date_modified]#</p>", -"<p class='date'>#[date_visited]#</p>", -"<p class='apicall_pk'>#[apicall_pk]#</p>", -"<p class='date'>#[date_recording]#</p>", -"<p class='date'>#[date_next_exec]#</p>", -"<p class='date'>#[date_last_exec]#</p>" -]}#[comma]# -#{/json}# -] -}
\ No newline at end of file diff --git a/htroot/api/ymarks/get_ymark.xml b/htroot/api/ymarks/get_ymark.xml deleted file mode 100644 index a28256206..000000000 --- a/htroot/api/ymarks/get_ymark.xml +++ /dev/null @@ -1,6 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<posts> -#{xml}# - <post href="#[url]#" id="#[id]#" title="#[title]#" tag="#[tags]#" folders="#[folders]#" added="#[date_added]#"/> -#{/xml}# -</posts>
\ No newline at end of file diff --git a/htroot/api/ymarks/import_ymark.html b/htroot/api/ymarks/import_ymark.html deleted file mode 100644 index ec376281a..000000000 --- a/htroot/api/ymarks/import_ymark.html +++ /dev/null @@ -1,14 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml"> - <head> - <title>Import Bookmarks</title> - #(redirect)#::<meta HTTP-EQUIV="REFRESH" content="0; url=#[url]#" />#(/redirect)# - </head> - <body> - #(status)# - <p>Status: error</p> - :: - <p>Status: ok</p> - #(/status)# - </body> -</html>
\ No newline at end of file diff --git a/htroot/api/ymarks/import_ymark.java b/htroot/api/ymarks/import_ymark.java deleted file mode 100644 index 5344539f9..000000000 --- a/htroot/api/ymarks/import_ymark.java +++ /dev/null @@ -1,254 +0,0 @@ -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.net.MalformedURLException; -import java.nio.charset.StandardCharsets; -import java.util.Iterator; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.regex.Pattern; -import java.util.zip.GZIPInputStream; - -import org.xml.sax.SAXException; - -import net.yacy.cora.document.encoding.UTF8; -import net.yacy.cora.protocol.ClientIdentification; -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.data.BookmarksDB.Bookmark; -import net.yacy.data.UserDB; -import net.yacy.data.WorkTables; -import net.yacy.data.ymark.MonitoredReader; -import net.yacy.data.ymark.YMarkAutoTagger; -import net.yacy.data.ymark.YMarkCrawlStart; -import net.yacy.data.ymark.YMarkDMOZImporter; -import net.yacy.data.ymark.YMarkEntry; -import net.yacy.data.ymark.YMarkHTMLImporter; -import net.yacy.data.ymark.YMarkJSONImporter; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.data.ymark.YMarkUtil; -import net.yacy.data.ymark.YMarkXBELImporter; -import net.yacy.document.Parser.Failure; -import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.workflow.OneTimeBusyThread; -import net.yacy.search.Switchboard; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - -public class import_ymark { - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - final Switchboard sb = (Switchboard) env; - final serverObjects prop = new serverObjects(); - final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header)); - final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - final int queueSize = 200; - - YMarkEntry bmk; - // String root = YMarkEntry.FOLDERS_IMPORTED; - String root = ""; - ByteArrayInputStream stream = null; - - if(isAdmin || isAuthUser) { - String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - final ArrayBlockingQueue<String> autoTaggingQueue = new ArrayBlockingQueue<String>(10*queueSize); - boolean autotag = false; - boolean merge = false; - boolean empty = false; - final String indexing = post.get("indexing", "off"); - final boolean medialink = post.getBoolean("medialink"); - - if(post.containsKey("autotag") && !post.get("autotag", "off").equals("off")) { - autotag = true; - if(post.get("autotag").equals("merge")) { - merge = true; - } - if(post.get("autotag").equals("empty")) { - empty = true; - } - YMarkAutoTagger autoTagger = new YMarkAutoTagger(autoTaggingQueue, sb.loader, sb.tables.bookmarks, bmk_user, merge); - OneTimeBusyThread.startFromRunnable(autoTagger, 0); - } - - if(isAdmin && post.containsKey("table") && post.get("table").length() > 0) { - bmk_user = post.get("table").substring(0, post.get("table").indexOf('_',0)); - } - if(post.containsKey("redirect") && post.get("redirect").length() > 0) { - prop.put("redirect_url", post.get("redirect")); - prop.put("redirect", "1"); - } - if(post.containsKey("root") && post.get("root").length() > 0) { - root = post.get("root"); - } - ClientIdentification.Agent agent = ClientIdentification.getAgent(post.get("agentName", ClientIdentification.yacyInternetCrawlerAgentName)); - if(post.containsKey("bmkfile") && !post.get("bmkfile").isEmpty() && post.containsKey("importer")){ - final byte[] bytes = UTF8.getBytes(post.get("bmkfile$file")); - stream = new ByteArrayInputStream(bytes); - if(post.get("importer").equals("surro") && stream != null) { - /** - SurrogateReader surrogateReader; - try { - surrogateReader = new SurrogateReader(stream, queueSize, sb.crawlStacker, sb.index.fulltext().getDefaultConfiguration()); - } catch (final IOException e) { - //TODO: display an error message - ConcurrentLog.logException(e); - prop.put("status", "0"); - return prop; - } - InstantBusyThread.oneTimeJob(surrogateReader, 0); - while ((bmk = new YMarkEntry(surrogateReader.take())) != YMarkEntry.POISON) { - putBookmark(sb, bmk_user, bmk, autoTaggingQueue, autotag, empty, indexing, medialink); - } - prop.put("status", "1"); - */ - } else { - MonitoredReader reader = new MonitoredReader(new InputStreamReader(stream, StandardCharsets.UTF_8), 1024*16, bytes.length); - if(post.get("importer").equals("html") && reader != null) { - final YMarkHTMLImporter htmlImporter = new YMarkHTMLImporter(reader, queueSize, root); - OneTimeBusyThread.startFromRunnable(htmlImporter, 0); - OneTimeBusyThread.startFromRunnable(htmlImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0); - prop.put("status", "1"); - } else if(post.get("importer").equals("xbel") && reader != null) { - final YMarkXBELImporter xbelImporter; - try { - //TODO: make RootFold - xbelImporter = new YMarkXBELImporter(reader, queueSize, root); - } catch (final SAXException e) { - //TODO: display an error message - ConcurrentLog.logException(e); - prop.put("status", "0"); - return prop; - } - OneTimeBusyThread.startFromRunnable(xbelImporter, 0); - OneTimeBusyThread.startFromRunnable(xbelImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0); - prop.put("status", "1"); - } else if(post.get("importer").equals("json") && reader != null) { - YMarkJSONImporter jsonImporter; - jsonImporter = new YMarkJSONImporter(reader, queueSize, root); - OneTimeBusyThread.startFromRunnable(jsonImporter, 0); - while ((bmk = jsonImporter.take()) != YMarkEntry.POISON) { - putBookmark(sb, bmk_user, bmk, autoTaggingQueue, autotag, empty, indexing, medialink); - } - prop.put("status", "1"); - } - } - } else if(post.containsKey("importer") && post.get("importer").equals("crawls")) { - if(!isAdmin) { - prop.authenticationRequired(); - return prop; - } - try { - final Pattern pattern = Pattern.compile("^crawl start for.*"); - final Iterator<Tables.Row> APIcalls = sb.tables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_COMMENT, pattern); - Tables.Row row = null; - while(APIcalls.hasNext()) { - row = APIcalls.next(); - if(row.get(WorkTables.TABLE_API_COL_TYPE, "").equals("crawler")) { - final String url = row.get(WorkTables.TABLE_API_COL_COMMENT, "").substring(16); - sb.tables.bookmarks.createBookmark(sb.loader, url, agent, bmk_user, autotag, "crawlStart", "/Crawl Start"); - } - } - prop.put("status", "1"); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final Failure e) { - ConcurrentLog.logException(e); - } - } else if(post.containsKey("importer") && post.get("importer").equals("bmks")) { - if(!isAdmin) { - prop.authenticationRequired(); - return prop; - } - final Iterator<String> bit=sb.bookmarksDB.getBookmarksIterator(isAdmin); - while(bit.hasNext()){ - Bookmark bookmark=sb.bookmarksDB.getBookmark(bit.next()); - if (bookmark != null) { - final YMarkEntry bmk_entry = new YMarkEntry(false); - bmk_entry.put(YMarkEntry.BOOKMARK.URL.key(), bookmark.getUrl()); - try { - if(!sb.tables.has(YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user), YMarkUtil.getBookmarkId(bookmark.getUrl()))) { - bmk_entry.put(YMarkEntry.BOOKMARK.PUBLIC.key(), bookmark.getPublic() ? "true" : "false"); - bmk_entry.put(YMarkEntry.BOOKMARK.TITLE.key(), bookmark.getTitle()); - bmk_entry.put(YMarkEntry.BOOKMARK.DESC.key(), bookmark.getDescription()); - bmk_entry.put(YMarkEntry.BOOKMARK.TAGS.key(), bookmark.getTagsString()); - bmk_entry.put(YMarkEntry.BOOKMARK.FOLDERS.key(), root+bookmark.getFoldersString().replaceAll(".*"+YMarkUtil.TAGS_SEPARATOR+YMarkUtil.FOLDERS_SEPARATOR, root+YMarkUtil.FOLDERS_SEPARATOR)); - } - if(autotag) { - bmk_entry.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkAutoTagger.autoTag(bookmark.getUrl(), sb.loader, agent, 3, sb.tables.bookmarks.getTags(bmk_user))); - } - sb.tables.bookmarks.addBookmark(bmk_user, bmk_entry, merge, true); - prop.put("status", "1"); - } catch (final MalformedURLException e) { - ConcurrentLog.logException(e); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - } - } - } else if(post.containsKey("importer") && post.get("importer").equals("dmoz")) { - if(!isAdmin) { - prop.authenticationRequired(); - return prop; - } - try { - final File in = new File(sb.workPath, "content.rdf.u8.gz"); - final InputStream gzip = new FileInputStream(in); - final InputStream content = new GZIPInputStream(gzip); - final InputStreamReader reader = new InputStreamReader(content, StandardCharsets.UTF_8); - final BufferedReader breader = new BufferedReader(reader); - final MonitoredReader mreader = new MonitoredReader(breader, 1024*1024, in.length()); - - final String source = post.get("source", ""); - final YMarkDMOZImporter DMOZImporter = new YMarkDMOZImporter(mreader, queueSize, root, source); - - mreader.addChangeListener(sb.tables.bookmarks.getProgressListener("DMOZImporter")); - DMOZImporter.setDepth(6); - OneTimeBusyThread.startFromRunnable(DMOZImporter, 0); - OneTimeBusyThread.startFromRunnable(DMOZImporter.getConsumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink), 0); - - prop.put("status", "1"); - } catch (final Exception e) { - ConcurrentLog.logException(e); - } - } - } else { - prop.put(serverObjects.ACTION_AUTHENTICATE, YMarkTables.USER_AUTHENTICATE_MSG); - } - // return rewrite properties - return prop; - } - - public static void putBookmark(final Switchboard sb, final String bmk_user, final YMarkEntry bmk, - final ArrayBlockingQueue<String> autoTaggingQueue, final boolean autotag, final boolean empty, final String indexing, final boolean medialink) { - try { - final String url = bmk.get(YMarkEntry.BOOKMARK.URL.key()); - // other protocols could cause problems - if(url != null && url.startsWith("http")) { - sb.tables.bookmarks.addBookmark(bmk_user, bmk, true, true); - if(autotag) { - if(!empty) { - autoTaggingQueue.put(url); - } else if(!bmk.containsKey(YMarkEntry.BOOKMARK.TAGS.key()) || bmk.get(YMarkEntry.BOOKMARK.TAGS.key()).equals(YMarkEntry.BOOKMARK.TAGS.deflt())) { - autoTaggingQueue.put(url); - } - } - // fill crawler - if (indexing.equals("single")) { - bmk.crawl(YMarkCrawlStart.CRAWLSTART.SINGLE, medialink, sb); - } else if (indexing.equals("onelink")) { - bmk.crawl(YMarkCrawlStart.CRAWLSTART.ONE_LINK, medialink, sb); - } else if (indexing.equals("fulldomain")) { - bmk.crawl(YMarkCrawlStart.CRAWLSTART.FULL_DOMAIN, medialink, sb); - } - } - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final InterruptedException e) { - ConcurrentLog.logException(e); - } - } -} diff --git a/htroot/api/ymarks/import_ymark.xml b/htroot/api/ymarks/import_ymark.xml deleted file mode 100644 index a72c699cc..000000000 --- a/htroot/api/ymarks/import_ymark.xml +++ /dev/null @@ -1,2 +0,0 @@ -<?xml version='1.0' encoding="UTF-8" standalone='yes'?> -<status code="#(status)#error::ok#(/status)#" />
\ No newline at end of file diff --git a/htroot/api/ymarks/info.html b/htroot/api/ymarks/info.html deleted file mode 100644 index a889d3630..000000000 --- a/htroot/api/ymarks/info.html +++ /dev/null @@ -1,64 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml"> - <head> - <meta http-equiv="content-type" content="text/html; charset=UTF-8" /> - <title>YaCy - YMark API Info</title> - </head> - <body> - <h3>YaCy - YMark API Info</h3> - <ul> - <li>add_ymark.xml - <ul> - <li><i>url: the bookmark url</i></li> - <li><i>title:</i></li> - <li><i>desc:</i></li> - <li><i>tags:</i></li> - <li><i>folders:</i></li> - <li><i>public:</i></li> - </ul> - </li> - <li>delete_ymark.xml - <ul> - <li><i>id:</i></li> - <li><i>url:</i></li> - </ul> - </li> - <li>get_ymark.xml - <ul> - <li><i>id:</i></li> - <li><i>url:</i></li> - <li><i>tags:</i></li> - <li><i>folders:</i></li> - </ul> - </li> - <li>get_treeview.[xml | json] - <ul> - <li><i>root:</i> the folder to display</li> - <li><i>bmtype:</i></li> - </ul> - </li> - <li>get_xml - <ul> - <li><i>root:</i> the root folder to start export from</li> - <li><i>type:</i> xbel | rss | rdf</li> - </ul> - </li> - <li>import_ymark.[xml | html] - <ul> - <li><i>bmfile:</i> the bookmark file you want to import - <ul> - <li>method="post"</li> - <li>enctype="multipart/form-data"</li> - <li>charset="UTF-8" - <br/>for special characters to display correctly, it is important that the file is utf-8 encoded - </li> - </ul> - </li> - <li><i>importer:</i> html | xbel </li> - <li><i>table:</i> the table you want to import in (admin authentication required)</li> - <li><i>redirect:</i> you can specify a html redirect (HTTP-EQUIV="REFRESH")</li> - </ul> - </li> - </ul> - </body> -</html>
\ No newline at end of file diff --git a/htroot/api/ymarks/manage_tags.java b/htroot/api/ymarks/manage_tags.java deleted file mode 100644 index ac43ec052..000000000 --- a/htroot/api/ymarks/manage_tags.java +++ /dev/null @@ -1,85 +0,0 @@ -import java.io.IOException; -import java.util.Iterator; -import java.util.regex.Pattern; - -import net.yacy.cora.protocol.RequestHeader; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.data.UserDB; -import net.yacy.data.ymark.YMarkTables; -import net.yacy.data.ymark.YMarkUtil; -import net.yacy.data.ymark.YMarkTables.TABLES; -import net.yacy.kelondro.blob.Tables.Row; -import net.yacy.search.Switchboard; -import net.yacy.server.serverObjects; -import net.yacy.server.serverSwitch; - - -public class manage_tags { - - private static Switchboard sb = null; - private static serverObjects prop = null; - - public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) { - sb = (Switchboard) env; - prop = new serverObjects(); - - String qtype; - String query; - String tags; - String replace; - - final UserDB.Entry user = sb.userDB.getUser(header); - final boolean isAdmin = (sb.verifyAuthentication(header)); - final boolean isAuthUser = user!= null && user.hasRight(UserDB.AccessRight.BOOKMARK_RIGHT); - - if(isAdmin || isAuthUser) { - final String bmk_user = (isAuthUser ? user.getUserName() : YMarkTables.USER_ADMIN); - - if(post != null) { - query = post.get("query", post.get("tags", YMarkUtil.EMPTY_STRING)); - qtype = post.get("qtype", "_tags"); - tags = YMarkUtil.cleanTagsString(post.get("tags", YMarkUtil.EMPTY_STRING)); - replace = post.get("replace", YMarkUtil.EMPTY_STRING); - - } else { - query = ".*"; - qtype = YMarkUtil.EMPTY_STRING; - tags = YMarkUtil.EMPTY_STRING; - replace = YMarkUtil.EMPTY_STRING; - } - - try { - final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); - final Iterator<Row> row_iter; - if(!query.isEmpty()) { - if(!qtype.isEmpty()) { - if(qtype.equals("_tags")) { - if(query.isEmpty()) - query = tags; - final String tagsString = YMarkUtil.cleanTagsString(query); - row_iter = sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagsString); - } else if(qtype.equals("_folder")) { - row_iter = sb.tables.bookmarks.getBookmarksByFolder(bmk_user, query); - } else { - row_iter = sb.tables.iterator(bmk_table, qtype, Pattern.compile(query)); - } - } else { - row_iter = sb.tables.iterator(bmk_table, Pattern.compile(query)); - } - } else { - final String tagsString = YMarkUtil.cleanTagsString(tags); - row_iter = sb.tables.bookmarks.getBookmarksByTag(bmk_user, tagsString); - // row_iter = sb.tables.iterator(bmk_table); - } - sb.tables.bookmarks.replaceTags(row_iter, bmk_user, tags, replace); - prop.put("status", 1); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } - } else { - prop.put(serverObjects.ACTION_AUTHENTICATE, YMarkTables.USER_AUTHENTICATE_MSG); - } - // return rewrite properties - return prop; - } -} diff --git a/htroot/api/ymarks/manage_tags.xml b/htroot/api/ymarks/manage_tags.xml deleted file mode 100644 index a72c699cc..000000000 --- a/htroot/api/ymarks/manage_tags.xml +++ /dev/null @@ -1,2 +0,0 @@ -<?xml version='1.0' encoding="UTF-8" standalone='yes'?> -<status code="#(status)#error::ok#(/status)#" />
\ No newline at end of file diff --git a/htroot/api/ymarks/test_import.html b/htroot/api/ymarks/test_import.html deleted file mode 100644 index e30fbf137..000000000 --- a/htroot/api/ymarks/test_import.html +++ /dev/null @@ -1,70 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml"> - <head> - <title>YaCy Bookmarks</title> - </head> - <body> - <form action="/api/ymarks/import_ymark.xml" method="post" enctype="multipart/form-data" accept-charset="UTF-8"> - <fieldset> - <legend>Import Bookmarks</legend> - <dl> - <dt> - <label for="importer">Bookmark File Format:</label> - </dt> - <dd> - <input type="radio" name="importer" value="html" /> Netscape HTML, - <small> - bookmark.htm (Firefox, IE, Safari, Opera) - </small> - <br /> - <input type="radio" name="importer" value="json" /> Firefox JSON - <br /> - <input type="radio" name="importer" value="xbel" /> XBEL, - <small> - <a href="http://pyxml.sourceforge.net/topics/xbel/" target="_blank"> - XML Bookmark Exchange Language - </a> - </small> - <br /> - <input type="radio" name="importer" value="surro" /> Surrogate Reader - </dd> - <br /> - <dt> - <label for="file">File:</label> - </dt> - <dd> - <input type="file" name="bmkfile" id="bmkfile" /> - </dd> - <br /> - <dt> - <label for="root">Source Folder:</label> - </dt> - <dd> - <input type="text" name="source" id="source" value="" disabled="disabled" /> - </dd> - <br /> - <dt> - <label for="root">Target Folder:</label> - </dt> - <dd> - <input type="text" name="root" id="root" value="/Imported Bookmarks" /> - </dd> - <br /> - <dt> - <label for="autotag">Settings for automatic tagging:</label> - </dt> - <dd> - <input type="radio" name="autotag" value="off" /> Off - <br /> - <input type="radio" name="autotag" value="empty" /> Only for empty tags - <br /> - <input type="radio" name="autotag" value="overwrite" /> Overwriting existing tags - <br /> - <input type="radio" name="autotag" value="merge" /> Merging with existing tags - </dd> - </dl> - <input type="submit" name="importbookmarks" value="import" class="btn btn-primary"/> - </fieldset> - </form> - </body> -</html>
\ No newline at end of file diff --git a/htroot/api/ymarks/test_treeview.html b/htroot/api/ymarks/test_treeview.html deleted file mode 100644 index cba3dbdf0..000000000 --- a/htroot/api/ymarks/test_treeview.html +++ /dev/null @@ -1,38 +0,0 @@ -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> -<html xmlns="http://www.w3.org/1999/xhtml"> - <head> - <link media="screen" type="text/css" href="yacy/ui/css/jquery.treeview.css" rel="stylesheet" /> - <title>YaCy Bookmarks</title> - <script src="yacy/ui/js/jquery-1.3.2.min.js" type="text/javascript"></script> - <script src="yacy/ui/js/jquery.treeview.min.js" type="text/javascript"></script> - <script src="yacy/ui/js/jquery.treeview.async.js" type="text/javascript"></script> - <script type="text/javascript"> - //<![CDATA[ - $(document).ready(function() { - $("#yfolder").treeview({ - url: "get_treeview.json?bmtype=href&display=bmk", - unique: false, - toggle: function() { - - } - }); - }); - $("button").click(function () { - alert("test"); - }); - //]]> - </script> - </head> - <body> - <h3>YaCy Bookmarks</h3> - <div> - <ul id="yfolder" class="filetree"></ul> - </div> - </body> -</html> - - - - - - diff --git a/htroot/env/yacy-ymarks.css b/htroot/env/yacy-ymarks.css deleted file mode 100644 index 7478c87e8..000000000 --- a/htroot/env/yacy-ymarks.css +++ /dev/null @@ -1,232 +0,0 @@ -/* Body */ -#ymarks_body { - -} - -/* Main */ -#ymarks_body div.main { - margin-top: 7px; -} - -/* Sidebar */ -#ymarks_body div.ymark-sidebar { - width:250px; - float:right; - margin-right: 10px; - margin-left:10px; - margin-top:5px; - padding-top:2px; -} - -#ymarks_folders_tab { - margin-left: -5px; - margin-right: -5px; -} - -/* YaCy Bookmarks ---------------------------*/ -#ymarks_flexigrid .url { - padding-top: 1px; - font-size: 0.9em; - line-height: 1.2em; - margin-bottom:0.6em; -} -#ymarks_flexigrid h3.linktitle { - margin-top:2px; - margin-bottom: -0.5em; - font-size: 1.0em; - line-height: 1.2em; - color:#239AFF; -} -#ymarks_flexigrid h3.linktitle a { - color:#239AFF; - text-decoration:none; -} -#ymarks_flexigrid p.url a { - color:#20A020; - text-decoration:none; -} -#ymarks_flexigrid .tag { - font-size: 1.1em; - line-height: 1.3em; - color:#20A020; -} -#ymarks_flexigrid .desc { - font-style:italic; - font-size: 0.9em; - color: #808080; - line-height: 1.2em; - margin-bottom:-1em; -} - -#ymarks_treeview { - overflow-x: hidden; - overflow-y: auto; -} -#ymarks_tagcloud { - margin: 10px 5px 10px 5px; /* top right bottom left */ - text-align: justify; - line-height: 1.2em; -} -#ymarks_tagcloud a { - cursor:pointer; -} -#ymarks_tagcloud a:hover { - background-color:#e6EEEE; -} -#ymarks_body .input { - width: 190px; - margin-bottom: .5em; - padding: 2px 0px 2px 0px; /* top right bottom left */ - border: 1px solid #CCCCCC; - font-family: "Trebuchet MS", Trebuchet, Verdana, Helvetica, Arial, sans-serif; - font-size: 12px; - /* color: #999999; */ -} -img.help { - vertical-align: middle; - padding-bottom:5px; - padding-left: 8px; -} - -/* Bookmark Dialogs */ -.bm_input, .bm_select { - padding: 2px 0px 2px 0px; /* top right bottom left */ - border: 1px solid #CCCCCC; - margin: 0px 0px 2px 5px; -} -.bm_input { - width: 280px; -} -.bm_select { - width: 60px; -} -.bm_dialog label { - margin: 6px 0px 2px 5px; - font-size: 0.8em; -} -.bm_dialog { - margin: 5px 5px 5px 5px; -} - -#bm_tags_tagsinput { - margin: 0px 0px 2px 5px; -} - - -/* YaCy Flexigrid ---------------------------*/ -.flexigrid div.fbutton .burst { - background: url(../../yacy/ui/img-2/burst.png) no-repeat center left; -} -.flexigrid div.fbutton .pictures { - background: url(../../yacy/ui/img-2/pictures.png) no-repeat center left; -} -.flexigrid div.fbutton .help { - background: url(../../yacy/ui/img-2/question_blue.png) no-repeat center left; -} -.flexigrid div.fbutton .delete { - background: url(../../yacy/ui/img-2/cancel.png) no-repeat center left; -} -.flexigrid div.fbutton .edit { - background: url(../../yacy/ui/img-2/pencil.png) no-repeat center left; -} -.flexigrid div.fbutton .import { - background: url(../../yacy/ui/img-2/download.png) no-repeat center left; -} -.flexigrid div.fbutton .rss { - background: url(../../yacy/ui/img-2/rss_round.png) no-repeat center left; -} -.flexigrid div.fbutton .xml { - background: url(../../yacy/ui/img-2/code.png) no-repeat center left; -} -.flexigrid div.fbutton .bookmark { - border-bottom: 0px; - background: url(../../yacy/ui/img/bookmark.png) no-repeat center left; -} -.flexigrid div.fbutton .blacklist { - background: url(../../yacy/ui/img/blacklist.png) no-repeat center left; -} -.flexigrid div.fbutton .crawl { - background: url(../../yacy/ui/img/crawl.png) no-repeat center left; -} -.flexigrid div.fbutton .next { - background: url(../../yacy/ui/img/next.png) no-repeat center left; -} -.flexigrid div.fbutton .prev { - background: url(../../yacy/ui/img/prev.png) no-repeat center left; -} -.flexigrid div.fbutton .load { - background: url(../../yacy/ui/img/flexigrid/load.png) no-repeat center left; -} -.flexigrid div.fbutton .addTag { - background: url(../../yacy/ui/img/tags/tag_blue_add.png) no-repeat center left; -} -.flexigrid div.fbutton .editTag { - background: url(../../yacy/ui/img/tags/tag_blue_edit.png) no-repeat center left; -} -.flexigrid div.fbutton .deleteTag { - background: url(../../yacy/ui/img/tags/tag_blue_delete.png) no-repeat center left; -} -.flexigrid div.fbutton .log-debug { - background: url(../../yacy/ui/img-2/magnify.png) no-repeat center left; -} -.flexigrid div.fbutton .log-info { - background: url(../../yacy/ui/img-2/info.png) no-repeat center left; -} -.flexigrid div.fbutton .log-system { - background: url(../../yacy/ui/img-2/accept_green.png) no-repeat center left; -} -.flexigrid div.fbutton .log-warning { - background: url(../../yacy/ui/img/blacklist.png) no-repeat center left; -} -.flexigrid div.fbutton .log-error { - background: url(../../yacy/ui/img-2/stop.png) no-repeat center left; -} -.flexigrid div.fbutton .log-new { - background: url(../../yacy/ui/img-2/new.png) no-repeat center left; -} -.flexigrid div.fbutton .clock { - background: url(../../yacy/ui/img-2/clock.png) no-repeat center left; -} -.flexigrid div.fbutton .calendar { - background: url(../../yacy/ui/img-2/calendar.png) no-repeat center left; -} -.flexigrid div.fbutton .refresh { - background: url(../../yacy/ui/img/flexigrid/load.png) no-repeat center left; -} -.flexigrid div.fbutton .accept { - background: url(../../yacy/ui/img-2/accept.png) no-repeat center left; -} -.flexigrid div.fbutton .log-more { - /* background: url(../img/flexigrid/uup.png) no-repeat center left; */ - margin-left: -20px; -} -/* overrides for ui-tab styles --------------------------------------*/ -#ymarks_sidebar .ui-tabs-nav li a:link, -#ymarks_sidebar .ui-tabs-nav li a:visited, -#ymarks_sidebar .ui-tabs-nav li a:hover, -#ymarks_sidebar .ui-tabs-nav li a:active, -#ymarks_sidebar .ui-tabs-nav li.ui-tabs-selected a:link, -#ymarks_sidebar .ui-tabs-nav li.ui-tabs-selected a:visited, -#ymarks_sidebar .ui-tabs-nav li.ui-tabs-selected a:hover, -#ymarks_sidebar .ui-tabs-nav li.ui-tabs-selected a:active { font-size: 12px; padding-left: 5px;} - - -/* overrides for treeview styles --------------------------------------*/ - -.filetree span.err { - background: url(../yacy/ui/img-2/stop.png) 0 0 no-repeat; - padding: 1px 0px 0px 18px; - display: block; -} -.filetree span.question { - background: url(../yacy/ui/img-2/question_blue.png) 0 0 no-repeat; - padding: 1px 0px 0px 18px; - display: block; -} -.filetree span.crawl { - background: url(../yacy/ui/img/crawl.png) 0 0 no-repeat; - padding: 1px 0px 0px 18px; - display: block; -} diff --git a/htroot/js/yacy-ymarks-bookmark-actions.js b/htroot/js/yacy-ymarks-bookmark-actions.js deleted file mode 100644 index c0f887348..000000000 --- a/htroot/js/yacy-ymarks-bookmark-actions.js +++ /dev/null @@ -1,213 +0,0 @@ -/* -* Copyright (C) 2011, 2012 Stefan Förster -* -* This file is part of YaCy. -* -* YaCy is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 2 of the License, or -* (at your option) any later version. -* -* YaCy is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with YaCy. If not, see <http://www.gnu.org/licenses/>. -*/ - -/* Initialize Bookmark Actions */ -function bm_action(com,grid) { - if (com=='Delete') { - var check = confirm('Delete ' + $('.trSelected',grid).length + ' bookmark(s)?'); - if(check == true) { - $('.trSelected',grid).each(function(){ - var url = "/api/ymarks/delete_ymark.xml?id="+$(this).find('td :first').text(); - $.ajax({ - type: 'POST', - url: url, - dataType: 'xml', - success: function(xml) { - $('#ymarks_flexigrid').flexReload(); - loadTreeView(); - } - }); // close $.ajax( - }); //close each( - } - } - else if (com=='Add') { - $('#bmaddform').resetForm(); - $('#bm_tags').importTags(''); - $("#bm_url").removeAttr("disabled"); - $("#bm_url").blur(function() { - var url = $("#bm_url").getValue(); - $.ajax({ - type: "GET", - url: "/api/ymarks/get_metadata.xml?url="+url, - dataType: "xml", - success: function(xml) { - var title = ""; - var desc = ""; - var tags = ""; - if ($(xml).find('info').attr('status') === "error") { - $("#bmaddimg").attr("src","/yacy/ui/img-1/Smiley Star Sad.png"); - } else { - var autotags = $(xml).find('autotags') - title = $(xml).find('title').text(); - desc = $(xml).find('desc').text(); - tags = ""; - $(autotags).find('tag').each(function(){ - tags = tags + "," + $(this).attr('name'); - }); - $("#bmaddimg").attr("src","/yacy/ui/img-1/Smiley Star.png"); - } - $("#bm_title").setValue(title); - $("#bm_desc").setValue(desc); - /* $("#bm_tags").setValue(tags); */ - $('#bm_tags').importTags(tags); - } - }); - }); - $("#ymarks_add_dialog").dialog('open'); - } else if (com=='Edit') { - if ($('.trSelected',grid).length > 1) { - alert("Editing of more than one selected bookmark is currently not supportet!"); - return false; - } - $("#bm_url").attr("disabled","disabled"); - $("#bm_url").setValue($('.trSelected',grid).find('.url').text()); - $("#bm_title").setValue($('.trSelected',grid).find('h3.linktitle').text().trim()); - $("#bm_desc").setValue($('.trSelected',grid).find('p.desc').text().trim()); - $('#bm_tags').importTags($('.trSelected',grid).find('p.tags').text().trim().replace(/,\s/g,",")); - /* $("#bm_tags").setValue($('.trSelected',grid).find('p.tags').text().trim().replace(/,\s/g,",")); */ - $("#bm_path").setValue($('.trSelected',grid).find('p.folders').text().replace(/, \s/g,",")); - $("#bm_public").setValue($('.trSelected',grid).find('img').attr('alt')); - $("#ymarks_add_dialog").dialog('open'); - } else if (com=='Crawl') { - if ($('.trSelected',grid).length == 1 && $(this).find('.apicall_pk').text() == "") { - var pk = $(this).find('.apicall_pk').text(); - $("input[name='crawlingURL']").setValue($('.trSelected',grid).find('.url').text()); - $("#ymarks_crawlstart").dialog('open'); - } else { - var param = []; - var i = 0; - $('.trSelected',grid).each(function() { - var pk = $(this).find('.apicall_pk').text(); - - if (pk == "") { - /* - if (crawl_param.length == 0) { - $('<td colspan="2">You have selected one or more bookmarks without a crawl start entry in the API table. You can define a default profile which will be used instead.</td>').appendTo("#ymarks_crawlstart_msg"); - $("input[name='crawlingURL']").attr("disabled","disabled"); - $("input[name='crawlingURL']").setValue("Default profile"); - $("#ymarks_crawlstart").dialog('open'); - } - */ - alert("Multiple selection currently only supports bookmarks"+"\n"+"with an existing crawl profile in the API table."); - } else { - var item = {name : 'item_'+i, value : "mark_"+pk}; - param[i] = item; - i++; - } - }); - param[param.length] = { name : 'execrows', value : 'true' }; - $.ajax({ - type: "POST", - data: param, - url: "Table_API_p.html", - dataType: "html", - success: function() { - $('#ymarks_flexigrid').flexReload(); - } - }); - - } - } else if (com=='XBEL') { - window.open("/api/ymarks/get_xbel.xml","_blank"); - return false; - } else { - alert("Sorry, the function you have requested is not yet available!"); - return false; - } -} - -function bm_dialog() { - /* Init Tag Input */ - $('#bm_tags').tagsInput({ - 'height':'105px', - 'width':'270px', - 'interactive':true, - 'removeWithBackspace' : true, - 'minChars' : 0, - 'maxChars' : 0, - 'placeholderColor' : '#666666' - }); - - /* Initialize Bookmark Dialog */ - $("#ymarks_add_dialog").dialog({ - autoOpen: false, - height: 500, - width: 340, - position: ['top',100], - modal: true, - resizable: false, - buttons: { - OK: function() { - var param = []; - var i = 0; - $("#bmaddform input,#bmaddform select,#bm_desc").each(function() { - var item = {name : $(this).attr("name"), value : $(this).attr("value")}; - param[i] = item; - i++; - }); - $.ajax({ - type: "POST", - url: "/api/ymarks/add_ymark.xml", - data: param, - dataType: "xml", - success: function(xml) { - $('#bmaddform').resetForm(); - $("#bm_url").unbind('blur'); - $("#ymarks_add_dialog").dialog("close"); - $('#ymarks_flexigrid').flexReload(); - loadTreeView(); - return false; - } - }); - } , - Cancel: function() { $("#ymarks_add_dialog").dialog("close"); } - } - }); - /* Initialize Crawl Start Dialog */ - $("#ymarks_crawlstart").dialog({ - autoOpen: false, - height: 450, - width: 470, - position: ['top',100], - modal: true, - resizable: false, - buttons: { - OK: function() { - var param = []; - var i = 0; - $("#ymarks_crawler input[type='text'],#ymarks_crawler input:checked,#ymarks_crawler select,#ymarks_crawler input[type='hidden']").each(function() { - var item = {name : $(this).attr("name"), value : $(this).attr("value")}; - param[i] = item; - i++; - }); - $.ajax({ - type: "POST", - data: param, - url: "Crawler_p.html", - dataType: "html", - success: function() { - } - }); - $('#ymarks_flexigrid').flexReload(); - $("#ymarks_crawlstart").dialog("close"); - } , - Cancel: function() { $("#ymarks_crawlstart").dialog("close"); } - } - }); -}
\ No newline at end of file diff --git a/htroot/js/yacy-ymarks-tag-actions.js b/htroot/js/yacy-ymarks-tag-actions.js deleted file mode 100644 index 5ae293605..000000000 --- a/htroot/js/yacy-ymarks-tag-actions.js +++ /dev/null @@ -1,31 +0,0 @@ -/* -* Copyright (C) 2011 Stefan Förster -* -* This file is part of YaCy. -* -* YaCy is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 2 of the License, or -* (at your option) any later version. -* -* YaCy is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with YaCy. If not, see <http://www.gnu.org/licenses/>. -*/ - -/* Initialize Tag Actions */ -function tag_action(com,grid) { - alert("Sorry, the function you have requested is not yet available!"); - if (com=='Add') { - flex = grid; - $('#tagaddform').resetForm(); - $("#tagadd").dialog('open'); - } else { - $('#tageditform').resetForm(); - $("#tagedit").dialog('open'); - } -};
\ No newline at end of file diff --git a/htroot/js/yacy-ymarks.js b/htroot/js/yacy-ymarks.js deleted file mode 100644 index dde7b6bd7..000000000 --- a/htroot/js/yacy-ymarks.js +++ /dev/null @@ -1,340 +0,0 @@ -/* -* Copyright (C) 2011, 2012 Stefan Förster -* -* This file is part of YaCy. -* -* YaCy is free software: you can redistribute it and/or modify -* it under the terms of the GNU General Public License as published by -* the Free Software Foundation, either version 2 of the License, or -* (at your option) any later version. -* -* YaCy is distributed in the hope that it will be useful, -* but WITHOUT ANY WARRANTY; without even the implied warranty of -* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -* GNU General Public License for more details. -* -* You should have received a copy of the GNU General Public License -* along with YaCy. If not, see <http://www.gnu.org/licenses/>. -*/ - -HTMLenc = function(s) { - return $('<div/>').text(s).html(); -} -$(document).ready(function() { - - height=document.documentElement.clientHeight - 200; - qtag = ""; - - /* Initialize Bookmark Dialog */ - bm_dialog(); - - /* Initialize Flexigrid */ - $('#ymarks_flexigrid').flexigrid({ - url: '/api/ymarks/get_ymark.json', - dataType: 'json', - method: 'GET', - colModel: [ - {display: 'Hash', name : 'hash', width : 85, sortable : false, align: 'left', hide: true}, - {display: 'Public', name : 'public', width : 20, sortable : true, align: 'center'}, - {display: 'Crawl start', name : 'crawl_start', width : 20, sortable : false, align: 'center'}, - {display: 'Title', name : 'title', width : 400, sortable : true, align: 'left'}, - {display: 'Tags', name : 'tags', width : 160, sortable : false, align: 'left'}, - {display: 'Folders', name : 'folders', width : 160, sortable : true, align: 'left'}, - {display: 'Date added', name : 'date_added', width : 100, sortable : true, align: 'left'}, - {display: 'Date modified', name : 'date_modified', width : 100, sortable : true, align: 'left'}, - {display: 'Date visited', name : 'date_visited', width : 100, sortable : true, align: 'left', hide: true}, - {display: 'API PK', name : 'apicall_pk', width : 85, sortable : true, align: 'left', hide: true}, - {display: 'Date recording', name : 'date_recording', width : 100, sortable : false, align: 'left', hide: true}, - {display: 'Date next exec', name : 'date_next_exec', width : 100, sortable : false, align: 'left', hide: true}, - {display: 'Date last exec', name : 'date_last_exec', width : 100, sortable : false, align: 'left', hide: true} - ], - buttons: [ - {name: '...', bclass: 'refresh', onpress: function() { - $('#ymarks_flexigrid').flexOptions({ - sortname: "title", - sortorder: "asc", - query: ".*", - qtype: "title" - }); - $('#ymarks_flexigrid').flexReload(); - loadTreeView(); - - }}, - {separator: true}, - {name: 'Add', bclass: 'bookmark', onpress: bm_action}, - {name: 'Edit', bclass: 'edit', onpress: bm_action}, - {name: 'Delete', bclass: 'delete', onpress: bm_action}, - {separator: true}, - {name: 'Crawl', bclass: 'crawl', onpress: bm_action}, - {name: 'Schedule', bclass: 'calendar', onpress: bm_action}, - {separator: true}, - {name: 'XBEL', bclass: 'xml', onpress: bm_action}, - {separator: true}, - {name: 'Help', bclass: 'help', onpress: bm_action} - ], - searchitems : [ - {display: 'Full text (regexp)', name : ''}, - {display: 'Tags (comma seperated)', name : '_tags'}, - {display: 'Tags (regexp)', name : 'tags'}, - {display: 'Folders (comma seperated)', name : '_folder'}, - {display: 'Folders (regexp)', name : 'folders'}, - {display: 'Title (regexp)', name : 'title'}, - {display: 'Description (regexp)', name : 'desc'} - ], - useRp: true, - rp: 15, - sortname: "title", - sortorder: "asc", - usepager: true, - striped: true, - nowrap: false, - height: height, - query: ".*", - qtype: "title" - }); - - /* Initialize Sidebar */ - $('#ymarks_sidebar').height(height+90); - $tabs = $('#ymarks_sidebar').tabs({ - // tabs options - }); - - $tabs.bind('tabsselect', function(event, ui) { - /* - Objects available in the function context: - ui.tab - anchor element of the selected (clicked) tab - ui.panel - element, that contains the selected/clicked tab contents - ui.index - zero-based index of the selected (clicked) tab - */ - tabid = "#"+ui.panel.id; - if (tabid == "#ymarks_tags_tab") { - loadTagCloud(); - } - return true; - }); - - loadTreeView(); - - $('input[name=importer]').change(function() { - if ($("input[name=importer]:checked").val() == 'crawls') { - $("input[name='root']").setValue("/Crawl Start"); - $("input[name='bmkfile']").attr("disabled","disabled"); - $("input[name='root']").attr("disabled","disabled"); - } else if ($("input[name=importer]:checked").val() == 'bmks') { - $("input[name='bmkfile']").attr("disabled","disabled"); - } else if ($("input[name=importer]:checked").val() == 'dmoz') { - $("input[name='bmkfile']").attr("disabled","disabled"); - $("input[name='root']").setValue("/DMOZ"); - $("input[name='source']").removeAttr("disabled"); - $("input[name='source']").setValue("Top/"); - alert("The DMOZ RDF dump is exspected on your YaCy peer at DATA/WORK/content.rdf.u8.gz" + - "\nYou can download the file from http://rdf.dmoz.org/rdf/content.rdf.u8.gz (ca. 320 MB)." + - "\n\nPlease check http://www.dmoz.org/license.html before you import any DMOZ data into YaCy!" + - "\n\nDue to the large number of links contained in the dmoz file it is recommended" + - "\nto limit the import volume with an appropriate value for the source folder (e.g. Top/Games).") - } else { - $("input[name='bmkfile']").removeAttr("disabled"); - $("input[name='root']").removeAttr("disabled"); - $("input[name='root']").setValue("/Imported Bookmarks"); - $("input[name='source']").attr("disabled","disabled"); - $("input[name='source']").setValue(""); - } - }); - - $("#tag_include").multiselect({ - noneSelectedText: "Select (multiple) tags ...", - height: height-50, - minWidth: 200, - maxWidth: 200, - selectedList: 4, - header: "", - click: function(event, ui) { - if(ui.checked) { - qtag = qtag + "," + ui.value; - } - }, - close: function() { - $('#ymarks_flexigrid').flexOptions({ - query: qtag, - qtype: "_tags", - newp: 1 - }); - $('#ymarks_flexigrid').flexReload(); - }, - beforeopen: function() { - loadTags("#tag_include", "alpha", ""); - }, - open: function() { - qtag = ""; - } - }).multiselectfilter(); - - $("#tag_select").multiselect({ - noneSelectedText: "Select tags to remove ...", - minWidth: 200, - maxWidth: 200, - header: "", - selectedList: 4, - height: height - 540, - beforeopen: function() { - loadTags("#tag_select", "alpha", ""); - } - }).multiselectfilter(); - - $("#ymarks_qtype").multiselect({ - noneSelectedText: "Select query type ...", - minWidth: 200, - maxWidth: 200, - header: "", - multiple: false, - selectedList: 1 - }); - - $("#ymarks_importer").multiselect({ - noneSelectedText: "Select an Importer ...", - minWidth: 200, - maxWidth: 200, - header: "", - multiple: false, - selectedList: 1 - }); - - $("#ymarks_autotag").multiselect({ - noneSelectedText: "Select an option ...", - minWidth: 200, - maxWidth: 200, - header: "", - multiple: false, - selectedList: 1 - }); - - $("#ymarks_indexing").multiselect({ - position: { - my: 'left bottom', - at: 'left top' - }, - noneSelectedText: "Select an option ...", - minWidth: 200, - maxWidth: 200, - header: "", - multiple: false, - selectedList: 1 - }); - - $('#ymarks_tagmanager').submit(function() { - var param = []; - $('#ymarks_tagmanager input[type="text"],#ymarks_tagmanager input[type="radio"]:checked').each(function(i){ - param[i] = { name : $(this).attr('name'), value : $(this).attr('value') }; - }); - var tags = ""; - var ta = $("#tag_select").val(); - var i = 0; - if(ta !== null) { - while (i<ta.length) { - tags = tags + ta[i] + ","; - i++; - } - } - param[param.length] = { name : 'tags', value : tags }; - $.ajax({ - type: "POST", - url: "/api/ymarks/manage_tags.xml", - data: param, - dataType: "xml", - cache: false, - success: function(xml) { - /* - $(xml).find('status').each(function(){ - var code = $(this).attr('code'); - alert("Request returned status: "+code); - }); //close each( - */ - loadTagCloud() - loadTags("#tag_select", "alpha", ""); - loadTags("#tag_include", "alpha", ""); - /* - $('#ymarks_flexigrid').flexOptions({ - sortname: "title", - sortorder: "asc", - query: query, - qtype: "title" - }); - */ - $('#ymarks_flexigrid').flexReload(); - } - }); //close $.ajax( - return false; - }); - -}); - -function loadTags(select, sortorder, tags) { - $(select).empty(); - $.ajax({ - type: "GET", - url: "/api/ymarks/get_tags.xml?sort="+sortorder+"&tag="+tags, - dataType: "xml", - cache: false, - success: function(xml) { - $(xml).find('tag').each(function(){ - var count = $(this).attr('count'); - var tag = $(this).attr('tag'); - $('<option value="'+tag+'">'+HTMLenc(tag)+' ['+count+']</option>').appendTo(select); - }); //close each( - $(select).multiselect('refresh'); - } - }); //close $.ajax( -} - -function loadTagCloud() { - $("#ymarks_tagcloud *").remove(); - $.ajax({ - type: "GET", - url: "/api/ymarks/get_tags.xml?top=25&sort=alpha", - dataType: "xml", - cache: false, - success: function(xml) { - $(xml).find('tag').each(function(){ - var count = $(this).attr('count'); - var tag = $(this).attr('tag'); - var size = ((count/20)+0.15); - if (size < 1) {size = 1;} - $('<a style="font-size:'+size+'em"></a>') - .html(HTMLenc(tag)+' ') - .appendTo('#ymarks_tagcloud') - .bind('click', function() { - var qtag = $(this).text().replace(/\s+$/g,""); - $('#ymarks_flexigrid').flexOptions({ - query: qtag, - qtype: "_tags", - newp: 1 - }); - $('#ymarks_flexigrid').flexReload(); - }); - }); //close each( - } - }); //close $.ajax( -}; - -function loadTreeView() { - $("#ymarks_treeview").empty(); - $("#ymarks_treeview").treeview({ - url: "/api/ymarks/get_treeview.json?bmtype=href", - unique: false, - persist: "location" - }); - - $("#ymarks_treeview").bind("click", function(event) { - if ($(event.target).is("li") || $(event.target).parents("li").length) { - var folder = $(event.target).parents("li").filter(":first").attr("id"); - $('#ymarks_flexigrid').flexOptions({ - query: folder, - qtype: "_folder", - newp: 1 - }); - $('#ymarks_flexigrid').flexReload(); - } - return false; - }); - return false; -} diff --git a/htroot/jslicense.html b/htroot/jslicense.html index 842347362..ea4e68b66 100755 --- a/htroot/jslicense.html +++ b/htroot/jslicense.html @@ -184,21 +184,6 @@ <td><a href="js/xml.js">xml.js</a></td>
</tr>
<tr>
- <td><a href="js/yacy-ymarks-bookmark-actions.js">yacy-ymarks-bookmark-actions.js</a></td>
- <td><a href="http://www.gnu.org/licenses/gpl-2.0.html">GNU-GPL-2.0-or-later</a></td>
- <td><a href="js/yacy-ymarks-bookmark-actions.js">yacy-ymarks-bookmark-actions.js</a></td>
- </tr>
- <tr>
- <td><a href="js/yacy-ymarks-tag-actions.js">yacy-ymarks-tag-actions.js</a></td>
- <td><a href="http://www.gnu.org/licenses/gpl-2.0.html">GNU-GPL-2.0-or-later</a></td>
- <td><a href="js/yacy-ymarks-tag-actions.js">yacy-ymarks-tag-actions.js</a></td>
- </tr>
- <tr>
- <td><a href="js/yacy-ymarks.js">yacy-ymarks.js</a></td>
- <td><a href="http://www.gnu.org/licenses/gpl-2.0.html">GNU-GPL-2.0-or-later</a></td>
- <td><a href="js/yacy-ymarks.js">yacy-ymarks.js</a></td>
- </tr>
- <tr>
<td><a href="js/yacyinteractive.js">yacyinteractive.js</a></td>
<td><a href="http://www.gnu.org/licenses/gpl-2.0.html">GNU-GPL-2.0-or-later</a></td>
<td><a href="js/yacyinteractive.js">yacyinteractive.js</a></td>
diff --git a/htroot/yacy/ui/index.html b/htroot/yacy/ui/index.html index 1f9c713b2..565232050 100644 --- a/htroot/yacy/ui/index.html +++ b/htroot/yacy/ui/index.html @@ -118,7 +118,6 @@ */ tabid = "#"+ui.panel.id; if (ui.index == bmk_tab) { - $("#searchbox select[name='resource']").setValue('ymarks'); $("#searchbox input[name='contentdom']").setValue('text'); $("#searchbox input[name='search']").setValue(qtag); } else if (ui.index > bmk_tab) { @@ -196,8 +195,6 @@ dataType: "xml", success: function(xml) { alert($(xml).find('result').text()); - loadTagCloud(); - reloadBM(); $("#tagadd").dialog('close'); } }); @@ -213,8 +210,6 @@ data: data, dataType: "xml" }); - loadTagCloud(); - reloadBM(); $("#tagedit").dialog('close'); }, Cancel: function() { $("#tagedit").dialog("close"); } @@ -242,8 +237,6 @@ dataType: 'xml' }); // close $.ajax( }); //close each( - loadTagCloud(); - reloadBM(); $("#tagadd").dialog("close"); }, Cancel: function() { @@ -272,20 +265,10 @@ success: function(xml) { $('#bmaddform').resetForm(); $("#bm_url").unbind('blur'); - $("#bmadd").dialog("close"); - loadTagCloud(); - reloadBM(); + $("#bmadd").dialog("close"); } }); } - function reloadBM() { - $('#ymarks').flexOptions({ - query: qtag, - qtype: "tags" - }); - $('div.ftitle').empty().html('YaCy Bookmarks: ' + HTMLenc(qtag)); - $('#ymarks').flexReload(); - } function switchStylestyle(styleName) { $('link[rel*=style][title]').each(function(i) { diff --git a/htroot/yacy/ui/sidebar/sidebar_1.html b/htroot/yacy/ui/sidebar/sidebar_1.html index b4bc30921..a8c22db26 100644 --- a/htroot/yacy/ui/sidebar/sidebar_1.html +++ b/htroot/yacy/ui/sidebar/sidebar_1.html @@ -43,19 +43,10 @@ /* Load Search Result into new Tab */ $('#searchbox').submit(function() { - if ($("select[name='resource']").getValue() == "ymarks") { - qtag = $("input[name='search']").getValue(); - if (tabid == "#Bookmarks") { - reloadBM(); - } else { - $tabs.tabs('select',2); - } - } else { - var tabnr = $tabs.tabs('length'); - var tabname = $("input[name='search']").getValue(); - $tabs.tabs('add', "yacyui-search.html", '*' + HTMLenc(tabname)); - $tabs.tabs('select',tabnr); - } + var tabnr = $tabs.tabs('length'); + var tabname = $("input[name='search']").getValue(); + $tabs.tabs('add', "yacyui-search.html", '*' + HTMLenc(tabname)); + $tabs.tabs('select',tabnr); return false; }); }); @@ -77,7 +68,6 @@ <select name="resource" class="selector"> <option value="/yacysearch.json?resource=global&verify=true" selected="selected">freeworld</option> <option value="/yacysearch.json?resource=local&verify=true" >local peer</option> - <option value="ymarks" >bookmarks</option> <option value="/api/ynetSearch.json?login=&url=http://sciencenet.fzk.de:8080/yacysearch.json?resource=global&verify=true" >sciencenet-ynetSearch</option> <option value="http://sciencenet.fzk.de:8080/yacysearch.json?callback=?&resource=global&verify=true" >sciencenet-JsonP</option> <option value="http://yacy.kicks-ass.net:8080/yacysearch.json?callback=?&resource=global&verify=true" >apfelmaennchen-JsonP</option> diff --git a/htroot/yacy/ui/sidebar/sidebar_2.html b/htroot/yacy/ui/sidebar/sidebar_2.html index 237bc797f..3d6e62385 100644 --- a/htroot/yacy/ui/sidebar/sidebar_2.html +++ b/htroot/yacy/ui/sidebar/sidebar_2.html @@ -7,54 +7,12 @@ clearStyle: true, header: "h3" }); - $("#side2").accordion("activate", 1); - /* Load TagCloud */ - loadTagCloud(); + $("#side2").accordion("activate", 1); }); $("#topwords").treeview({ unique: true }); - - function loadTagCloud() { - $("#tagcloud *").remove(); - $("#sidebar-2-1").toggleClass("ui.loading"); - $.ajax({ - type: "POST", - url: "/api/ymarks/get_tags.xml?top=25&sort=alpha", - dataType: "xml", - cache: false, - success: function(xml) { - var bhtml = "<li id='bmk_tags_folder' class='closed'><span class='folder'>Bookmark Tags</span><ul>"; - $(xml).find('tag').each(function(){ - var count = $(this).attr('count'); - var tag = $(this).attr('tag'); - var size = ((count/20)+0.3); - if (size < 1) {size = 1;} - $('<a style="font-size:'+size+'em"></a>') - .html(HTMLenc(tag)+' ') - .appendTo('#tagcloud') - .bind('click', function() { - qtag = $(this).text().replace(/\s+/g,""); - $("select[name='resource']").setValue('ymarks'); - $("input[name='search']").setValue(qtag); - if ($tabs.data('selected.tabs') == bmk_tab) { - reloadBM(); - } else { - $tabs.tabs('select',2); - } - }); - bhtml = bhtml + "<li><span class='tag'>"+tag+"</span></li>"; - }); //close each( - bhtml = bhtml + "</ul></li>"; - var bfold = $(bhtml).prependTo("#yfolder"); - $("#yfolder").treeview({ - add: bfold - }); - $("#sidebar-2-1").toggleClass("ui.loading"); - } - }); //close $.ajax( - } //]]> </script> diff --git a/htroot/yacy/ui/yacyui-bookmarks.html b/htroot/yacy/ui/yacyui-bookmarks.html index 6e46bde39..69c600109 100644 --- a/htroot/yacy/ui/yacyui-bookmarks.html +++ b/htroot/yacy/ui/yacyui-bookmarks.html @@ -1,139 +1,6 @@ <script type="text/javascript"> - //<![CDATA[ - $(document).ready(function() { - $("#side1").accordion("activate", 1); - $("#side2").accordion("activate", 1); - // $('#yoptions').resetForm(); - - var height=document.documentElement.clientHeight - 240; - $('#ymarks').flexigrid({ - url: '/api/ymarks/get_ymark.json', - dataType: 'json', - method: 'GET', - colModel: [ - {display: 'Hash', name : 'hash', width : 50, sortable : false, align: 'center', hide: true}, - {display: 'Public', name : 'public', width : 25, sortable : true, align: 'center'}, - {display: 'Title', name : 'title', width : 400, sortable : true, align: 'left'}, - {display: 'Tags', name : 'tags', width : 160, sortable : false, align: 'left'}, - {display: 'Folders', name : 'folders', width : 160, sortable : false, align: 'left', hide: false}, - {display: 'Date', name : 'date', width : 100, sortable : true, align: 'left'} - ], - buttons: [ - {name: 'Add', bclass: 'bookmark', onpress: bm_action}, - {name: 'Crawl', bclass: 'crawl', onpress: bm_action}, - {name: 'Edit', bclass: 'edit', onpress: bm_action}, - {name: 'Delete', bclass: 'delete', onpress: bm_action}, - {separator: true}, - {name: 'RSS', bclass: 'rss', onpress: bm_action}, - {name: 'XBEL', bclass: 'xml', onpress: bm_action}, - {name: 'XML', bclass: 'xml', onpress: bm_action}, - {separator: true}, - {name: 'Add', bclass: 'addTag', onpress: tag_action}, - {name: 'Rename', bclass: 'editTag', onpress: tag_action}, - {separator: true}, - {name: 'Help', bclass: 'help', onpress: bm_action}, - ], - searchitems : [ - {display: 'Tags', name : 'tags'}, - {display: 'Folders', name : 'folders'}, - {display: 'Title', name : 'title'}, - ], - useRp: true, - rp: 15, - sortname: "title", - sortorder: "asc", - usepager: true, - striped: true, - nowrap: false, - height: height, - query: ".*", - qtype: "title", - title: 'YaCy Bookmarks: ' + HTMLenc(qtag) - }); - - }); - function bm_action(com,grid) { - if (com=='XBEL') { - var qtype = "tags"; - if (qtag.startsWith("/")) qtype = "folders"; - window.document.location.href = '/api/bookmarks/get_bookmarks.xml?display=xbel&qtype='+qtype+'&query='+qtag; - } - else if (com=='RSS') { - window.document.location.href = '/Bookmarks.rss?qtype=tags&query='+qtag; - } - else if (com=='XML') { - window.document.location.href = '/api/bookmarks/get_bookmarks.xml?tag='+qtag; - } - else if (com=='Delete') { - var check = confirm('Delete ' + $('.trSelected',grid).length + ' bookmark(s)?'); - if(check == true) { - $('.trSelected',grid).each(function(){ - var url = "/api/bookmarks/posts/delete_p.xml?login=&urlhash="+$(this).find('td :first').text(); - $.ajax({ - type: 'POST', - url: url, - dataType: 'xml' - }); // close $.ajax( - }); //close each( - loadTagCloud(); - reloadBM(); - } - } - else if (com=='Add') { - $('#bmaddform').resetForm(); - $("#bm_url").blur(function() { - var url = $("input[name='bm_url']").getValue(); - $.ajax({ - type: "GET", - url: "/api/getpageinfo_p.xml?url="+url, - dataType: "xml", - success: function(xml) { - var title = $(xml).find('title').text(); - $("input[name='bm_title']").setValue(title); - var desc = $(xml).find('desc').text(); - $("textarea[name='bm_desc']").setValue(desc); - tags = ""; - $(xml).find('tag').each(function(){ - tags = tags + "," + $(this).attr('name'); - }); - $("input[name='bm_tags']").setValue(tags); - } - }); - }); - $("#bmadd").dialog('open'); - } - else if (com=='Edit') { - if ($('.trSelected',grid).length > 1) { - alert("Edit of more than one selected bookmark is currently not supportet!"); - return false; - } - $("input[name='bm_url']").setValue($('.trSelected',grid).find('.url').text()); - $("input[name='bm_title']").setValue($('.trSelected',grid).find('h3.linktitle').text().trim()); - $("textarea[name='bm_desc']").setValue($('.trSelected',grid).find('p.desc').text().trim()); - $("input[name='bm_tags']").setValue($('.trSelected',grid).find('p.tags').text().trim().replace(/,\s/g,",")); - $("input[name='bm_path']").setValue($('.trSelected',grid).find('p.folders').text().replace(/,\s/g,",")); - $("select[name='bm_public']").setValue($('.trSelected',grid).find('img').attr('alt')); - $("#bmadd").dialog('open'); - } - } - function tag_action(com,grid) { - if (com=='Add') { - flex = grid; - $('#tagaddform').resetForm(); - $("#tagadd").dialog('open'); - } else { - $('#tageditform').resetForm(); - $("#tagedit").dialog('open'); - } - } - //]]> </script> -<!-- Display Bookmarks Table --> -<table id="ymarks" summary="YaCy Bookmarks"> - <tbody> - </tbody> -</table> diff --git a/htroot/yacysearch.java b/htroot/yacysearch.java index c25afd85b..e252a61bb 100644 --- a/htroot/yacysearch.java +++ b/htroot/yacysearch.java @@ -57,16 +57,13 @@ import net.yacy.cora.federate.yacy.CacheStrategy; import net.yacy.cora.geo.GeoLocation; import net.yacy.cora.lod.vocabulary.Tagging; import net.yacy.cora.lod.vocabulary.Tagging.Metatag; -import net.yacy.cora.protocol.ClientIdentification; import net.yacy.cora.protocol.Domains; import net.yacy.cora.protocol.HeaderFramework; import net.yacy.cora.protocol.RequestHeader; import net.yacy.cora.protocol.ResponseHeader; import net.yacy.cora.util.ConcurrentLog; -import net.yacy.data.BookmarksDB.Bookmark; import net.yacy.data.DidYouMean; import net.yacy.data.UserDB; -import net.yacy.data.ymark.YMarkTables; import net.yacy.document.LibraryProvider; import net.yacy.document.Tokenizer; import net.yacy.http.servlets.TemplateProcessingException; @@ -96,7 +93,6 @@ import net.yacy.search.ranking.RankingProfile; import net.yacy.server.serverObjects; import net.yacy.server.serverSwitch; import net.yacy.server.servletProperties; -import net.yacy.utils.crypt; public class yacysearch { @@ -219,7 +215,7 @@ public class yacysearch { /* * Access to authentication protected features is explicitely requested here * but no authentication is provided : ask now for authentication. - * Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information + * Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information * and as this page is not private, protected features would simply be hidden without asking browser again for authentication. * (see mantis 766 : http://mantis.tokeek.de/view.php?id=766) * */ @@ -305,8 +301,8 @@ public class yacysearch { /* Maximum number of suggestions to display in the first results page */ final int meanMax = post.getInt("meanCount", 0); - boolean jsResort = global - && (contentdom == ContentDomain.ALL || contentdom == ContentDomain.TEXT) // For now JavaScript resorting can only be applied for text search + boolean jsResort = global + && (contentdom == ContentDomain.ALL || contentdom == ContentDomain.TEXT) // For now JavaScript resorting can only be applied for text search && sb.getConfigBool(SwitchboardConstants.SEARCH_JS_RESORT, SwitchboardConstants.SEARCH_JS_RESORT_DEFAULT); // check the search tracker @@ -701,41 +697,6 @@ public class yacysearch { } } - // if a bookmarks-button was hit, create new bookmark entry - if (post != null && post.containsKey("bookmarkref")) { - if (!sb.verifyAuthentication(header)) { - prop.authenticationRequired(); - return prop; - } - //final String bookmarkHash = post.get("bookmarkref", ""); // urlhash - final String urlstr = crypt.simpleDecode(post.get("bookmarkurl")); - if (urlstr != null) { - final Bookmark bmk = sb.bookmarksDB.createorgetBookmark(urlstr, YMarkTables.USER_ADMIN); - if (bmk != null) { - bmk.setProperty(Bookmark.BOOKMARK_QUERY, querystring); - bmk.addTag("/search"); // add to bookmark folder - bmk.addTag("searchresult"); // add tag - String urlhash = post.get("bookmarkref"); - final URIMetadataNode urlentry = indexSegment.fulltext().getMetadata(UTF8.getBytes(urlhash)); - if (urlentry != null && !urlentry.dc_title().isEmpty()) { - bmk.setProperty(Bookmark.BOOKMARK_TITLE, urlentry.dc_title()); - } - sb.bookmarksDB.saveBookmark(bmk); - } - // do the same for YMarks ? - try { - sb.tables.bookmarks.createBookmark( - sb.loader, - urlstr, - ClientIdentification.yacyInternetCrawlerAgent, - YMarkTables.USER_ADMIN, - true, - "searchresult", - "/search"); - } catch (final Throwable e) { } - } - } - // check filters try { Pattern.compile(urlmask); diff --git a/htroot/yacysearchitem.java b/htroot/yacysearchitem.java index 9e1765680..f36055500 100644 --- a/htroot/yacysearchitem.java +++ b/htroot/yacysearchitem.java @@ -104,12 +104,12 @@ public class yacysearchitem { final String eventID = post.get("eventID", ""); final boolean adminAuthenticated = sb.verifyAuthentication(header); - + final UserDB.Entry user = sb.userDB != null ? sb.userDB.getUser(header) : null; final boolean authenticated = adminAuthenticated || user != null; final boolean extendedSearchRights = adminAuthenticated || (user != null && user.hasRight(UserDB.AccessRight.EXTENDED_SEARCH_RIGHT)); - + final int item = post.getInt("item", -1); final RequestHeader.FileType fileType = header.fileType(); @@ -117,14 +117,14 @@ public class yacysearchitem { /* * Access to authentication protected features is explicitely requested here * but no authentication is provided : ask now for authentication. - * Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information + * Wihout this, after timeout of HTTP Digest authentication nonce, browsers no more send authentication information * and as this page is not private, protected features would simply be hidden without asking browser again for authentication. * (see mantis 766 : http://mantis.tokeek.de/view.php?id=766) * */ prop.authenticationRequired(); return prop; } - + // default settings for blank item prop.put("content", "0"); prop.put("rss", "0"); @@ -159,7 +159,7 @@ public class yacysearchitem { final boolean noreferrer = sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_NOREFERRER, SwitchboardConstants.SEARCH_RESULT_NOREFERRER_DEFAULT); long timeout = item == 0 ? 10000 : (theSearch.query.isLocal() ? 1000 : 3000); - + if (theSearch.query.contentdom == Classification.ContentDomain.TEXT || theSearch.query.contentdom == Classification.ContentDomain.ALL) { // text search @@ -183,12 +183,12 @@ public class yacysearchitem { prop.putJSON("content_title-json", result.title()); prop.putHTML("content_showPictures_link", resultUrlstring); prop.put("content_showPictures_authSearch", authenticated); - + /* Add information about the current search navigators to let browser refresh yacysearchtrailer only if needed */ prop.put("content_nav-generation", theSearch.getNavGeneration()); - + //prop.putHTML("content_link", resultUrlstring); - + // START interaction if (sb.getConfigBool("proxyURL.useforresults", false) && sb.getConfigBool("proxyURL", false)) { String modifyURL = resultUrlstring; @@ -218,7 +218,7 @@ public class yacysearchitem { prop.putXML("content_link", resultUrlstring); // putXML for rss } prop.put("content_noreferrer", noreferrer ? 1 : 0); - + // END interaction boolean isAtomFeed = header.get(HeaderFramework.CONNECTION_PROP_EXT, "").equals("atom"); @@ -227,7 +227,7 @@ public class yacysearchitem { DigestURL faviconURL = null; final boolean showFavicon = sb.getConfigBool(SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON, SwitchboardConstants.SEARCH_RESULT_SHOW_FAVICON_DEFAULT); - + if (((fileType == FileType.HTML && showFavicon) || fileType == FileType.JSON) && (resultURL.isHTTP() || resultURL.isHTTPS())) { faviconURL = getFaviconURL(result, new Dimension(16, 16)); @@ -239,7 +239,7 @@ public class yacysearchitem { } prop.putHTML("content_favicon_faviconUrl", processFaviconURL(ImageViewer.hasFullViewingRights(header, sb), faviconURL)); prop.putHTML("content_favicon_urlhash", urlhash); - + if (result.limage() == 0) { if (faviconURL == null) { prop.put("content_image", 0); @@ -257,7 +257,7 @@ public class yacysearchitem { } } - + prop.put("content_urlhash", urlhash); prop.put("content_ranking", Float.toString(result.score())); Date[] events = result.events(); @@ -358,7 +358,7 @@ public class yacysearchitem { selectedExt = ext; break; } else if("pdf".equals(ext)) { - selectedExt = ext; + selectedExt = ext; } else if("xml".equals(ext) && selectedExt == null) { /* Use the XML metadata snapshot in last resort */ selectedExt = ext; @@ -380,7 +380,7 @@ public class yacysearchitem { if (showEvent) prop.put("content_showEvent_date822", isAtomFeed ? ISO8601Formatter.FORMATTER.format(events[0]) : HeaderFramework.formatRFC1123(events[0])); //prop.put("content_ybr", RankingProcess.ybr(result.hash())); prop.putHTML("content_size", Integer.toString(result.filesize())); // we don't use putNUM here because that number shall be usable as sorting key. To print the size, use 'sizename' - prop.putHTML("content_sizename", RSSMessage.sizename(result.filesize())); + prop.putHTML("content_sizename", RSSMessage.sizename(result.filesize())); prop.putHTML("content_host", resultURL.getHost() == null ? "" : resultURL.getHost()); prop.putXML("content_file", resultFileName); // putXML for rss prop.putXML("content_path", resultURL.getPath()); // putXML for rss @@ -431,7 +431,7 @@ public class yacysearchitem { prop.put("content_loc_lat", result.lat()); prop.put("content_loc_lon", result.lon()); } - + final boolean clustersearch = sb.isRobinsonMode() && sb.getConfig(SwitchboardConstants.CLUSTER_MODE, "").equals(SwitchboardConstants.CLUSTER_MODE_PUBLIC_CLUSTER); final boolean indexReceiveGranted = sb.getConfigBool(SwitchboardConstants.INDEX_RECEIVE_ALLOW_SEARCH, true) || clustersearch; boolean p2pmode = sb.peers != null && sb.peers.sizeConnected() > 0 && indexReceiveGranted; @@ -488,7 +488,7 @@ public class yacysearchitem { } /** - * + * * @param prop the target properties * @param theSearch the search event * @param result a result entry @@ -571,7 +571,7 @@ public class yacysearchitem { private static void appendEmbeddedAudio(final URIMetadataNode mainResult, final MultiProtocolURL audioLink, final serverObjects prop, final String propPrefix) { prop.putHTML(propPrefix + "_href", audioLink.toString()); - + /* Add a title to help user distinguish embedded elements of the list */ final String title; if(audioLink.getHost().equals(mainResult.url().getHost())) { @@ -583,11 +583,11 @@ public class yacysearchitem { } prop.putHTML(propPrefix+ "_title", title); } - + /** * Add to the target set, valid URLs from the iterator that are classified as * audio from their file name extension. - * + * * @param linksIter an iterator on URL strings * @param target the target set to fill * @param targetMaxSize the maximum target set size @@ -611,7 +611,7 @@ public class yacysearchitem { * Tries to retrieve favicon url from solr result document, or generates * default favicon URL (i.e. "http://host/favicon.ico") from resultURL and * port. - * + * * @param result * solr document result. Must not be null. * @param preferredSize preferred icon size. If no one matches, most close icon is returned. @@ -656,7 +656,7 @@ public class yacysearchitem { final String iconUrlExt = MultiProtocolURL.getFileExtension(faviconURL.getFileName()); /* Image format ouput for ViewFavicon servlet : default is png, except with gif and svg icons */ final String viewFaviconExt = !iconUrlExt.isEmpty() && ImageViewer.isBrowserRendered(iconUrlExt) ? iconUrlExt : "png"; - + contentFaviconURL.append("ViewFavicon.").append(viewFaviconExt).append("?maxwidth=16&maxheight=16&isStatic=true&quadratic"); if (hasFullViewingRights) { contentFaviconURL.append("&url=").append(faviconURL.toNormalform(true)); @@ -666,7 +666,7 @@ public class yacysearchitem { } return contentFaviconURL.toString(); } - + /** * Add action links reserved to authorized users. All parameters must be non null. * @param sb the main Switchboard instance @@ -683,11 +683,11 @@ public class yacysearchitem { // check if url exists in bookmarks boolean bookmarkexists = sb.bookmarksDB.getBookmark(urlhash) != null; prop.put("content_authorized_bookmark", !bookmarkexists); - + final StringBuilder linkBuilder = QueryParams.navurl(RequestHeader.FileType.HTML, theSearch.query.offset / theSearch.query.itemsPerPage(), theSearch.query, null, false, true); final int baseUrlLength = linkBuilder.length(); - + String encodedURLString; try { encodedURLString = URLEncoder.encode(crypt.simpleEncode(resultUrlstring), StandardCharsets.UTF_8.name()); @@ -695,24 +695,23 @@ public class yacysearchitem { ConcurrentLog.warn("YACY_SEARCH_ITEM", "UTF-8 encoding is not supported!"); encodedURLString = crypt.simpleEncode(resultUrlstring); } - final String bookmarkLink = linkBuilder.append("&bookmarkref=").append(urlhash) - .append("&bookmarkurl=").append(encodedURLString).toString(); + final String bookmarkLink = linkBuilder.append("&bookmarkurl=").append(encodedURLString).toString(); linkBuilder.setLength(baseUrlLength); - + String deleteLink = linkBuilder.append("&deleteref=").append(urlhash).toString(); linkBuilder.setLength(baseUrlLength); - + String recommendLink = linkBuilder.append("&recommendref=").append(urlhash).toString(); linkBuilder.setLength(baseUrlLength); - + prop.put("content_authorized_bookmark_bookmarklink", bookmarkLink); prop.put("content_authorized_recommend_deletelink", deleteLink); prop.put("content_authorized_recommend_recommendlink", recommendLink); - + prop.put("content_authorized_recommend", (sb.peers.newsPool.getSpecific(NewsPool.OUTGOING_DB, NewsPool.CATEGORY_SURFTIPP_ADD, "url", resultUrlstring) == null) ? "1" : "0"); prop.put("content_authorized_urlhash", urlhash); } - + /** * Process search of image type and feed prop object. All parameters must not be null. @@ -762,10 +761,10 @@ public class yacysearchitem { prop.putHTML("content_item_name", shorten(image.imagetext, MAX_NAME_LENGTH)); prop.put("content_item_mimetype", image.mimetype); prop.put("content_item_fileSize", 0); - + String itemWidth = DEFAULT_IMG_WIDTH + "px", itemHeight = DEFAULT_IMG_HEIGHT + "px", itemStyle=""; /* When image content is rendered by browser : - * - set smaller dimension to 100% in order to crop image on other dimension with CSS style 'overflow:hidden' on image container + * - set smaller dimension to 100% in order to crop image on other dimension with CSS style 'overflow:hidden' on image container * - set negative margin top behave like ViewImage which sets an offset when cutting to square */ if (ImageViewer.isBrowserRendered(imageUrlExt)) { if (image.width > image.height) { diff --git a/locales/de.lng b/locales/de.lng index d23b78faf..aeb170039 100644 --- a/locales/de.lng +++ b/locales/de.lng @@ -3386,38 +3386,6 @@ Primary Key==Primärschlüssel "Commit"=="Absenden" #----------------------------- -#File: Table_YMark_p.html -#--------------------------- -Table Viewer==Tabellen Anzeige -YMark Table Administration==YMark Tabellen Administration -Table Editor: showing table==Tabellen Editor: Angezeigte Tabelle -"Edit Selected Row"=="Bearbeite angezeigte Reihe" -"Add a new Row"=="Füge neue Zeile ein" -"Delete Selected Rows"=="Lösche ausgewählte Reihe" -"Delete Table"=="Lösche Tabelle" -"Rebuild Index"=="Index neu aufbauen" -Primary Key==Primärschlüssel ->Row Editor<==>Reihen Editor< -"Commit"=="Eintragen" -Table Selection==Tabellen Auswahl -Select Table:==Wähle Tabelle: -show max. entries==Zeige max. Einträge ->all<==>Alle< -Display columns:==Zeige Spalten: -"load"=="Laden" -Search/Filter Table==Suche/Filter Tabelle -search rows for==Suche Reihen mit -"Search"=="Suche" -#>Tags<==>Tags< ->select a tag<==>Tag auswählen< ->Folders<==>Ordner< ->select a folder<==>Ordner auswählen< ->Import Bookmarks<==>Importiere Lesezeichen< -#Importer:==Importer: -#>XBEL Importer<==>XBEL Importer< -#>Netscape HTML Importer<==>Netscape HTML Importer< -"import"=="Importieren" -#----------------------------- #File: terminal_p.html #--------------------------- diff --git a/locales/es.lng b/locales/es.lng index f44d16bc4..82b9dabcf 100644 --- a/locales/es.lng +++ b/locales/es.lng @@ -1295,12 +1295,6 @@ years<==años< #--------------------------- #----------------------------- -#File: Table_YMark_p.html -#--------------------------- ->Tags<==>Tag< ->Folders<==>Directorios< ->select a folder<==>selecciona un directorio< -#----------------------------- #File: terminal_p.html #--------------------------- diff --git a/locales/it.lng b/locales/it.lng index 3f1d0c911..e767631aa 100644 --- a/locales/it.lng +++ b/locales/it.lng @@ -1274,12 +1274,6 @@ years<==anni< #---------------------------
#-----------------------------
-#File: Table_YMark_p.html
-#---------------------------
->Tags<==>Tag<
->Folders<==>Directory<
->select a folder<==>seleziona una directory<
-#-----------------------------
#File: terminal_p.html
#---------------------------
diff --git a/locales/ja.lng b/locales/ja.lng index 3fa384930..bf8e4ca27 100644 --- a/locales/ja.lng +++ b/locales/ja.lng @@ -1288,13 +1288,6 @@ entries,==エントリー, "Commit"=="コミット" #----------------------------- -#File: Table_YMark_p.html -#--------------------------- -Table Viewer==テーブル ヴューワー -YMark Table Administration==YMark テーブル管理 -Table Editor: showing table==テーブル エディター: テーブルを表示しています -"import"=="取り込み" -#----------------------------- #File: terminal_p.html #--------------------------- diff --git a/locales/master.lng.xlf b/locales/master.lng.xlf index 83d5b1317..c836d9d33 100644 --- a/locales/master.lng.xlf +++ b/locales/master.lng.xlf @@ -8957,85 +8957,6 @@ </body> </file> -<file original="Table_YMark_p.html" source-language="en" datatype="html"> - <body> - <trans-unit id="33445ae4" xml:space="preserve" approved="no" translate="yes"> - <source>Table Viewer</source> - </trans-unit> - <trans-unit id="2fce7e2e" xml:space="preserve" approved="no" translate="yes"> - <source>Table Editor: showing table</source> - </trans-unit> - <trans-unit id="c9308c19" xml:space="preserve" approved="no" translate="yes"> - <source>"Edit Selected Row"</source> - </trans-unit> - <trans-unit id="7890c6a8" xml:space="preserve" approved="no" translate="yes"> - <source>"Add a new Row"</source> - </trans-unit> - <trans-unit id="8be76bb7" xml:space="preserve" approved="no" translate="yes"> - <source>"Delete Selected Rows"</source> - </trans-unit> - <trans-unit id="f14a5d47" xml:space="preserve" approved="no" translate="yes"> - <source>"Delete Table"</source> - </trans-unit> - <trans-unit id="d2f14a17" xml:space="preserve" approved="no" translate="yes"> - <source>"Rebuild Index"</source> - </trans-unit> - <trans-unit id="2f9b9701" xml:space="preserve" approved="no" translate="yes"> - <source>Primary Key</source> - </trans-unit> - <trans-unit id="54c534eb" xml:space="preserve" approved="no" translate="yes"> - <source>>Row Editor<</source> - </trans-unit> - <trans-unit id="67c68b49" xml:space="preserve" approved="no" translate="yes"> - <source>"Commit"</source> - </trans-unit> - <trans-unit id="f6b5bf7a" xml:space="preserve" approved="no" translate="yes"> - <source>Table Selection</source> - </trans-unit> - <trans-unit id="292939d0" xml:space="preserve" approved="no" translate="yes"> - <source>Select Table:</source> - </trans-unit> - <trans-unit id="1979b5fd" xml:space="preserve" approved="no" translate="yes"> - <source>show max. entries</source> - </trans-unit> - <trans-unit id="3976bf9" xml:space="preserve" approved="no" translate="yes"> - <source>>all<</source> - </trans-unit> - <trans-unit id="196729bb" xml:space="preserve" approved="no" translate="yes"> - <source>Display columns:</source> - </trans-unit> - <trans-unit id="402a9d1a" xml:space="preserve" approved="no" translate="yes"> - <source>"load"</source> - </trans-unit> - <trans-unit id="bb440a0d" xml:space="preserve" approved="no" translate="yes"> - <source>Search/Filter Table</source> - </trans-unit> - <trans-unit id="a5365d7a" xml:space="preserve" approved="no" translate="yes"> - <source>search rows for</source> - </trans-unit> - <trans-unit id="a472dcf8" xml:space="preserve" approved="no" translate="yes"> - <source>"Search"</source> - </trans-unit> - <trans-unit id="bfe3ec67" xml:space="preserve" approved="no" translate="yes"> - <source>>select a tag<</source> - </trans-unit> - <trans-unit id="fdf50d75" xml:space="preserve" approved="no" translate="yes"> - <source>>Folders<</source> - </trans-unit> - <trans-unit id="b05a2ee9" xml:space="preserve" approved="no" translate="yes"> - <source>>select a folder<</source> - </trans-unit> - <trans-unit id="607cb8bc" xml:space="preserve" approved="no" translate="yes"> - <source>>Import Bookmarks<</source> - </trans-unit> - <trans-unit id="3eb4f6bb" xml:space="preserve" approved="no" translate="yes"> - <source>"import"</source> - </trans-unit> - <trans-unit id="3a0ad84a" xml:space="preserve" approved="no" translate="yes"> - <source>YMark Table Administration</source> - </trans-unit> - </body> -</file> <file original="Tables_p.html" source-language="en" datatype="html"> <body> @@ -9895,260 +9816,6 @@ </body> </file> -<file original="YMarks.html" source-language="en" datatype="html"> - <body> - <trans-unit id="462ff49" xml:space="preserve" approved="no" translate="yes"> - <source>Login</source> - </trans-unit> - <trans-unit id="567a8783" xml:space="preserve" approved="no" translate="yes"> - <source>TreeView</source> - </trans-unit> - <trans-unit id="2fa67342" xml:space="preserve" approved="no" translate="yes"> - <source>Import Bookmarks</source> - </trans-unit> - <trans-unit id="c06dddb3" xml:space="preserve" approved="no" translate="yes"> - <source>Bookmarks (XBEL)</source> - </trans-unit> - <trans-unit id="c36499fb" xml:space="preserve" approved="no" translate="yes"> - <source>YaCy Bookmarks</source> - </trans-unit> - <trans-unit id="8d4cbaf" xml:space="preserve" approved="no" translate="yes"> - <source>The information that is presented on this page can also be retrieved as XML.</source> - </trans-unit> - <trans-unit id="affac95e" xml:space="preserve" approved="no" translate="yes"> - <source>Click the API icon to see the XML.</source> - </trans-unit> - <trans-unit id="68813072" xml:space="preserve" approved="no" translate="yes"> - <source>To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de/wiki/index.php/Dev:API" target="_blank">API wiki page</a>.</source> - </trans-unit> - <trans-unit id="6340942" xml:space="preserve" approved="no" translate="yes"> - <source>Bookmarks (user: #[user]# size: #[size]#)</source> - </trans-unit> - <trans-unit id="90e1bc5f" xml:space="preserve" approved="no" translate="yes"> - <source>Explorer</source> - </trans-unit> - <trans-unit id="a38e70e7" xml:space="preserve" approved="no" translate="yes"> - <source>Tag Manager</source> - </trans-unit> - <trans-unit id="fe8e7719" xml:space="preserve" approved="no" translate="yes"> - <source>>Import<</source> - </trans-unit> - <trans-unit id="7c8dffd4" xml:space="preserve" approved="no" translate="yes"> - <source>Export</source> - </trans-unit> - <trans-unit id="f3d757fa" xml:space="preserve" approved="no" translate="yes"> - <source>All tag actions are applied to the sub-set of bookmarks defined by this query.</source> - </trans-unit> - <trans-unit id="609b4652" xml:space="preserve" approved="no" translate="yes"> - <source>>Query<</source> - </trans-unit> - <trans-unit id="6945f3d2" xml:space="preserve" approved="no" translate="yes"> - <source>Query Type</source> - </trans-unit> - <trans-unit id="735b1dd8" xml:space="preserve" approved="no" translate="yes"> - <source>Tags (comma seperated)</source> - </trans-unit> - <trans-unit id="d6c7731f" xml:space="preserve" approved="no" translate="yes"> - <source>Tags (regexp)</source> - </trans-unit> - <trans-unit id="b8dd7c44" xml:space="preserve" approved="no" translate="yes"> - <source>Folders (comma seperated)</source> - </trans-unit> - <trans-unit id="3128eb33" xml:space="preserve" approved="no" translate="yes"> - <source>Folders (regexp)</source> - </trans-unit> - <trans-unit id="a2efb800" xml:space="preserve" approved="no" translate="yes"> - <source>Title (regexp)</source> - </trans-unit> - <trans-unit id="fd656dc" xml:space="preserve" approved="no" translate="yes"> - <source>Description (regexp)</source> - </trans-unit> - <trans-unit id="cd6c7fc1" xml:space="preserve" approved="no" translate="yes"> - <source>Enter tags to add (<i>replace with</i>)</source> - </trans-unit> - <trans-unit id="6cc957e" xml:space="preserve" approved="no" translate="yes"> - <source>(comma separated tags)</source> - </trans-unit> - <trans-unit id="9b4ac630" xml:space="preserve" approved="no" translate="yes"> - <source>"Replace"</source> - </trans-unit> - <trans-unit id="4bbf86bc" xml:space="preserve" approved="no" translate="yes"> - <source>Bookmark Importer</source> - </trans-unit> - <trans-unit id="4cf97d83" xml:space="preserve" approved="no" translate="yes"> - <source>If you put in your bookmarks here, you can access them anywhere where you have access to your YaCy peer. Think of it as your 'personal cloud' for bookmarking.</source> - </trans-unit> - <trans-unit id="db4a9a8f" xml:space="preserve" approved="no" translate="yes"> - <source>Surrogate XML</source> - </trans-unit> - <trans-unit id="490c9847" xml:space="preserve" approved="no" translate="yes"> - <source>YaCy White/Black List</source> - </trans-unit> - <trans-unit id="56ec468e" xml:space="preserve" approved="no" translate="yes"> - <source>YaCy Crawl Starts (admin)</source> - </trans-unit> - <trans-unit id="422f4206" xml:space="preserve" approved="no" translate="yes"> - <source>Bookmark file</source> - </trans-unit> - <trans-unit id="86d45375" xml:space="preserve" approved="no" translate="yes"> - <source>Folder settings</source> - </trans-unit> - <trans-unit id="50affd1a" xml:space="preserve" approved="no" translate="yes"> - <source>A folder structure is helpful to organize your bookmarks in a hierarchical way.</source> - </trans-unit> - <trans-unit id="232e07f3" xml:space="preserve" approved="no" translate="yes"> - <source>Source folder</source> - </trans-unit> - <trans-unit id="a6de8dbd" xml:space="preserve" approved="no" translate="yes"> - <source>Target folder</source> - </trans-unit> - <trans-unit id="518d7fa0" xml:space="preserve" approved="no" translate="yes"> - <source>Automatic tagging</source> - </trans-unit> - <trans-unit id="e3dd991d" xml:space="preserve" approved="no" translate="yes"> - <source>Tags are words that are attached to documents as metadata. It is possible to read all the documents and find the attached tags automatically.</source> - </trans-unit> - <trans-unit id="1354f" xml:space="preserve" approved="no" translate="yes"> - <source>Off</source> - </trans-unit> - <trans-unit id="ff932d97" xml:space="preserve" approved="no" translate="yes"> - <source>Only for empty tags</source> - </trans-unit> - <trans-unit id="af0a58d6" xml:space="preserve" approved="no" translate="yes"> - <source>Overwriting existing tags</source> - </trans-unit> - <trans-unit id="6949eadf" xml:space="preserve" approved="no" translate="yes"> - <source>Merging with existing tags</source> - </trans-unit> - <trans-unit id="17ccf465" xml:space="preserve" approved="no" translate="yes"> - <source>Automatic Indexing</source> - </trans-unit> - <trans-unit id="f0838ba6" xml:space="preserve" approved="no" translate="yes"> - <source>While doing the bookmark import, YaCy can push all URLs to the indexing process</source> - </trans-unit> - <trans-unit id="d8c9492f" xml:space="preserve" approved="no" translate="yes"> - <source>No indexing</source> - </trans-unit> - <trans-unit id="87a21d43" xml:space="preserve" approved="no" translate="yes"> - <source>>Index every bookmark entry<</source> - </trans-unit> - <trans-unit id="cb9150c7" xml:space="preserve" approved="no" translate="yes"> - <source>Index every bookmark entry plus all directly linked pages</source> - </trans-unit> - <trans-unit id="99fc7200" xml:space="preserve" approved="no" translate="yes"> - <source>Index all domains from all bookmarks completely</source> - </trans-unit> - <trans-unit id="3c3ce7df" xml:space="preserve" approved="no" translate="yes"> - <source>include all media (image/movie/document) links</source> - </trans-unit> - <trans-unit id="59da6a73" xml:space="preserve" approved="no" translate="yes"> - <source>Add & Edit Bookmark</source> - </trans-unit> - <trans-unit id="553132d1" xml:space="preserve" approved="no" translate="yes"> - <source>Public:</source> - </trans-unit> - <trans-unit id="1d2e7" xml:space="preserve" approved="no" translate="yes"> - <source>yes</source> - </trans-unit> - <trans-unit id="dc1" xml:space="preserve" approved="no" translate="yes"> - <source>no</source> - </trans-unit> - <trans-unit id="95551502" xml:space="preserve" approved="no" translate="yes"> - <source>Title:</source> - </trans-unit> - <trans-unit id="97464fde" xml:space="preserve" approved="no" translate="yes"> - <source>Description:</source> - </trans-unit> - <trans-unit id="9b51dd7" xml:space="preserve" approved="no" translate="yes"> - <source>Folder (/folder/subfolder):</source> - </trans-unit> - <trans-unit id="5883e9e6" xml:space="preserve" approved="no" translate="yes"> - <source>Tags (comma separated):</source> - </trans-unit> - <trans-unit id="246c7d67" xml:space="preserve" approved="no" translate="yes"> - <source>Craw Start</source> - </trans-unit> - <trans-unit id="1aa88728" xml:space="preserve" approved="no" translate="yes"> - <source>>Bookmark<</source> - </trans-unit> - <trans-unit id="f0df21dd" xml:space="preserve" approved="no" translate="yes"> - <source>Bookmarks</source> - </trans-unit> - <trans-unit id="bf7c2a83" xml:space="preserve" approved="no" translate="yes"> - <source>Restrict to start domain</source> - </trans-unit> - <trans-unit id="e505f0" xml:space="preserve" approved="no" translate="yes"> - <source>Restrict to sub-path of given url</source> - </trans-unit> - <trans-unit id="78fcb8be" xml:space="preserve" approved="no" translate="yes"> - <source>Crawling Depth</source> - </trans-unit> - <trans-unit id="fae14957" xml:space="preserve" approved="no" translate="yes"> - <source>bookmark only (0)</source> - </trans-unit> - <trans-unit id="5f1afe18" xml:space="preserve" approved="no" translate="yes"> - <source>shallow crawl (4)</source> - </trans-unit> - <trans-unit id="7fb20b6c" xml:space="preserve" approved="no" translate="yes"> - <source>deep crawl (8)</source> - </trans-unit> - <trans-unit id="153ce9bc" xml:space="preserve" approved="no" translate="yes"> - <source>deeper crawl (16)</source> - </trans-unit> - <trans-unit id="9914048c" xml:space="preserve" approved="no" translate="yes"> - <source>indefinite (99)</source> - </trans-unit> - <trans-unit id="c6f01f79" xml:space="preserve" approved="no" translate="yes"> - <source>Limitations</source> - </trans-unit> - <trans-unit id="3bc1727f" xml:space="preserve" approved="no" translate="yes"> - <source>not more than</source> - </trans-unit> - <trans-unit id="383d52b8" xml:space="preserve" approved="no" translate="yes"> - <source>documents</source> - </trans-unit> - <trans-unit id="28ee1465" xml:space="preserve" approved="no" translate="yes"> - <source>Dynamic URLs</source> - </trans-unit> - <trans-unit id="a96ab043" xml:space="preserve" approved="no" translate="yes"> - <source>allow <a href="http://en.wikipedia.org/wiki/Query_string" target="_blank"></source> - </trans-unit> - <trans-unit id="1172477d" xml:space="preserve" approved="no" translate="yes"> - <source>query-strings</a> (urls with a '?' in the path)</source> - </trans-unit> - <trans-unit id="6ddde09b" xml:space="preserve" approved="no" translate="yes"> - <source>Scheduler</source> - </trans-unit> - <trans-unit id="d2e68607" xml:space="preserve" approved="no" translate="yes"> - <source>run this crawl once</source> - </trans-unit> - <trans-unit id="cc5c83fb" xml:space="preserve" approved="no" translate="yes"> - <source>scheduled, look every</source> - </trans-unit> - <trans-unit id="3f791cdf" xml:space="preserve" approved="no" translate="yes"> - <source>minutes</source> - </trans-unit> - <trans-unit id="5edc70f" xml:space="preserve" approved="no" translate="yes"> - <source>hours</source> - </trans-unit> - <trans-unit id="2ef057" xml:space="preserve" approved="no" translate="yes"> - <source>days</source> - </trans-unit> - <trans-unit id="461591ca" xml:space="preserve" approved="no" translate="yes"> - <source>for new documents automatically.</source> - </trans-unit> - <trans-unit id="884c7a97" xml:space="preserve" approved="no" translate="yes"> - <source>No filter</source> - </trans-unit> - <trans-unit id="26f8a610" xml:space="preserve" approved="no" translate="yes"> - <source>multiple</source> - </trans-unit> - <trans-unit id="a1ed8e9b" xml:space="preserve" approved="no" translate="yes"> - <source>"Import"</source> - </trans-unit> - </body> -</file> - <file original="api/citation.html" source-language="en" datatype="html"> <body> <trans-unit id="4128aad0" xml:space="preserve" approved="no" translate="yes"> diff --git a/locales/ru.lng b/locales/ru.lng index e90ef4f18..8aaf55661 100644 --- a/locales/ru.lng +++ b/locales/ru.lng @@ -3537,136 +3537,6 @@ Primary Key==Первичный ключ "Commit"=="Отправить" #----------------------------- -#File: YMarks.html -#----------------------------- -Login==Логин -TreeView==Вид дерева -Import Bookmarks==Импорт закладок -Bookmarks (XBEL)==Закладки (XBEL) -YaCy Bookmarks==Закладки YaCy -The information that is presented on this page can also be retrieved as XML.==Информация представленная на этой странице, также может быть получена в виде XML. -Click the API icon to see the XML.==Нажмите на иконку API для просмотра XML. -To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de/wiki/index.php/Dev:API" target="_blank">API wiki page</a>.==Для просмотра списка всех API, пожалуйста, посетите <a href="http://www.yacy-websuche.de/wiki/index.php/Dev:API" target="_blank">wiki-страницу API</a>. -Bookmarks (user: #[user]# size: #[size]#)==Закладки (пользователь: #[user]# размер: #[size]#) -Explorer==Проводник -Tag Manager==Менеджер тэгов -Select (multiple) tags ...==Выбрать (несколько) тэгов ->Import<==>Импорт< -Export==Экспорт -All tag actions are applied to the sub-set of bookmarks defined by this query.==Все действия тэгов применяются к наборам закладок, определённых этим запросом. ->help<==>Помощь< ->Query<==>Запрос< -Query Type==Тип запроса -Tags (comma seperated)==Тэги (через запятую) -Tags (regexp)==Тэги (регулярные выражения) -Folders (comma seperated)==Папки (через запятую) -Folders (regexp)==Папки (регулярные выражения) -Title (regexp)==Заголовок (регулярные выражения) -Description (regexp)==Описание (регулярное выражение) -Enter tags to add (<i>replace with</i>)==Введите тэги для добавления (с перезаписью) -(comma separated tags)==(укажите тэги через запятую) -"Replace"=="Заменить" -Select tags to remove ...==Выберите тэги для удаления -Bookmark Importer==Импортёр закладок -If you put in your bookmarks here, you can access them anywhere where you have access to your YaCy peer. Think of it as your 'personal cloud' for bookmarking.==К хранимым здесь закладкам, вы будете иметь доступ из любого места, где вам будет доступен ваш узел. Ваше "персональное облако" закладок. -Surrogate XML==Замещение XML -YaCy White/Black List==Белый/черный список YaCy -YaCy Crawl Starts (admin)==Запустить индексацию ->Crawl Start<==>Запустить индексацию< -Bookmark file==Файл закладок -Folder settings==Настройки папки -A folder structure is helpful to organize your bookmarks in a hierarchical way.==Папки ваших закладок будут огранизованы иерархически. -Source folder==Исходная папка -Target folder==Папка назначения ->Imported Bookmarks==>Импортированные -Automatic tagging==Автоматическое присвоение тэгов -Tags are words that are attached to documents as metadata. It is possible to read all the documents and find the attached tags automatically.==Тэги слов прикрепляются к документам как метаданные, что делает возможным чтение всех документов и поиск прикреплённого тэга -Off==Отключить -Only for empty tags==Только с пустыми тэгами -Overwriting existing tags==Перезаписать существующие тэги -Merging with existing tags==Объединить с существующими тэгами -Automatic Indexing==Автоматическая индексация -While doing the bookmark import, YaCy can push all URLs to the indexing process==Пока производится импорт закладок, YaCy может начать индексирование всех ссылок -No indexing==Нет индексации ->Index every bookmark entry<==>Индекс каждой закладки< -Index every bookmark entry plus all directly linked pages==Индекс всех закладок вместе со всеми ссылающимися страницами -Index all domains from all bookmarks completely==Индекс всех доменов из всех закладок полностью -include all media (image/movie/document) links==Включая все ссылки на медиа-файлы (изображения/кино/документы) -Add & Edit Bookmark==Добавить и редактировать закладку -Public:==Публичная -yes==да -no==нет -#URL:==URL -Title:==Заголовок -Description:==Описание -Folder (/folder/subfolder):==Папка (папка/подпапка) -Tags (comma separated):==Тэги (через запятую) -Craw Start==Начать индексирование ->Bookmark<==>Закладка< -Bookmarks==Закладки -Restrict to start domain==Ограничение начального домена -Restrict to sub-path of given url==Ограничение части указанной ссылки -Crawling Depth==Глубина индексирования -bookmark only (0)==только закладки -shallow crawl (4)==неглубокое индексирование -deep crawl (8)==глубокое индексирование -deeper crawl (16)==очень глубокое индексирование -indefinite (99)==неограниченное -Limitations==Ограничения -not more than==не более чем -documents==документов -Dynamic URLs==Динамические ссылки -allow <a href="http://en.wikipedia.org/wiki/Query_string" target="_blank">==Разрешены <a href="http://en.wikipedia.org/wiki/Query_string" target="_blank"> -query-strings</a> (urls with a '?' in the path)==строки-запросы</a>(ссылки с '?' в пути) -Scheduler==Планировщик -run this crawl once==запустить этот индексатор только один раз -scheduled, look every==запланировано, выполнять каждые -minutes==минут -hours==часов -days==дней -for new documents automatically.==для новых документов автоматически. -Filter:==Фильтр -No filter==Без фильтра -multiple==несколько -Sorry, the function you have requested is not yet available!==Извините, запрошенная вами функция не доступна! -"Import"=="Импорт" -#--------------------------- - - - - -#File: Table_YMark_p.html -#--------------------------- -Table Viewer==Просмотр таблиц -#YMark Table Administration==YMark Tabellen Administration -Table Editor: showing table==Редактор таблицы: показать таблицу -"Edit Selected Row"=="Изменить выбранную строку" -"Add a new Row"=="Добавить новую строку" -"Delete Selected Rows"=="Удалить выбранные строки" -"Delete Table"=="Удалить таблицу" -"Rebuild Index"=="Перестроить индекс" -Primary Key==Первичный ключ ->Row Editor<==>Редактор строк< -"Commit"=="Отправить" -Table Selection==Выбор таблицы -Select Table:==Выбрать таблицу: -show max. entries==Показать максимум значений ->all<==>все< -Display columns:==Показать колонки: -"load"=="загрузка" -Search/Filter Table==Поиск/Фильтр таблицы -search rows for==Найти строки -"Search"=="Искать" -#>Tags<==>Тэги< ->select a tag<==>выбрать тэг< ->Folders<==>Папки< ->select a folder<==>выбрать папку< ->Import Bookmarks<==>Импортировать закладки< -#Importer:==Импортёр: -#>XBEL Importer<==>XBEL импортёр< -#>Netscape HTML Importer<==>Netscape HTML импортёр< -"import"=="импортировать" -#----------------------------- #File: terminal_p.html #--------------------------- diff --git a/locales/uk.lng b/locales/uk.lng index 28e2bb8de..276a61e0a 100644 --- a/locales/uk.lng +++ b/locales/uk.lng @@ -3048,38 +3048,6 @@ Primary Key==Первинний ключ </body>==<script>window.onload = function () {$("td:contains('type')").text('Тип'); $("td:contains('comment')").text('Коментар'); $("td:contains('date_recording')").text('Дата запису'); $("td:contains('date_last_exec')").text('Дата використання'); $("td:contains('apicall_count')").text('Виклики API');}</script></body> #----------------------------- -#File: Table_YMark_p.html -#--------------------------- -Table Viewer==Перегляд таблиць -YMark Table Administration==Керування таблицями YMark -Table Editor: showing table==Редактор таблиць: Показ таблиці -"Edit Selected Row"=="Редагувати вибраний рядок" -"Add a new Row"=="Додати новий рядок" -"Delete Selected Rows"=="Видалити вибрані рядки" -"Delete Table"=="Видалити таблицю" -"Rebuild Index"=="Перебудувати індекс" -Primary Key==Основний ключ ->Row Editor<==>Редактор рядка< -"Commit"=="Здійснити" -Table Selection==Вибір таблиці -Select Table:==Виберіть таблицю: -show max. entries==Показано найбільше записів ->all<==>Всі< -Display columns:==Відображення стовпчиків: -"load"=="Завантажити" -Search/Filter Table==Пошук/Фільтр таблиці -search rows for==Шукати рядки з -"Search"=="Пошук" ->Tags<==>Ключі< ->select a tag<==>Вибрати ключ< ->Folders<==>Папки< ->select a folder<==>Виберіть папку< ->Import Bookmarks<==>Імпортувати закладки< -Importer:==Імпортувач: ->XBEL Importer<==>Імпортувач XBEL< ->Netscape HTML Importer<==>Імпортувач Netscape HTML< -"import"=="Імпортувати" -#----------------------------- #File: terminal_p.html #--------------------------- @@ -4001,113 +3969,6 @@ Server Access Grid==Сітка доступу до сервера This images shows incoming connections to your YaCy peer and outgoing connections from your peer to other peers and web servers==Ці зображення показують вхідні з’єднання до вашого вузла і вихідні з’єднання до інших вузлів та веб-серверів #----------------------------- -#File: YMarks.html -#--------------------------- ->YaCy Bookmarks<==>Закладки YaCy< ->Explorer==>Огляд -The information that is presented on this page can also be retrieved as XML.==Інформацію на цій сторінці також можна отримати в форматі XML. -Click the API icon to see the XML.==Натисніть значок API для відображення XML. -To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de/wiki/index.php/Dev:API">API wiki page</a>.==Для перегляду списку всіх API, будь-ласка, відвідайте <a href="http://www.yacy-websuche.de/wiki/index.php/Dev:API">сторінку API у Wiki</a>. -#>Tag Manager==>Ключові слова ->Tag Manager==>Ключі ->Import<==>Імпорт< -#All tag actions are applied to the sub-set of bookmarks defined by this query.== ->Query==>Запит ->Query Type==>Тип запиту ->Tags (comma seperated)==>Ключі (через кому) ->Tags (regexp)==>Ключі (рег.вираз) ->Folders (comma seperated)==>Папки (через кому) ->Folders (regexp)==>Папки (рег.вираз) ->Title (regexp)==>Заголовок (рег.вираз) ->Description (regexp)==>Опис (рег.вираз) -#Enter tags to add (<i>replace with</i>)== ->(comma separated tags)==>(ключі через кому) -#>Bookmark Importer==>Імпортувач закладок ->Bookmark Importer==>Імпорт закладок -> Netscape HTML==>Netscape HTML -> Firefox JSON==>Firefox JSON -> XBEL==>XBEL -> DMOZ XML==>DMOZ XML ->YaCy White/Black List==>Ч/Б список YaCy ->YaCy Crawl Starts (admin)==>Запуски сканування (адм.) ->Bookmark file==>Файл закладок -#>Folder settings==>Налаштування папок ->Folder settings==>Настройки папок ->Source folder==>Папка-джерело ->Target folder==>Кінцева папка -#>Automatic tagging==>Автоматичні закладки ->Automatic tagging==>Самозакладки ->Off==>Вимкнені ->Only for empty tags==>Тільки порожні ключі -> Overwriting existing tags==> Перезаписувати ключі -> Merging with existing tags==> Об’єднувати ключі -#>URL:==>URL: ->Title:==>Заголовок: ->Description:==>Опис: ->Folder (/folder/subfolder):==>Папка (/папка/підпапка) ->Tags (comma separated):==>Ключі (через кому) ->Public:==>Загальнодоступна ->yes<==>так< ->no<==>ні< ->Site==>Сайт ->Scheduler==>Планувальник ->run this crawl once==>запистити це сканування один раз ->scheduled, look every==>запланувати, див. кожні ->minutes<==>хвилин< ->hours<==>годин< ->days<==>днів< -for new documents automatically.==для нових документів автоматично. ->Path==>Шлях ->load all files in domain==>завантажувати всі файли в домені ->load only files in a sub-path of given url==>завантажуати тільки файли з підшляху даного посилання ->Limitations==>Обмеження -> not more than==> не більше, ніж - documents== документів ->Dynamic URLs==>Мінливі URL -query-strings</a> (urls with a '?' in the path)==рядки-запити</a> (посилання з "?" в шляху) -> allow <a href="http://en.wikipedia.org/wiki/Query_string">==> дозволити <a href="http://en.wikipedia.org/wiki/Query_string"> -"Add & Edit Bookmark"==Додати/Змінити закладку -"Import"=="Імпорт" -> Only for empty/unsorted tags==> Тільки для порожніх/несортованих ключів -"Run"=="Записк" -"Add"=="Додати" ->Add==>Додати -"Delete"=="Видалити" ->Delete==>Видалити -"Edit"=="Редагувати" ->Edit==>Редагувати -"Help"=="Допомога" ->Help==>Допомога ->TreeView==>Деревовидний вигляд ->Import Bookmarks==>Імпортувати закладки ->Bookmarks (XBEL)==>Закладки (XBEL) -#"API"=="API" -#Мабуть, краще залишити як є, для сумісності з іншими мовами та попередніми налаштуваннями -#"/Imported Bookmarks"=="/Імпортовані закладки" -"Replace"=="Замінити" ->Bookmarks<==>Закладки< -A folder structure is helpful to organize your bookmarks in a hierarchical way==Папки корисні для розміщення закладок в ієрархічному порядку -If you put in your bookmarks here, you can access them anywhere where you have access to your YaCy peer==Після розміщення в закладки тут, ви можете отримати до них доступ скрізь, де у вас є доступ до вашого вузла YaCy -Think of it as your 'personal cloud' for bookmarking==Це як ваша "особиста хмара" для закладок -Tags are words that are attached to documents as metadata==Ключі це слова, які додаються до документів у вигляді метаданих -It is possible to read all the documents and find the attached tags automatically==Можливо автоматично читати всі документи і знаходити приєднані ключі -While doing the bookmark import, YaCy can push all URLs to the indexing process==При імпортуванні закладок, YaCy може помістити всі URL-адреси на індексування -alt="help"==alt="допомога" -#тобто title="import" -"import"=="Імпортувач закладок" -"tags"=="Ключові слова" -"navigation"=="Огляд" -#>Automatic Indexing==>Автоіндексування ->Automatic Indexing==>Самоіндексування ->No indexing==>Без індексування ->Index every bookmark entry==>Кожну закладку ->Index every bookmark entry plus all directly linked pages==>Кожну закладку + прямо з’єднані сторінки ->Index all domains from all bookmarks completely==>Всі домени з усіх закладок повністю ->include all media (image/movie/document) links==>включити всі медіа (image/movie/document) -title="help"==title="допомога" -#Портрібно дозволити локалізацію js як всіх інших і видалити цей трюк -</body>==<script>window.onload = function () {$('div.ui-multiselect-filter').replaceWith('<div class="ui-multiselect-filter" style="margin-right: 5px;"><div class="ui-multiselect-filter" style="margin: 0px; margin-top: 3px; vertical-align: middle;">Фільтр:</div><input placeholder="Введіть ключ" type="search"></div>'); $('.nBtn,.srtd').attr( 'title',"Показати/Приховати стовпчики"); $("span:contains('Select (multiple) tags ...')").text('Виберіть (кілька) ключів'); $("span:contains('Select tags to remove ...')").text('Ключі для видалення ...'); $("span.bookmark").text('Додати'); $("span.edit").text('Редагувати'); $("span.delete").text('Видалити'); $("span.crawl").text('Сканувати'); $("span.calendar").text('Запланувати'); $("span.help").text('Допомога'); $("div.sDiv2").html('<div class="sDiv2">Знайти <input value=".*" size="30" name="q" class="qsbox" type="text"> <select name="qtype"><option value="">Текст повністю (regexp) </option><option value="_tags">Ключі (через кому) </option><option value="tags">Ключі (regexp) </option><option value="_folder">Папки (через кому) </option><option value="folders">Папки (regexp) </option><option value="title">Заголовок (regexp) </option><option value="desc">Опис (regexp) </option></select></div>');}</script></body> -#----------------------------- ### Subdirectory jquery/js ### #File: jquery/js/jquery.multiselect.filter.min.js @@ -4115,18 +3976,6 @@ title="help"==title="допомога" "Enter keywords"=="Введіть ключ" #----------------------------- -#File: js/yacy-ymarks.js -#--------------------------- -"Hide/Show Columns"=="Показати/Приховати стовпчики" -Select (multiple) tags ...==Виберіть (кілька) ключів -Select tags to remove ...==Ключі для видалення ... ->Add<==>Додати< ->Edit<==>Редагувати< ->Delete<==>Видалити< ->Crawl<==>Сканувати< ->Schedule<==>Запланувати< -#>XBEL<==>XBEL< ->Help<==>Допомога< #Hash==Хеш #Public==Загальнодоступна diff --git a/locales/zh.lng b/locales/zh.lng index fa9fbeabe..27af5813d 100644 --- a/locales/zh.lng +++ b/locales/zh.lng @@ -3388,38 +3388,6 @@ To see a list of all APIs, please visit the <a href="http://www.yacy-websuche.de >robots.txt table<==>爬虫协议列表< #----------------------------- -#File: Table_YMark_p.html -#--------------------------- -Table Viewer==表格查看 -YMark Table Administration==YMark表格管理 -Table Editor: showing table==表格编辑器: 显示表格 -"Edit Selected Row"=="编辑选中行" -"Add a new Row"=="添加新行" -"Delete Selected Rows"=="删除选中行" -"Delete Table"=="删除表格" -"Rebuild Index"=="重建索引" -Primary Key==主键 ->Row Editor<==>行编辑器< -"Commit"=="备注" -Table Selection==选择表格 -Select Table:==选择表格: -show max. entries==显示最多条目 ->all<==>所有< -Display columns:==显示列: -"load"=="载入" -Search/Filter Table==搜索/过滤表格 -search rows for==搜索 -"Search"=="搜索" -#>Tags<==>Tags< ->select a tag<==>选择标签< ->Folders<==>目录< ->select a folder<==>选择目录< ->Import Bookmarks<==>导入书签< -#Importer:==Importer: -#>XBEL Importer<==>XBEL Importer< -#>Netscape HTML Importer<==>Netscape HTML Importer< -"import"=="导入" -#----------------------------- ### This Tables section is removed in current SVN Versions #File: Tables_p.html @@ -3740,22 +3708,6 @@ This tag creates links to external websites.==此标记创建外部网站链接. alt text==文本备案 #----------------------------- -#File: YMarks.html -#--------------------------- -"Import"=="导入" -documents=="文件" -days==天 -hours==小时 -minutes==分钟 -for new documents automatically==自动地对新文件 -run this crawl once==爬取一次 ->Query<==>查询< -Query Type==查询类型 ->Import<==>导入< -Tag Manager==标签管理器 -Bookmarks (user: #[user]# size: #[size]#)==书签(用户: #[user]# 大小: #[size]#) -"Replace"=="替换" -#----------------------------- #File: api/citation.html #--------------------------- diff --git a/source/net/yacy/data/WorkTables.java b/source/net/yacy/data/WorkTables.java index 2405ed206..679c33057 100644 --- a/source/net/yacy/data/WorkTables.java +++ b/source/net/yacy/data/WorkTables.java @@ -56,7 +56,6 @@ import net.yacy.cora.protocol.http.HTTPClient; import net.yacy.cora.storage.HandleSet; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; -import net.yacy.data.ymark.YMarkTables; import net.yacy.kelondro.blob.Tables; import net.yacy.kelondro.data.word.WordReference; import net.yacy.kelondro.rwi.IndexCell; @@ -84,29 +83,27 @@ public class WorkTables extends Tables { public final static String TABLE_API_COL_APICALL_SCHEDULE_TIME = "apicall_schedule_time"; // factor for SCHEULE_UNIT time units public final static String TABLE_API_COL_APICALL_SCHEDULE_UNIT = "apicall_schedule_unit"; // may be 'minutes', 'hours', 'days' public final static String TABLE_API_COL_APICALL_EVENT_KIND = "apicall_event_kind"; // - public final static String TABLE_API_COL_APICALL_EVENT_ACTION = "apicall_event_action"; // + public final static String TABLE_API_COL_APICALL_EVENT_ACTION = "apicall_event_action"; // public final static String TABLE_ROBOTS_NAME = "robots"; public final static String TABLE_ACTIVECRAWLS_NAME = "crawljobsActive"; public final static String TABLE_PASSIVECRAWLS_NAME = "crawljobsPassive"; - public YMarkTables bookmarks; public WorkTables(final File workPath) { super(workPath, 12); - this.bookmarks = new YMarkTables(this); } - + /** - * + * * @param post the api call eventual request parameters. * @param servletName the name of the servlet. Must not be null. * @return the API URL to be recorded, formatted to include request parameters as URL query parameters */ public static String generateRecordedURL(final serverObjects post, final String servletName) { - /* Before API URL serialization, we set any eventual transaction token value to empty : - * this will later help identify a new valid transaction token will be necessary, + /* Before API URL serialization, we set any eventual transaction token value to empty : + * this will later help identify a new valid transaction token will be necessary, * but prevents revealing it in the URL displayed in the process scheduler and prevents storing an outdated value */ final String transactionToken; if(post != null) { @@ -117,13 +114,13 @@ public class WorkTables extends Tables { if(transactionToken != null && post != null) { post.put(TransactionManager.TRANSACTION_TOKEN_PARAM, ""); } - + // generate the apicall url - without the apicall attributes String apiurl = "/" + servletName; if(post != null) { apiurl += "?" + post.toString(); } - + /* Now restore the eventual transaction token to prevent side effects on the post object eventually still used by the caller */ if(post != null) { if(transactionToken != null) { @@ -132,10 +129,10 @@ public class WorkTables extends Tables { post.remove(TransactionManager.TRANSACTION_TOKEN_PARAM); } } - + return apiurl; } - + /** * @param servletName the servlet name used to identify the API when the call is recorded. * @param post Servlet request parameters. Must not be null. @@ -199,7 +196,7 @@ public class WorkTables extends Tables { public byte[] recordAPICall(final serverObjects post, final String servletName, final String type, final String comment) { // remove the apicall attributes from the post object String[] pks = post.remove(TABLE_API_COL_APICALL_PK); - + byte[] pk = pks == null ? null : UTF8.getBytes(pks[0]); // generate the apicall url - without the apicall attributes @@ -275,25 +272,25 @@ public class WorkTables extends Tables { if (unit.equals("minutes") && time < 10) time = 10; } - - /* Before API URL serialization, we set any eventual transaction token value to empty : - * this will later help identify a new valid transaction token will be necessary, + + /* Before API URL serialization, we set any eventual transaction token value to empty : + * this will later help identify a new valid transaction token will be necessary, * but without revealing it in the URL displayed in the process scheduler and storing an invalid value */ final String transactionToken = post.get(TransactionManager.TRANSACTION_TOKEN_PARAM); if(transactionToken != null) { post.put(TransactionManager.TRANSACTION_TOKEN_PARAM, ""); } - + // generate the apicall url - without the apicall attributes final String apiurl = /*"http://localhost:" + getConfig("port", "8090") +*/ "/" + servletName + "?" + post.toString(); - + /* Now restore the eventual transaction token to prevent side effects on the post object eventually still used by the caller */ if(transactionToken != null) { post.put(TransactionManager.TRANSACTION_TOKEN_PARAM, transactionToken); } else { post.remove(TransactionManager.TRANSACTION_TOKEN_PARAM); } - + byte[] pk = null; // insert entry try { @@ -360,7 +357,7 @@ public class WorkTables extends Tables { try { client.GETbytes(url, username, pass, false); // use GETbytes(MultiProtocolURL,..) form to allow url in parameter (&url=path% if(client.getStatusCode() == HttpStatus.SC_METHOD_NOT_ALLOWED) { - /* GET method not allowed (HTTP 450 status) : this may be an old API entry, + /* GET method not allowed (HTTP 450 status) : this may be an old API entry, * now restricted to HTTP POST and requiring a transaction token. We try now with POST. */ execPostAPICall(host, port, username, pass, client, l, url, true); } else { @@ -391,7 +388,7 @@ public class WorkTables extends Tables { * @throws MalformedURLException when the HTTP POST url could not be derived from apiURL */ private void execPostAPICall(String host, int port, final String username, final String pass, - final HTTPClient client, final LinkedHashMap<String, Integer> results, + final HTTPClient client, final LinkedHashMap<String, Integer> results, final MultiProtocolURL apiURL, final boolean isTokenProtectedAPI) throws MalformedURLException { Map<String, ContentBody> post = new HashMap<>(); for (Map.Entry<String, String> a: apiURL.getAttributes().entrySet()) { @@ -399,13 +396,13 @@ public class WorkTables extends Tables { } final MultiProtocolURL url = new MultiProtocolURL("http", host, port, apiURL.getPath()); - + try { if (isTokenProtectedAPI) { // Eventually acquire first a new valid transaction token before posting data client.GETbytes(url, username, pass, false); if (client.getStatusCode() != HttpStatus.SC_OK) { - /* Do not fail immediately, the token may be no more necessary on this API : + /* Do not fail immediately, the token may be no more necessary on this API : * let's log a warning but try anyway the POST call that will eventually reject the request */ ConcurrentLog.warn("APICALL", "Could not retrieve a transaction token for " + apiURL.toNormalform(true)); } else { @@ -426,20 +423,20 @@ public class WorkTables extends Tables { UTF8.StringBody(transactionTokenHeader.getValue())); } } - + } - + client.POSTbytes(url, "localhost", post, username, pass, false, false); - + results.put(apiURL.toNormalform(true), client.getStatusCode()); } catch (final IOException e) { ConcurrentLog.logException(e); results.put(apiURL.toNormalform(true), -1); } } - + /** - * Executes an HTTP GET API call + * Executes an HTTP GET API call * @param host target host name * @param port target port * @param path target path @@ -480,7 +477,7 @@ public class WorkTables extends Tables { final static long hour = 1000L * 60L * 60L; final static long day = hour * 24L; - + /** * calculate the execution time in a api call table based on given scheduling time and last execution time * @param row the database row in the api table @@ -491,7 +488,7 @@ public class WorkTables extends Tables { date = update ? row.get(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, date) : row.get(WorkTables.TABLE_API_COL_DATE_LAST_EXEC, date); if (date == null) return; long d = 0; - + final String kind = row.get(WorkTables.TABLE_API_COL_APICALL_EVENT_KIND, "off"); if ("off".equals(kind)) { int time = row.get(WorkTables.TABLE_API_COL_APICALL_SCHEDULE_TIME, -1); diff --git a/source/net/yacy/data/ymark/MonitoredReader.java b/source/net/yacy/data/ymark/MonitoredReader.java deleted file mode 100644 index 86a98ebfb..000000000 --- a/source/net/yacy/data/ymark/MonitoredReader.java +++ /dev/null @@ -1,102 +0,0 @@ -package net.yacy.data.ymark; - -import java.io.FilterReader; -import java.io.IOException; -import java.io.Reader; -import java.nio.CharBuffer; - -import javax.swing.event.ChangeEvent; -import javax.swing.event.ChangeListener; - -/** - * This class monitors the read progress - * - */ -public class MonitoredReader extends FilterReader { - private volatile long mark = 0; - private volatile long location = 0; - private final int threshold; - private final long maxProgress; - private long lastTriggeredLocation = 0; - private ChangeListener listener = null; - - public MonitoredReader(Reader in, int threshold, long maxProgress) { - super(in); - this.threshold = threshold; - this.maxProgress = maxProgress; - } - - public void addChangeListener(ChangeListener l) { - this.listener = l; - } - - protected void triggerChanged(final long location) { - if ( threshold > 0 && Math.abs( location-lastTriggeredLocation ) < threshold ) - return; - lastTriggeredLocation = location; - if (listener == null) - return; - listener.stateChanged(new ChangeEvent(this)); - } - - public long getProgress() { - return this.location; - } - - public long maxProgress() { - return this.maxProgress; - } - - @Override - public int read() throws IOException { - final int i = super.read(); - if ( i != -1 ) - triggerChanged(location++); - return i; - } - - @Override - public int read(char[] cbuf, int off, int len) throws IOException { - final int i = super.read(cbuf, off, len); - if ( i != -1 ) - triggerChanged(location+=i); - return i; - } - - @Override - public int read(char[] cbuf) throws IOException { - final int i = super.read(cbuf); - if ( i != -1 ) - triggerChanged(location+=i); - return i; - } - - @Override - public int read(CharBuffer target) throws IOException { - final int i = super.read(target); - if ( i != -1 ) - triggerChanged(location+=i); - return i; - } - - @Override - public long skip(long n) throws IOException { - final long i = super.skip(n); - if ( i != -1 ) - triggerChanged(location+=i); - return i; - } - - @Override - public synchronized void mark(int readlimit) throws IOException { - super.mark(readlimit); - mark = location; - } - - @Override - public synchronized void reset() throws IOException { - super.reset(); - if ( location != mark ) - triggerChanged(location = mark); - } -} diff --git a/source/net/yacy/data/ymark/TablesRowComparator.java b/source/net/yacy/data/ymark/TablesRowComparator.java deleted file mode 100644 index 5bae5a61d..000000000 --- a/source/net/yacy/data/ymark/TablesRowComparator.java +++ /dev/null @@ -1,35 +0,0 @@ -package net.yacy.data.ymark; - -import java.util.Comparator; - -import net.yacy.cora.document.encoding.UTF8; -import net.yacy.kelondro.blob.Tables; - -public class TablesRowComparator implements Comparator<Tables.Row> { - private String sortname; - private boolean desc; - - public TablesRowComparator(final String sortname, final String sortorder) { - setSortName(sortname); - if(sortorder.equals("desc")) - this.desc = true; - else - this.desc = false; - } - - public void setSortName(final String sortname) { - this.sortname = sortname; - } - - @Override - public int compare(Tables.Row row0, Tables.Row row1) { - if(row0 != null && row1 != null) { - if(row0.containsKey(this.sortname) && row1.containsKey(this.sortname)) { - String name1 = UTF8.String(row0.get(this.sortname)).toLowerCase(); - String name2 = UTF8.String(row1.get(this.sortname)).toLowerCase(); - return (this.desc) ? name2.compareTo(name1) : name1.compareTo(name2); - } - } - return 0; - } -}
\ No newline at end of file diff --git a/source/net/yacy/data/ymark/YMarkAutoTagger.java b/source/net/yacy/data/ymark/YMarkAutoTagger.java deleted file mode 100644 index df5a2939d..000000000 --- a/source/net/yacy/data/ymark/YMarkAutoTagger.java +++ /dev/null @@ -1,284 +0,0 @@ -package net.yacy.data.ymark; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Iterator; -import java.util.Map; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.concurrent.ArrayBlockingQueue; - -import net.yacy.cora.document.id.DigestURL; -import net.yacy.cora.document.id.MultiProtocolURL; -import net.yacy.cora.federate.yacy.CacheStrategy; -import net.yacy.cora.protocol.ClientIdentification; -import net.yacy.cora.util.CommonPattern; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.crawler.retrieval.Response; -import net.yacy.document.Condenser; -import net.yacy.document.Document; -import net.yacy.document.LibraryProvider; -import net.yacy.document.Parser.Failure; -import net.yacy.document.SentenceReader; -import net.yacy.document.WordTokenizer; -import net.yacy.kelondro.data.word.Word; -import net.yacy.repository.LoaderDispatcher; - -public class YMarkAutoTagger implements Runnable, Thread.UncaughtExceptionHandler { - - private static final String EMPTY_STRING = new String(); - - - public final static String SPACE = " "; - public final static String POISON = ""; - public final static HashSet<String> stopwords = new HashSet<String>(Arrays.asList(".", "!", "?", "nbsp", "uuml", "ouml", "auml", "amp", "quot", "laquo", "raquo", - "and", "with", "the", "gt", "lt")); - - - private final ArrayBlockingQueue<String> bmkQueue; - private final YMarkTables ymarks; - private final String bmk_user; - private final LoaderDispatcher loader; - - private final boolean merge; - - public YMarkAutoTagger(final ArrayBlockingQueue<String> bmkQueue, final LoaderDispatcher loader, final YMarkTables ymarks, final String bmk_user, final boolean merge) { - this.bmkQueue = bmkQueue; - this.ymarks = ymarks; - this.bmk_user = bmk_user; - this.loader = loader; - this.merge = merge; - } - - public YMarkAutoTagger(final LoaderDispatcher loader, final YMarkTables ymarks, final String bmk_user) { - this.bmkQueue = new ArrayBlockingQueue<String>(1); - this.ymarks = ymarks; - this.bmk_user = bmk_user; - this.loader = loader; - this.merge = true; - } - - private static Document loadDocument(final String url, final LoaderDispatcher loader, ClientIdentification.Agent agent) throws IOException { - DigestURL uri; - Response response; - try { - uri = new DigestURL(url); - } catch (final MalformedURLException e) { - ConcurrentLog.warn(YMarkTables.BOOKMARKS_LOG, "loadDocument failed due to malformed url: "+url); - return null; - } - response = loader.load(loader.request(uri, true, false), CacheStrategy.IFEXIST, Integer.MAX_VALUE, null, agent); - try { - return Document.mergeDocuments(response.url(), response.getMimeType(), response.parse()); - } catch (final Failure e) { - ConcurrentLog.warn(YMarkTables.BOOKMARKS_LOG, "loadDocument failed due to a parser failure for url: "+url); - return null; - } - } - - public static String autoTag(final Document document, final int max, final TreeMap<String, YMarkTag> tags) { - final TreeSet<YMarkTag> topwords = new TreeSet<YMarkTag>(); - StringBuilder token; - - if(document == null) { - return EMPTY_STRING; - } - - //get words from document - final Map<String, Word> words = new Condenser(document, null, true, true, LibraryProvider.dymLib, false, false, 0).words(); - - // generate potential tags from document title, description and subject - final int bufferSize = document.dc_title().length() + document.dc_description().length + document.dc_subject(' ').length() + 32; - final StringBuilder buffer = new StringBuilder(bufferSize); - final StringBuilder pwords = new StringBuilder(1000); - buffer.append(document.dc_title().toLowerCase()); - for (String s:document.dc_description()) buffer.append(s.toLowerCase()); - buffer.append(document.dc_subject(' ').toLowerCase()); - int score = 0; - - // get phrases - final TreeMap<String, YMarkTag> phrases = getPhrases(document, 2); - phrases.putAll(getPhrases(document, 3)); - final Iterator<String> iter = phrases.keySet().iterator(); - while(iter.hasNext()) { - score = 10; - final String phrase = iter.next(); - if(phrases.get(phrase).size() > 3 && phrases.get(phrase).size() < 10) { - score = phrases.get(phrase).size() * CommonPattern.SPACE.split(phrase).length * 20; - } - if(isDigitSpace(phrase)) { - score = 10; - } - if(phrases.get(phrase).size() > 2 && buffer.indexOf(phrase) > 1) { - score = score * 10; - } - if (tags.containsKey(phrase)) { - score = score * 20; - } - topwords.add(new YMarkTag(phrase, score)); - pwords.append(phrase); - pwords.append(' '); - } - - // loop through potential tag and rank them - WordTokenizer tokens = new WordTokenizer(new SentenceReader(buffer.toString()), LibraryProvider.dymLib); - try { - while (tokens.hasMoreElements()) { - score = 0; - token = tokens.nextElement(); - - // check if the token appears in the text - if (words.containsKey(token.toString())) { - final Word word = words.get(token.toString()); - // token appears in text and matches an existing bookmark tag - if (tags.containsKey(token.toString())) { - score = word.occurrences() * tags.get(token.toString()).size() * 200; - } - // token appears in text and has more than 3 characters - else if (token.length()>3) { - score = word.occurrences() * 100; - } - // if token is already part of a phrase, reduce score - if(pwords.toString().indexOf(token.toString())>1) { - score = score / 3; - } - topwords.add(new YMarkTag(token.toString(), score)); - } - } - } finally { - tokens.close(); - tokens = null; - } - score = 0; - buffer.setLength(0); - for(final YMarkTag tag : topwords) { - if(score < max) { - if(tag.size() > 100) { - buffer.append(tag.name()); - buffer.append(YMarkUtil.TAGS_SEPARATOR); - score++; - } - } else { - break; - } - } - final String clean = YMarkUtil.cleanTagsString(buffer.toString()); - if(clean.equals(YMarkEntry.BOOKMARK.TAGS.deflt())) { - return MultiProtocolURL.getFileExtension(document.dc_source().getFileName()); - } - return clean; - } - - private static TreeMap<String, YMarkTag> getPhrases(final Document document, final int size) { - final TreeMap<String, YMarkTag> phrases = new TreeMap<String, YMarkTag>(); - final StringBuilder phrase = new StringBuilder(128); - WordTokenizer tokens = new WordTokenizer(new SentenceReader(document.getTextString()), LibraryProvider.dymLib); - try { - StringBuilder token; - int count = 0; - - // loop through text - while(tokens.hasMoreElements()) { - - token = tokens.nextElement(); - if(stopwords.contains(token.toString()) || isDigitSpace(token.toString())) - continue; - - // if we have a full phrase, delete the first token - count++; - if(count > size) - phrase.delete(0, phrase.indexOf(SPACE)+1); - - // append new token - if(phrase.length() > 1) - phrase.append(SPACE); - phrase.append(token); - - if(count >= size) { // make sure we really have a phrase - if(phrases.containsKey(phrase.toString())) { - phrases.get(phrase.toString()).inc(); - } else { - phrases.put(phrase.toString(), new YMarkTag(phrase.toString())); - } - } - } - - return phrases; - } finally { - tokens.close(); - tokens = null; - } - } - - public static String autoTag(final String url, final LoaderDispatcher loader, ClientIdentification.Agent agent, final int max, final TreeMap<String, YMarkTag> tags) { - Document document = null; - String exception = "/IOExceptions"; - try { - document = loadDocument(url, loader, agent); - } catch (final IOException e) { - exception = e.getMessage(); - int start = exception.indexOf('\'')+9; - int end = exception.indexOf('\'', start); - if(start >= 0 && end > 0 && start < exception.length() && end < exception.length()) - exception = "/IOExceptions/" + exception.substring(start, end); - } - return (document != null) ? autoTag(document, max, tags) : exception; - } - - public static boolean isDigitSpace(String str) { - if (str == null) { - return false; - } - int sz = str.length(); - for (int i = 0; i < sz; i++) { - if ((Character.isDigit(str.charAt(i)) == false) && (str.charAt(i) != ' ')) { - return false; - } - } - return true; - } - - @Override - public void run() { - Thread.currentThread().setUncaughtExceptionHandler(this); - String url = null; - String tagString; - Iterator<String> tit; - try { - final TreeMap<String, YMarkTag> tags = this.ymarks.getTags(this.bmk_user); - while((url = this.bmkQueue.take()) != POISON) { - tagString = autoTag(url, this.loader, ClientIdentification.yacyInternetCrawlerAgent, 5, tags); - if (tagString.startsWith("/IOExceptions")) { - this.ymarks.addFolder(this.bmk_user, url, tagString); - tagString = ""; - } - // update tags - this.ymarks.addTags(this.bmk_user, url, tagString, this.merge); - - // update tags - tit = YMarkUtil.keysStringToSet(tagString).iterator(); - while(tit.hasNext()) { - final String tag = tit.next(); - if(tags.containsKey(tag)) { - tags.get(tag).inc(); - } else { - tags.put(tag, new YMarkTag(tag)); - } - } - } - } catch (final InterruptedException e) { - ConcurrentLog.logException(e); - } catch (final IOException e) { - ConcurrentLog.warn(YMarkTables.BOOKMARKS_LOG.toString(), "autoTagger - IOException for URL: "+url); - } finally { - } - } - - @Override - public void uncaughtException(final Thread t, final Throwable e) { - ConcurrentLog.warn(YMarkTables.BOOKMARKS_LOG, "I caught an uncaughtException in thread "+t.getName()); - ConcurrentLog.logException(e); - } -} diff --git a/source/net/yacy/data/ymark/YMarkCrawlStart.java b/source/net/yacy/data/ymark/YMarkCrawlStart.java deleted file mode 100644 index a2aa7d41e..000000000 --- a/source/net/yacy/data/ymark/YMarkCrawlStart.java +++ /dev/null @@ -1,205 +0,0 @@ -// YMarkCrawlStart.java -// (C) 2012 by Stefan F��rster, sof@gmx.de, Norderstedt, Germany -// first published 2011 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate: 2011-03-09 13:50:39 +0100 (Mi, 09 Mrz 2011) $ -// $LastChangedRevision: 7574 $ -// $LastChangedBy: apfelmaennchen $ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.data.ymark; - -import java.io.IOException; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.regex.Pattern; - -import net.yacy.cora.document.encoding.UTF8; -import net.yacy.cora.document.id.DigestURL; -import net.yacy.cora.federate.yacy.CacheStrategy; -import net.yacy.cora.protocol.ClientIdentification; -import net.yacy.crawler.CrawlSwitchboard; -import net.yacy.crawler.data.CrawlProfile; -import net.yacy.crawler.retrieval.Request; -import net.yacy.data.WorkTables; -import net.yacy.kelondro.blob.Tables; -import net.yacy.search.Switchboard; - -public class YMarkCrawlStart extends HashMap<String,String>{ - - private static final long serialVersionUID = 1L; - private final WorkTables worktables; - private Date date_last_exec; - private Date date_next_exec; - private Date date_recording; - private String apicall_pk; - private String url; - - public static enum CRAWLSTART { - SINGLE, ONE_LINK, FULL_DOMAIN - } - - public YMarkCrawlStart(final WorkTables worktables, final String url) { - super(); - this.worktables = worktables; - this.url = url; - this.date_recording = new Date(0); - this.clear(); - this.load(); - } - - public String getPK() { - if(this.isEmpty()) - return ""; - return this.apicall_pk; - } - - public Date date_last_exec() { - if(this.isEmpty()) - return new Date(0); - return this.date_last_exec; - } - - public Date date_next_exec() { - if(this.isEmpty()) - return new Date(0); - return this.date_next_exec; - } - - public boolean hasSchedule() { - return (!this.isEmpty() && this.date_next_exec.after(new Date())); - } - - public boolean isRunning(final CrawlSwitchboard crawler) { - final Iterator<byte[]> iter = crawler.getActive().iterator(); - while(iter.hasNext()) { - final byte[] key = iter.next(); - final CrawlProfile crawl = crawler.getActive(key); - if (crawl != null) return true; - } - return false; - } - - public Date date_recording() { - return this.date_recording; - } - - public void set_url(final String url) { - if(!this.url.equals(url)) { - this.url = url; - this.clear(); - this.load(); - } - } - - public int exec(final String host, final int port, final String username, final String pass) { - return this.worktables.execAPICall(this.apicall_pk, host, port, username, pass); - } - - private void load() { - try { - final StringBuilder buffer = new StringBuilder(500); - buffer.append("^crawl start for "); - buffer.append(Pattern.quote(this.url)); - buffer.append("?.*"); - final Pattern pattern = Pattern.compile(buffer.toString()); - //final Iterator<Tables.Row> APIcalls = this.worktables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_URL, pattern); - final Iterator<Tables.Row> APIcalls = this.worktables.iterator(WorkTables.TABLE_API_NAME, WorkTables.TABLE_API_COL_COMMENT, pattern); - Tables.Row row = null; - while(APIcalls.hasNext()) { - row = APIcalls.next(); - if(row.get(WorkTables.TABLE_API_COL_TYPE, "").equals("crawler")) { - Date date = row.get(WorkTables.TABLE_API_COL_DATE_RECORDING, row.get(WorkTables.TABLE_API_COL_DATE, new Date())); - if(date.after(this.date_recording)) { - this.clear(); - this.apicall_pk = UTF8.String(row.getPK()); - this.date_recording = date; - this.date_next_exec = row.get(WorkTables.TABLE_API_COL_DATE_NEXT_EXEC, new Date(0)); - this.date_last_exec = row.get(WorkTables.TABLE_API_COL_DATE_LAST_EXEC, new Date(0)); - buffer.setLength(0); - buffer.append(row.get(WorkTables.TABLE_API_COL_URL, "")); - buffer.delete(0, buffer.indexOf("?")+1); - int start = 0; - int end = 0; - String key; - String value; - while(start < buffer.length()) { - end = buffer.indexOf("=", start); - key = buffer.substring(start, end); - start = end+1; - end = buffer.indexOf("&", start); - if(end < 0 || end > buffer.length()) - end = buffer.length()-1; - value = buffer.substring(start, end); - start = end+1; - put(key, value); - } - } - } - } - } catch (final IOException e) { - // TODO Auto-generated catch block - } - } - - protected static String crawlStart( - final Switchboard sb, - final DigestURL startURL, - final String urlMustMatch, - final String urlMustNotMatch, - final int depth, - final boolean crawlingQ, final boolean medialink) { - final CrawlProfile pe = new CrawlProfile( - (startURL.getHost() == null) ? startURL.toNormalform(true) : startURL.getHost(), - urlMustMatch, - urlMustNotMatch, - CrawlProfile.MATCH_ALL_STRING, - CrawlProfile.MATCH_NEVER_STRING, - CrawlProfile.MATCH_NEVER_STRING, - CrawlProfile.MATCH_NEVER_STRING, - CrawlProfile.MATCH_ALL_STRING, - CrawlProfile.MATCH_NEVER_STRING, - CrawlProfile.MATCH_ALL_STRING, - CrawlProfile.MATCH_NEVER_STRING, - depth, - medialink, - CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_PROXY_RECRAWL_CYCLE), - -1, - crawlingQ, - true, true, true, false, - true, true, false, - -1, false, true, CrawlProfile.MATCH_NEVER_STRING, - CacheStrategy.IFFRESH, - "robot_" + CrawlSwitchboard.CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA, - ClientIdentification.yacyIntranetCrawlerAgentName, - null, null, - 0); // TODO: make this a default profile in CrawlSwitchboard - sb.crawler.putActive(pe.handle().getBytes(), pe); - return sb.crawlStacker.stackCrawl(new Request( - sb.peers.mySeed().hash.getBytes(), - startURL, - null, - "CRAWLING-ROOT", - new Date(), - pe.handle(), 0, pe.timezoneOffset() - )); - } -} diff --git a/source/net/yacy/data/ymark/YMarkDMOZImporter.java b/source/net/yacy/data/ymark/YMarkDMOZImporter.java deleted file mode 100644 index e991e3cf1..000000000 --- a/source/net/yacy/data/ymark/YMarkDMOZImporter.java +++ /dev/null @@ -1,156 +0,0 @@ -// YMarkDMOZImporter.java -// (C) 2012 by Stefan Foerster (apfelmaennchen), sof@gmx.de, Norderstedt, Germany -// first published 2012 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.data.ymark; - -import net.yacy.cora.lod.vocabulary.DMOZ; -import net.yacy.cora.lod.vocabulary.DublinCore; - -import org.xml.sax.Attributes; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; -import org.xml.sax.helpers.DefaultHandler; -import org.xml.sax.helpers.XMLReaderFactory; - -public class YMarkDMOZImporter extends YMarkImporter { - // Statics - public static String IMPORTER = "DMOZ"; - - // Importer Variables - private final XMLReader xmlReader; - private int depth; - - public YMarkDMOZImporter(final MonitoredReader dmoz_file, final int queueSize, final String targetFolder, final String sourceFolder) throws SAXException { - super(dmoz_file, queueSize, sourceFolder, targetFolder); - setImporter(IMPORTER); - this.xmlReader = XMLReaderFactory.createXMLReader(); - this.xmlReader.setFeature(XML_NAMESPACE_PREFIXES, false); - this.xmlReader.setFeature(XML_NAMESPACES, false); - this.xmlReader.setFeature(XML_VALIDATION, false); - this.xmlReader.setContentHandler(new DMOZParser()); - this.depth = Integer.MAX_VALUE; - } - - @Override - public void parse() throws Exception { - xmlReader.parse(new InputSource(bmk_file)); - } - - public void setDepth(int d) { - this.depth = d + YMarkUtil.FOLDERS_SEPARATOR_PATTERN.split(this.targetFolder).length-1; - } - - public class DMOZParser extends DefaultHandler { - - private YMarkEntry bmk; - private boolean isNewEntry; - private boolean isSubtopic; - private String tag; - private final StringBuilder buffer; - - public DMOZParser() { - this.bmk = new YMarkEntry(); - this.isNewEntry = false; - this.isSubtopic = false; - this.buffer = new StringBuilder(512); - } - - @Override - public void startElement(final String uri, String localName, final String qName, final Attributes attributes) throws SAXException { - // get rid of namespace prefixes - if (localName.isEmpty()) { - localName = qName.substring(qName.indexOf(':')+1); - } - this.tag = null; - if (localName.equals(DMOZ.ExternalPage.name())) { - this.bmk = new YMarkEntry(); - this.bmk.put(YMarkEntry.BOOKMARK.URL.key(), attributes.getValue(0)); - this.isNewEntry = true; - } - if(isNewEntry && localName.equals(DublinCore.Title.name())) { - this.tag = YMarkEntry.BOOKMARK.TITLE.key(); - } - if(isNewEntry && localName.equals(DublinCore.Description.name())) { - this.tag = YMarkEntry.BOOKMARK.DESC.key(); - } - if(isNewEntry && localName.equals(DMOZ.topic.name())) { - this.tag = YMarkEntry.BOOKMARK.FOLDERS.key(); - buffer.append(targetFolder); - buffer.append(YMarkUtil.FOLDERS_SEPARATOR); - } - } - - @Override - public void endElement(final String uri, String localName, final String qName) throws SAXException { - // get rid of namespace prefixes - if (localName.isEmpty()) { - localName = qName.substring(qName.indexOf(':')+1); - } - if (this.isNewEntry && this.isSubtopic && localName.equals(DMOZ.ExternalPage.name())) { - try { - bookmarks.put(this.bmk); - } catch (final InterruptedException e) { - e.printStackTrace(); - } finally { - this.isSubtopic = false; - this.isNewEntry = false; - } - } else if(localName.equals(DMOZ.topic.name())) { - int d = 0; - for(int i=0; i<this.buffer.length(); i++) { - if (this.buffer.charAt(i) == '/') { - d++; - if (d > depth) { - this.buffer.setLength(i); - break; - } - } - } - if (this.buffer.substring(targetFolder.length()+1).startsWith(sourceFolder)) { - this.isSubtopic = true; - this.bmk.put(this.tag, YMarkUtil.cleanFoldersString(buffer)); - } else { - this.isSubtopic = false; - this.isNewEntry = false; - } - } else if (this.tag != null) { - this.bmk.put(this.tag, buffer.toString()); - } - this.tag = null; - this.buffer.setLength(0); - } - - @Override - public void characters(final char ch[], final int start, final int length) throws SAXException { - // no processing here, as the SAX Parser characters method could be called more than once per tag! - if(this.tag != null) { - buffer.append(ch, start, length); - } - } - } -} - diff --git a/source/net/yacy/data/ymark/YMarkDate.java b/source/net/yacy/data/ymark/YMarkDate.java deleted file mode 100644 index a1ba23388..000000000 --- a/source/net/yacy/data/ymark/YMarkDate.java +++ /dev/null @@ -1,106 +0,0 @@ -// YMarkDate.java -// (C) 2011 by Stefan Förster, sof@gmx.de, Norderstedt, Germany -// first published 2010 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate: 2011-03-09 13:50:39 +0100 (Mi, 09 Mrz 2011) $ -// $LastChangedRevision: 7574 $ -// $LastChangedBy: apfelmaennchen $ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.data.ymark; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; - -import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.encoding.UTF8; - -public class YMarkDate { - - private long date; - - public YMarkDate() { - this.date = System.currentTimeMillis(); - } - - public YMarkDate(final byte[] date) { - this.set(date); - } - - public YMarkDate(final Date date) { - this.date = date.getTime(); - } - - public long parseISO8601(final String s) throws ParseException { - if(s == null || s.length() < 1) { - throw new ParseException("parseISO8601 - empty string, nothing to parse", 0); - } - SimpleDateFormat dateformat; - StringBuilder date = new StringBuilder(s); - if(s.length()==10) - dateformat = new SimpleDateFormat("yyyy-MM-dd"); - else { - dateformat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssz"); - if(date.charAt(date.length()-1) == 'Z') { - date.deleteCharAt(date.length()-1); - date.append("GMT-00:00"); - } else { - date.insert(date.length()-6, "GMT"); - } - } - this.date = dateformat.parse(date.toString()).getTime(); - return this.date; - } - - public String toISO8601() { - return (this.date == 0) ? YMarkEntry.BOOKMARK.DATE_MODIFIED.deflt() : ISO8601(new Date(this.date)); - } - - public static String ISO8601(final Date date) { - return ISO8601Formatter.FORMATTER.format(date); - } - - public byte[] toBytes() { - return String.valueOf(this.date).getBytes(); - } - - @Override - public String toString() { - return String.valueOf(this.date); - } - - public long get() { - return this.date; - } - - public void set(long date) { - this.date = date; - } - - public void set(byte[] date) { - final String s = UTF8.String(date); - if(!s.isEmpty()) { - this.date = Long.parseLong(s); - } else { - this.date = 0; - } - } -} diff --git a/source/net/yacy/data/ymark/YMarkEntry.java b/source/net/yacy/data/ymark/YMarkEntry.java deleted file mode 100644 index 3b6162cb2..000000000 --- a/source/net/yacy/data/ymark/YMarkEntry.java +++ /dev/null @@ -1,255 +0,0 @@ -// YMarkEntry.java -// (C) 2011 by Stefan Förster, sof@gmx.de, Norderstedt, Germany -// first published 2011 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.data.ymark; - -import java.net.MalformedURLException; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Map; -import java.util.TreeMap; - -import net.yacy.cora.document.id.DigestURL; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.crawler.data.CrawlProfile; -import net.yacy.document.content.DCEntry; -import net.yacy.kelondro.blob.Tables; -import net.yacy.search.Switchboard; - -public class YMarkEntry extends TreeMap<String, String> { - - private static final long serialVersionUID = 2179622977348536148L; - - public static final YMarkEntry POISON = new YMarkEntry(); - public static final YMarkEntry EMPTY = new YMarkEntry(); - public static final String BOOKMARKS_ID = "id"; - public static final String BOOKMARKS_REF = "ref"; - public static final String FOLDERS_IMPORTED = "/imported"; - - public static enum BOOKMARK { - // key dc_attrb dflt html_attrb xbel_attrb json_attrb type index separator - URL ("url", "dc:identifier", "", "href", "href", "uri", "link", false, YMarkUtil.EMPTY_STRING), - TITLE ("title", "dc:title", "", "", "", "title", "meta", false, YMarkUtil.EMPTY_STRING), - DESC ("desc", "dc:description", "", "", "", "", "comment", false, YMarkUtil.EMPTY_STRING), - DATE_ADDED ("date_added", "", "", "add_date", "added", "dateAdded", "date", false, YMarkUtil.EMPTY_STRING), - DATE_MODIFIED ("date_modified", "", "", "last_modified", "modified", "lastModified", "date", false, YMarkUtil.EMPTY_STRING), - DATE_VISITED ("date_visited", "", "", "last_visited", "visited", "", "date", false, YMarkUtil.EMPTY_STRING), - PUBLIC ("public", "", "false", "private", "yacy:public", "", "lock", false, YMarkUtil.EMPTY_STRING), - TAGS ("tags", "dc:subject", "unsorted", "shortcuturl", "yacy:tags", "keyword", "tag", true, YMarkUtil.TAGS_SEPARATOR), - VISITS ("visits", "", "0", "", "yacy:visits", "", "stat", false, YMarkUtil.EMPTY_STRING), - FOLDERS ("folders", "", "/unsorted", "", "", "", "folder", true, YMarkUtil.TAGS_SEPARATOR), - FILTER ("filter", "", "", "", "yacy:filter", "", "filter", false, YMarkUtil.EMPTY_STRING), - OAI ("oai", "", "", "", "yacy:oai", "", "oai", false, YMarkUtil.EMPTY_STRING), - URLHASH ("urlhash", "", "", "", "yacy:urlhash", "", "urlhash", false, YMarkUtil.EMPTY_STRING), - STARRATING ("starrating", "", "", "", "yacy:starrating", "", "stat", false, YMarkUtil.EMPTY_STRING); - - private String key; - private String dc_attrb; - private String dflt; - private String html_attrb; - private String xbel_attrb; - private String json_attrb; - private String type; - private boolean index; - private String seperator; - - private static final Map<String,BOOKMARK> lookup = new HashMap<String,BOOKMARK>(); - private static final Map<String,String> indexColumns = new HashMap<String,String>(); - static { - for(BOOKMARK b : EnumSet.allOf(BOOKMARK.class)) { - lookup.put(b.key, b); - if(b.index) { - indexColumns.put(b.key, b.seperator); - } - } - } - - private static StringBuilder buffer = new StringBuilder(25); - - private BOOKMARK(final String k, final String d, final String s, final String a, final String x, final String j, final String t, final boolean index, final String separator) { - this.key = k; - this.dc_attrb = d; - this.dflt = s; - this.html_attrb = a; - this.xbel_attrb = x; - this.json_attrb = j; - this.type = t; - this.index = index; - this.seperator = separator; - } - public static Map<String,String> indexColumns() { - return Collections.unmodifiableMap(indexColumns); - } - public static BOOKMARK get(String key) { - return lookup.get(key); - } - public static boolean contains(String key) { - return lookup.containsKey(key); - } - public String key() { - return this.key; - } - public String deflt() { - return this.dflt; - } - public String html_attrb() { - return this.html_attrb; - } - public String xbel_attrb() { - return this.xbel_attrb; - } - public String json_attrb() { - return this.json_attrb; - } - public String dc_attrb() { - return this.dc_attrb; - } - public String xbel() { - buffer.setLength(0); - buffer.append('"'); - buffer.append('\n'); - buffer.append(' '); - buffer.append(this.xbel_attrb); - buffer.append('='); - buffer.append('"'); - return buffer.toString(); - } - public String type() { - return this.type; - } - public boolean index() { - return this.index; - } - public String seperator() { - return this.seperator; - } - - } - - public YMarkEntry() { - this(true); - } - - public YMarkEntry(final boolean setDefaults) { - super(); - if(setDefaults) { - setCurrentTimeMillis(BOOKMARK.DATE_ADDED); - setCurrentTimeMillis(BOOKMARK.DATE_MODIFIED); - setDefaults(); - } - } - - public YMarkEntry(final DCEntry dc) { - super(); - for (BOOKMARK b : BOOKMARK.values()) { - if (dc.getMap().containsKey(b.dc_attrb)) { - this.put(b.key(), dc.get(b.dc_attrb)); - } - } - setCurrentTimeMillis(BOOKMARK.DATE_ADDED); - setCurrentTimeMillis(BOOKMARK.DATE_MODIFIED); - setDefaults(); - } - - public YMarkEntry(final Tables.Row bmk_row) { - super(); - for (BOOKMARK b : BOOKMARK.values()) { - if(bmk_row.containsKey(b.key())) { - this.put(b.key(), bmk_row.get(b.key(), b.deflt())); - } - } - } - - private void setCurrentTimeMillis(BOOKMARK b) { - switch(b) { - case DATE_ADDED: - case DATE_MODIFIED: - case DATE_VISITED: - this.put(b.key(), String.valueOf(System.currentTimeMillis())); - break; - default: - break; - } - } - - public void setDefaults() { - for (BOOKMARK b : BOOKMARK.values()) { - if(!b.deflt().isEmpty() && !this.containsKey(b.key())) { - this.put(b.key(), b.deflt()); - } - } - } - - public byte[] getUrlHash() { - if(this.containsKey(YMarkEntry.BOOKMARK.URL.key())) - try { - return YMarkUtil.getBookmarkId(this.get(YMarkEntry.BOOKMARK.URL.key())); - } catch (final MalformedURLException e) { - ConcurrentLog.warn(YMarkTables.BOOKMARKS_LOG, "getUrlHash - MalformedURLException for YMarkEntry: "+this.get(YMarkEntry.BOOKMARK.URL.key())); - } - return null; - } - - public DCEntry getDCEntry() { - final DCEntry dc = new DCEntry(); - for (BOOKMARK b : BOOKMARK.values()) { - if(!b.dc_attrb.isEmpty() && this.containsKey(b.key())) { - dc.getMap().put(b.dc_attrb, new String[]{this.get(b.key())}); - } - } - return dc; - } - - public Tables.Data getData() { - final Tables.Data data = new Tables.Data(); - for (BOOKMARK b : BOOKMARK.values()) { - if(this.containsKey(b.key()) && this.get(b.key()) != null) { - data.put(b.key(), this.get(b.key())); - } else { - data.put(b.key(), b.deflt()); - } - } - return data; - } - - public void crawl(final YMarkCrawlStart.CRAWLSTART type, final boolean medialink, final Switchboard sb) throws MalformedURLException { - final DigestURL url = new DigestURL(this.get(BOOKMARK.URL.key())); - switch(type) { - case SINGLE: - YMarkCrawlStart.crawlStart(sb, url, CrawlProfile.MATCH_ALL_STRING, CrawlProfile.MATCH_NEVER_STRING, 0, true, medialink); - break; - case ONE_LINK: - YMarkCrawlStart.crawlStart(sb, url, CrawlProfile.MATCH_ALL_STRING, CrawlProfile.MATCH_NEVER_STRING, 1, true, medialink); - break; - case FULL_DOMAIN: - YMarkCrawlStart.crawlStart(sb, url, CrawlProfile.mustMatchFilterFullDomain(url), CrawlProfile.MATCH_NEVER_STRING, 99, false, medialink); - break; - default: - break; - } - } -} diff --git a/source/net/yacy/data/ymark/YMarkHTMLImporter.java b/source/net/yacy/data/ymark/YMarkHTMLImporter.java deleted file mode 100644 index 91fc53af7..000000000 --- a/source/net/yacy/data/ymark/YMarkHTMLImporter.java +++ /dev/null @@ -1,188 +0,0 @@ -// YMarkHTMLImporter.java -// (C) 2011 by Stefan F��rster, sof@gmx.de, Norderstedt, Germany -// first published 2010 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.data.ymark; - -import javax.swing.text.MutableAttributeSet; -import javax.swing.text.html.HTML; -import javax.swing.text.html.HTMLEditorKit; -import javax.swing.text.html.parser.ParserDelegator; - -import net.yacy.cora.util.ConcurrentLog; - -public class YMarkHTMLImporter extends YMarkImporter { - - // Importer Variables - private final ParserDelegator htmlParser; - - // Statics - public static String IMPORTER = "HTML"; - public static enum STATE { - NOTHING, - BOOKMARK, - FOLDER, - BMK_DESC, - FOLDER_DESC - } - public static final String MILLIS = "000"; - - public YMarkHTMLImporter(final MonitoredReader bmk_file, final int queueSize, final String targetFolder, final String sourceFolder) { - super(bmk_file, queueSize, targetFolder, sourceFolder); - setImporter(IMPORTER); - this.htmlParser = new ParserDelegator(); - } - - public YMarkHTMLImporter (final MonitoredReader bmk_file, final int queueSize, final String targetFolder) { - this(bmk_file, queueSize, targetFolder, ""); - } - - @Override - public void parse() throws Exception { - htmlParser.parse(bmk_file, new HTMLParser(), true); - } - - public class HTMLParser extends HTMLEditorKit.ParserCallback { - - private YMarkEntry bmk; - private final StringBuilder folderstring; - private STATE state; - private HTML.Tag prevTag; - - public HTMLParser() { - this.folderstring = new StringBuilder(YMarkTables.BUFFER_LENGTH); - this.folderstring.append(targetFolder); - this.bmk = new YMarkEntry(); - this.state = STATE.NOTHING; - this.prevTag = null; - } - - @Override - public void handleText(char[] data, int pos) { - switch (state) { - case NOTHING: - break; - case BOOKMARK: - this.bmk.put(YMarkEntry.BOOKMARK.TITLE.key(), new String(data)); - this.bmk.put(YMarkEntry.BOOKMARK.FOLDERS.key(), this.folderstring.toString()); - this.bmk.put(YMarkEntry.BOOKMARK.PUBLIC.key(), YMarkEntry.BOOKMARK.PUBLIC.deflt()); - this.bmk.put(YMarkEntry.BOOKMARK.VISITS.key(), YMarkEntry.BOOKMARK.VISITS.deflt()); - break; - case FOLDER: - this.folderstring.append(YMarkUtil.FOLDERS_SEPARATOR); - this.folderstring.append(data); - break; - case FOLDER_DESC: - ConcurrentLog.info(YMarkTables.BOOKMARKS_LOG, "YMarksHTMLImporter - folder: "+this.folderstring+" desc: " + new String(data)); - break; - case BMK_DESC: - this.bmk.put(YMarkEntry.BOOKMARK.DESC.key(), new String(data)); - break; - default: - break; - } - } - - @Override - public void handleStartTag(HTML.Tag t, MutableAttributeSet a, int pos) { - if (t == HTML.Tag.A) { - if (!this.bmk.isEmpty()) { - try { - bookmarks.put(this.bmk); - bmk = new YMarkEntry(); - } catch (final InterruptedException e) { - ConcurrentLog.logException(e); - } - } - final String url = (String)a.getAttribute(HTML.Attribute.HREF); - this.bmk.put(YMarkEntry.BOOKMARK.URL.key(), url); - final StringBuilder sb = new StringBuilder(255); - for (YMarkEntry.BOOKMARK bmk : YMarkEntry.BOOKMARK.values()) { - sb.setLength(0); - if (a.isDefined(bmk.html_attrb())) { - sb.append((String)a.getAttribute(bmk.html_attrb())); - ConcurrentLog.info(YMarkTables.BOOKMARKS_LOG, bmk.key()+" : "+sb.toString()); - } - switch(bmk) { - case TAGS: - // sb already contains the mozilla shortcuturl - // add delicious.com tags that are stored in the tags attribute - if (a.isDefined(YMarkEntry.BOOKMARK.TAGS.key())) { - sb.append(YMarkUtil.TAGS_SEPARATOR); - sb.append((String)a.getAttribute(YMarkEntry.BOOKMARK.TAGS.key())); - } - this.bmk.put(bmk.key(), YMarkUtil.cleanTagsString(sb.toString())); - break; - case PUBLIC: - // look for delicious.com private attribute - if(sb.toString().equals("0")) - this.bmk.put(bmk.key(), "true"); - break; - case DATE_ADDED: - case DATE_MODIFIED: - case DATE_VISITED: - sb.append(MILLIS); - this.bmk.put(bmk.key(), sb.toString()); - break; - default: - break; - } - } - state = STATE.BOOKMARK; - } else if (t == HTML.Tag.H3) { - state = STATE.FOLDER; - } else if (t == HTML.Tag.DD && this.prevTag == HTML.Tag.A) { - state = STATE.BMK_DESC; - } else { - state = STATE.NOTHING; - } - this.prevTag = t; - } - - @Override - public void handleEndTag(HTML.Tag t, int pos) { - // write the last bookmark, as no more <a> tags are following - if (t == HTML.Tag.HTML) { - if (!this.bmk.isEmpty()) { - try { - bookmarks.put(this.bmk); - } catch (final InterruptedException e) { - ConcurrentLog.logException(e); - } - } - } - if (t == HTML.Tag.H3) { - state = STATE.FOLDER_DESC; - } else if (t == HTML.Tag.DL) { - //TODO: get rid of .toString.equals() - if(!this.folderstring.toString().equals(targetFolder)) { - folderstring.setLength(folderstring.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR)); - } - } else { - state = STATE.NOTHING; - } - } - } -} diff --git a/source/net/yacy/data/ymark/YMarkImporter.java b/source/net/yacy/data/ymark/YMarkImporter.java deleted file mode 100644 index e00a87bb5..000000000 --- a/source/net/yacy/data/ymark/YMarkImporter.java +++ /dev/null @@ -1,156 +0,0 @@ -// YMarkImporter.java -// (C) 2012 by Stefan Foerster (apfelmaennchen), sof@gmx.de, Norderstedt, Germany -// first published 2012 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.data.ymark; - -import java.io.IOException; -import java.util.concurrent.ArrayBlockingQueue; - -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.search.Switchboard; - -public abstract class YMarkImporter implements Runnable { - // Statics - public final static String XML_NAMESPACE_PREFIXES = "http://xml.org/sax/features/namespace-prefixes"; - public final static String XML_NAMESPACES = "http://xml.org/sax/features/namespaces"; - public final static String XML_VALIDATION = "http://xml.org/sax/features/validation"; - - protected String importer; - protected ArrayBlockingQueue<YMarkEntry> bookmarks; - protected final MonitoredReader bmk_file; - protected final String targetFolder; - protected final String sourceFolder; - - public YMarkImporter(final MonitoredReader bmk_file, final int queueSize, final String sourceFolder, final String targetFolder) { - this.bookmarks = new ArrayBlockingQueue<YMarkEntry>(queueSize); - this.bmk_file = bmk_file; - this.sourceFolder = YMarkUtil.cleanFoldersString(sourceFolder); - this.targetFolder = YMarkUtil.cleanFoldersString(targetFolder); - } - - @Override - public void run() { - try { - parse(); - } catch (final Exception e) { - ConcurrentLog.logException(e); - } finally { - try { - ConcurrentLog.info(YMarkTables.BOOKMARKS_LOG, this.importer+" Importer inserted poison pill in queue"); - this.bookmarks.put(YMarkEntry.POISON); - } catch (final InterruptedException e1) { - ConcurrentLog.logException(e1); - } - } - } - - public YMarkEntry take() { - try { - return this.bookmarks.take(); - } catch (final InterruptedException e) { - ConcurrentLog.logException(e); - return null; - } - } - - public void setImporter(final String importer) { - this.importer = importer; - } - - public long getProgress() { - return this.bmk_file.getProgress(); - } - - public long maxProgress() { - return this.bmk_file.maxProgress(); - } - - public abstract void parse() throws Exception; - - public Consumer getConsumer(final Switchboard sb, final String bmk_user, final ArrayBlockingQueue<String> autoTaggingQueue, - final boolean autotag, final boolean empty, final String indexing, final boolean medialink) { - return new Consumer(sb, bmk_user, autoTaggingQueue, autotag, empty, indexing, medialink); - } - - public class Consumer implements Runnable { - private final Switchboard sb; - private final String bmk_user; - private final ArrayBlockingQueue<String> autoTaggingQueue; - private final String indexing; - - private final boolean autotag; - private final boolean empty; - private final boolean medialink; - - public Consumer(final Switchboard sb, final String bmk_user, final ArrayBlockingQueue<String> autoTaggingQueue, - final boolean autotag, final boolean empty, final String indexing, final boolean medialink) { - this.sb = sb; - this.bmk_user = bmk_user; - this.autoTaggingQueue = autoTaggingQueue; - this.autotag = autotag; - this.empty = empty; - this.indexing = indexing; - this.medialink = medialink; - } - - @Override - public void run() { - YMarkEntry bmk; - while ((bmk = take()) != YMarkEntry.POISON) { - try { - final String url = bmk.get(YMarkEntry.BOOKMARK.URL.key()); - // other protocols could cause problems - if(url != null && url.startsWith("http")) { - this.sb.tables.bookmarks.addBookmark(this.bmk_user, bmk, true, true); - if(this.autotag) { - if(!this.empty) { - this.autoTaggingQueue.put(url); - } else if(!bmk.containsKey(YMarkEntry.BOOKMARK.TAGS.key()) || bmk.get(YMarkEntry.BOOKMARK.TAGS.key()).equals(YMarkEntry.BOOKMARK.TAGS.deflt())) { - this.autoTaggingQueue.put(url); - } - } - // fill crawler - if (this.indexing.equals("single")) { - bmk.crawl(YMarkCrawlStart.CRAWLSTART.SINGLE, this.medialink, this.sb); - } else if (this.indexing.equals("onelink")) { - bmk.crawl(YMarkCrawlStart.CRAWLSTART.ONE_LINK, this.medialink, this.sb); - } else if (this.indexing.equals("fulldomain")) { - bmk.crawl(YMarkCrawlStart.CRAWLSTART.FULL_DOMAIN, this.medialink, this.sb); - } - } - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final InterruptedException e) { - ConcurrentLog.logException(e); - } - } - if(this.autotag) { - try { - this.autoTaggingQueue.put(YMarkAutoTagger.POISON); - ConcurrentLog.info(YMarkTables.BOOKMARKS_LOG, YMarkImporter.this.importer+" inserted poison pill into autoTagging queue"); - } catch (final InterruptedException e) { - ConcurrentLog.logException(e); - } - } - } - } -} diff --git a/source/net/yacy/data/ymark/YMarkJSONImporter.java b/source/net/yacy/data/ymark/YMarkJSONImporter.java deleted file mode 100644 index 5076fb042..000000000 --- a/source/net/yacy/data/ymark/YMarkJSONImporter.java +++ /dev/null @@ -1,204 +0,0 @@ -package net.yacy.data.ymark; - -import java.io.IOException; -import java.io.Reader; -import java.util.HashMap; -import java.util.concurrent.ArrayBlockingQueue; - -import net.yacy.cora.util.ConcurrentLog; - -import org.json.simple.parser.ContentHandler; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; - -public class YMarkJSONImporter implements Runnable, ContentHandler{ - - // Importer Variables - private final ArrayBlockingQueue<YMarkEntry> bookmarks; - private final Reader bmk_file; - private final String RootFolder; - private final StringBuilder folderstring; - private YMarkEntry bmk; - private final JSONParser parser; - - // Statics - public final static String FOLDER = "text/x-moz-place-container"; - public final static String BOOKMARK = "text/x-moz-place"; - public final static String ANNOS = "annos"; - public final static String TYPE = "type"; - public final static String CHILDREN = "children"; - - // Parser Variables - private final StringBuilder value; - private final StringBuilder key; - private final StringBuilder date; - private final HashMap<String,String> obj; - private int depth; - - private Boolean isFolder; - private Boolean isBookmark; - private Boolean isAnnos; - - public YMarkJSONImporter(final Reader bmk_file, final int queueSize, final String root) { - this.bookmarks = new ArrayBlockingQueue<YMarkEntry>(queueSize); - this.bmk_file = bmk_file; - this.RootFolder = root; - this.folderstring = new StringBuilder(YMarkTables.BUFFER_LENGTH); - this.folderstring.append(this.RootFolder); - this.bmk = new YMarkEntry(); - - this.parser = new JSONParser(); - - this.value = new StringBuilder(128); - this.key = new StringBuilder(16); - this.date = new StringBuilder(32); - this.obj = new HashMap<String,String>(); - this.depth = 0; - - this.isAnnos = false; - this.isBookmark = false; - this.isFolder = true; - } - - @Override - public void startJSON() throws ParseException, IOException { - } - - @Override - public void endJSON() throws ParseException, IOException { - } - - @Override - public boolean startArray() throws ParseException, IOException { - final String key = this.key.toString(); - if(key.equals(CHILDREN) && this.isFolder) { - if(this.depth > 0) { - this.folderstring.append(YMarkUtil.FOLDERS_SEPARATOR); - this.folderstring.append(this.obj.get(YMarkEntry.BOOKMARK.TITLE.json_attrb())); - } - this.depth++; - } else if(key.equals(ANNOS)) { - this.isAnnos = true; - } - return true; - } - - @Override - public boolean endArray() throws ParseException, IOException { - if(this.isAnnos) { - this.isAnnos = false; - } else if(this.depth > 0) { - if(this.depth == 1) - folderstring.setLength(0); - else - folderstring.setLength(folderstring.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR)); - this.depth--; - } - return true; - } - - @Override - public boolean startObject() throws ParseException, IOException { - if(!this.isAnnos) { - this.obj.clear(); - } - return true; - } - - @Override - public boolean endObject() throws ParseException, IOException { - if(this.isBookmark) { - this.bmk.put(YMarkEntry.BOOKMARK.TITLE.key(),obj.get(YMarkEntry.BOOKMARK.TITLE.json_attrb())); - this.bmk.put(YMarkEntry.BOOKMARK.URL.key(),obj.get(YMarkEntry.BOOKMARK.URL.json_attrb())); - date.setLength(0); - date.append(obj.get(YMarkEntry.BOOKMARK.DATE_ADDED.json_attrb())); - date.setLength(date.length()-3); - this.bmk.put(YMarkEntry.BOOKMARK.DATE_ADDED.key(), date.toString()); - date.setLength(0); - date.append(obj.get(YMarkEntry.BOOKMARK.DATE_MODIFIED.json_attrb())); - date.setLength(date.length()-3); - this.bmk.put(YMarkEntry.BOOKMARK.DATE_MODIFIED.key(), date.toString()); - this.bmk.put(YMarkEntry.BOOKMARK.FOLDERS.key(),this.folderstring.toString()); - if(this.obj.containsKey(YMarkEntry.BOOKMARK.TAGS.json_attrb())) { - this.bmk.put(YMarkEntry.BOOKMARK.TAGS.key(),obj.get(YMarkEntry.BOOKMARK.TAGS.json_attrb())); - } - try { - this.bookmarks.put(this.bmk); - } catch (final InterruptedException e) { - ConcurrentLog.logException(e); - } - this.bmk = new YMarkEntry(); - } - this.isBookmark = false; - return true; - } - - @Override - public boolean startObjectEntry(String key) throws ParseException, IOException { - if(!this.isAnnos) { - this.key.setLength(0); - this.key.append(key); - } - return true; - } - - @Override - public boolean primitive(Object value) throws ParseException, IOException { - if(!this.isAnnos) { - this.value.setLength(0); - if(value instanceof java.lang.String) { - this.value.append((String)value); - } else if(value instanceof java.lang.Boolean) { - this.value.append(value); - } else if(value instanceof java.lang.Number) { - this.value.append(value); - } - } - return true; - } - - @Override - public boolean endObjectEntry() throws ParseException, IOException { - if(!this.isAnnos) { - final String key = this.key.toString(); - final String value = this.value.toString(); - if(key.equals(TYPE)) { - if(value.equals(FOLDER)) { - this.isFolder = true; - } else if(value.equals(BOOKMARK)) { - this.isBookmark = true; - } - } - this.obj.put(key, value); - } - return true; - } - - @Override - public void run() { - try { - ConcurrentLog.info(YMarkTables.BOOKMARKS_LOG, "JSON Importer run()"); - this.parser.parse(this.bmk_file, this, true); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final ParseException e) { - ConcurrentLog.logException(e); - } finally { - try { - ConcurrentLog.info(YMarkTables.BOOKMARKS_LOG, "JSON Importer inserted poison pill in queue"); - this.bookmarks.put(YMarkEntry.POISON); - } catch (final InterruptedException e) { - ConcurrentLog.logException(e); - } - } - } - - public YMarkEntry take() { - try { - return this.bookmarks.take(); - } catch (final InterruptedException e) { - ConcurrentLog.logException(e); - return null; - } - } -} diff --git a/source/net/yacy/data/ymark/YMarkMetadata.java b/source/net/yacy/data/ymark/YMarkMetadata.java deleted file mode 100644 index ee25ee3bc..000000000 --- a/source/net/yacy/data/ymark/YMarkMetadata.java +++ /dev/null @@ -1,149 +0,0 @@ -// YMarkMetadata.java -// (C) 2011 by Stefan Förster, sof@gmx.de, Norderstedt, Germany -// first published 2010 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate: 2011-03-09 13:50:39 +0100 (Mi, 09 Mrz 2011) $ -// $LastChangedRevision: 7574 $ -// $LastChangedBy: apfelmaennchen $ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.data.ymark; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.EnumMap; - -import net.yacy.cora.date.ISO8601Formatter; -import net.yacy.cora.document.encoding.ASCII; -import net.yacy.cora.document.id.DigestURL; -import net.yacy.cora.federate.yacy.CacheStrategy; -import net.yacy.cora.protocol.ClientIdentification; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.crawler.retrieval.Response; -import net.yacy.document.Document; -import net.yacy.document.Parser.Failure; -import net.yacy.kelondro.data.meta.URIMetadataNode; -import net.yacy.repository.LoaderDispatcher; -import net.yacy.search.index.Segment; - -public class YMarkMetadata { - private DigestURL uri; - Document document; - Segment indexSegment; - - public enum METADATA { - TITLE, - DESCRIPTION, - FAVICON, - KEYWORDS, - LANGUAGE, - CREATOR, - PUBLISHER, - CHARSET, - MIMETYPE, - SIZE, - WORDCOUNT, - IN_URLDB, - FRESHDATE, - LOADDATE, - MODDATE, - SNIPPET, - AUTOTAG - } - - public YMarkMetadata(final DigestURL uri) { - this.uri = uri; - this.document = null; - this.indexSegment = null; - } - - public YMarkMetadata(final DigestURL uri, final Segment indexSegment) { - this.uri = uri; - this.document = null; - this.indexSegment = indexSegment; - } - - public YMarkMetadata(final byte[] urlHash, final Segment indexSegment) { - this.document = null; - this.indexSegment = indexSegment; - try { - this.uri = this.indexSegment.fulltext().getURL(ASCII.String(urlHash)); - } catch (IOException e) { - this.uri = null; - ConcurrentLog.logException(e); - } - } - - public YMarkMetadata(final Document document) { - this.document = document; - try { - this.uri = new DigestURL(this.document.dc_identifier()); - } catch (final MalformedURLException e) { - this.uri = null; - } - this.indexSegment = null; - } - - public Document loadDocument(final LoaderDispatcher loader, ClientIdentification.Agent agent) throws IOException, Failure { - if(this.document == null) { - Response response = null; - response = loader.load(loader.request(this.uri, true, false), CacheStrategy.IFEXIST, Integer.MAX_VALUE, null, agent); - Document[] docs = response.parse(); - this.document = Document.mergeDocuments(response.url(), response.getMimeType(), docs); - } - return this.document; - } - - public EnumMap<METADATA, String> getMetadata() { - final EnumMap<METADATA, String> metadata = new EnumMap<METADATA, String>(METADATA.class); - final URIMetadataNode urlEntry = this.indexSegment.fulltext().getMetadata(this.uri.hash()); - if (urlEntry != null) { - metadata.put(METADATA.SIZE, String.valueOf(urlEntry.filesize())); - metadata.put(METADATA.FRESHDATE, ISO8601Formatter.FORMATTER.format(urlEntry.freshdate())); - metadata.put(METADATA.LOADDATE, ISO8601Formatter.FORMATTER.format(urlEntry.loaddate())); - metadata.put(METADATA.MODDATE, ISO8601Formatter.FORMATTER.format(urlEntry.moddate())); - metadata.put(METADATA.SNIPPET, String.valueOf(urlEntry.snippet())); - metadata.put(METADATA.WORDCOUNT, String.valueOf(urlEntry.wordCount())); - metadata.put(METADATA.MIMETYPE, String.valueOf(urlEntry.mime())); - metadata.put(METADATA.LANGUAGE, urlEntry.language()); - metadata.put(METADATA.TITLE, urlEntry.dc_title()); - metadata.put(METADATA.CREATOR, urlEntry.dc_creator()); - metadata.put(METADATA.KEYWORDS, urlEntry.dc_subject()); - metadata.put(METADATA.PUBLISHER, urlEntry.dc_publisher()); - } - return metadata; - } - - public EnumMap<METADATA, String> loadMetadata() { - final EnumMap<METADATA, String> metadata = new EnumMap<METADATA, String>(METADATA.class); - if(this.document != null) { - metadata.put(METADATA.TITLE, this.document.dc_title()); - metadata.put(METADATA.CREATOR, this.document.dc_creator()); - metadata.put(METADATA.KEYWORDS, this.document.dc_subject(' ')); - metadata.put(METADATA.PUBLISHER, this.document.dc_publisher()); - metadata.put(METADATA.DESCRIPTION, this.document.dc_description().length > 0 ? this.document.dc_description()[0] : ""); - metadata.put(METADATA.MIMETYPE, this.document.dc_format()); - metadata.put(METADATA.LANGUAGE, this.document.dc_language()); - metadata.put(METADATA.CHARSET, this.document.getCharset()); - // metadata.put(METADATA.SIZE, String.valueOf(document.getTextLength())); - } - return metadata; - } -} diff --git a/source/net/yacy/data/ymark/YMarkTables.java b/source/net/yacy/data/ymark/YMarkTables.java deleted file mode 100644 index fdc3c83ad..000000000 --- a/source/net/yacy/data/ymark/YMarkTables.java +++ /dev/null @@ -1,554 +0,0 @@ -// YMarkTables.java -// (C) 2011 by Stefan Förster, sof@gmx.de, Norderstedt, Germany -// first published 2010 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.data.ymark; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.concurrent.ConcurrentHashMap; - -import javax.swing.event.ChangeEvent; -import javax.swing.event.ChangeListener; - -import net.yacy.cora.document.encoding.ASCII; -import net.yacy.cora.document.id.DigestURL; -import net.yacy.cora.protocol.ClientIdentification; -import net.yacy.cora.util.ConcurrentLog; -import net.yacy.cora.util.SpaceExceededException; -import net.yacy.data.WorkTables; -import net.yacy.document.Document; -import net.yacy.document.Parser.Failure; -import net.yacy.kelondro.blob.TableColumnIndexException; -import net.yacy.kelondro.blob.Tables; -import net.yacy.kelondro.blob.Tables.Row; -import net.yacy.kelondro.blob.TablesColumnIndex; -import net.yacy.repository.LoaderDispatcher; - -public class YMarkTables { - - public static enum TABLES { - BOOKMARKS ("_bookmarks"), - TAGS ("_tags"), - FOLDERS ("_folders"); - - private String basename; - - private TABLES(final String b) { - this.basename = b; - } - public String basename() { - return this.basename; - } - public String tablename(final String bmk_user) { - return bmk_user+this.basename; - } - } - - public static enum PROTOCOLS { - HTTP ("http://"), - HTTPS ("https://"); - - private String protocol; - - private PROTOCOLS(final String s) { - this.protocol = s; - } - public String protocol() { - return this.protocol; - } - public String protocol(final String s) { - return this.protocol+s; - } - } - - public final static String FOLDERS_ROOT = "/"; - public final static String BOOKMARKS_LOG = "BOOKMARKS"; - public final static String USER_ADMIN = "admin"; - public final static String USER_AUTHENTICATE_MSG = "Bookmark user authentication required!"; - - public final static int BUFFER_LENGTH = 256; - - private final WorkTables worktables; - private final Map<String, ChangeListener> progressListeners; - - public boolean dirty = false; - - public YMarkTables(final Tables wt) { - this.worktables = (WorkTables)wt; - this.progressListeners = new ConcurrentHashMap<String, ChangeListener>(); - this.buildIndex(); - } - - public ChangeListener getProgressListener(String thread) { - final ChangeListener l = new ProgressListener(); - this.progressListeners.put(thread, l); - return l; - } - - public void removeProgressListener(String thread) { - this.progressListeners.remove(thread); - } - - public class ProgressListener implements ChangeListener { - // the progress in % - private int progress = 0; - @Override - public void stateChanged(ChangeEvent e) { - final MonitoredReader mreader = (MonitoredReader)e.getSource(); - this.progress = (int)((mreader.getProgress() / mreader.maxProgress())*100); - } - public int progress() { - return this.progress; - } - } - - public void buildIndex() { - final Iterator<String> iter = this.worktables.iterator(); - while(iter.hasNext()) { - final String bmk_table = iter.next(); - if(bmk_table.endsWith(TABLES.BOOKMARKS.basename())) { - try { - final long time = System.currentTimeMillis(); - final TablesColumnIndex index = this.worktables.getIndex(bmk_table); - if(index.getType() == TablesColumnIndex.INDEXTYPE.RAM || index.size() == 0) { - ConcurrentLog.info(YMarkTables.BOOKMARKS_LOG, "buildIndex() "+YMarkEntry.BOOKMARK.indexColumns().keySet().toString()); - index.buildIndex(YMarkEntry.BOOKMARK.indexColumns(), this.worktables.iterator(bmk_table)); - ConcurrentLog.info(YMarkTables.BOOKMARKS_LOG, "build "+index.getType().name()+" index for columns "+YMarkEntry.BOOKMARK.indexColumns().keySet().toString() - +" of table "+bmk_table+" containing "+this.worktables.size(bmk_table)+ " bookmarks" - +" ("+(System.currentTimeMillis()-time)+"ms)"); - } - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final TableColumnIndexException e) { - // currently nothing to do... - } - } - } - } - - public void deleteBookmark(final String bmk_user, final byte[] urlHash) throws IOException, SpaceExceededException { - final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); - Tables.Row bmk_row = null; - bmk_row = this.worktables.select(bmk_table, urlHash); - if(bmk_row != null) { - this.worktables.delete(bmk_table,urlHash); - } - if(this.worktables.hasIndex(bmk_table, YMarkEntry.BOOKMARK.FOLDERS.key())) { - try { - this.worktables.getIndex(bmk_table).delete(urlHash); - } catch (final TableColumnIndexException e) { - // currently nothing to do... - } - } - } - - public void deleteBookmark(final String bmk_user, final String url) throws IOException, SpaceExceededException { - final byte[] urlHash = YMarkUtil.getBookmarkId(url); - this.deleteBookmark(bmk_user, urlHash); - } - - public TreeMap<String, YMarkTag> getTags(final Iterator<Row> rowIterator) { - final TreeMap<String,YMarkTag> tags = new TreeMap<String,YMarkTag>(); - Tables.Row bmk_row = null; - Iterator<String> tit = null; - String tag; - while(rowIterator.hasNext()) { - bmk_row = rowIterator.next(); - if(bmk_row.containsKey(YMarkEntry.BOOKMARK.TAGS.key())) { - tit = YMarkUtil.keysStringToSet(bmk_row.get(YMarkEntry.BOOKMARK.TAGS.key(), YMarkEntry.BOOKMARK.TAGS.deflt())).iterator(); - while(tit.hasNext()) { - tag = tit.next(); - if(tags.containsKey(tag)) { - tags.get(tag).inc(); - } else { - tags.put(tag, new YMarkTag(tag)); - } - } - } - } - return tags; - } - - public TreeMap<String, YMarkTag> getTags(final String bmk_user) throws IOException { - final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); - final TreeMap<String,YMarkTag> tags = new TreeMap<String,YMarkTag>(); - if(this.worktables.hasIndex(bmk_table, YMarkEntry.BOOKMARK.TAGS.key())) { - try { - final TablesColumnIndex index = this.worktables.getIndex(bmk_table); - final Iterator<String> iter = index.keySet(YMarkEntry.BOOKMARK.TAGS.key()).iterator(); - while(iter.hasNext()) { - final String tag = iter.next(); - tags.put(tag, new YMarkTag(tag, index.get(YMarkEntry.BOOKMARK.TAGS.key(), tag).size())); - } - return tags; - } catch (final Exception e) { - // nothing to do - } - } - return getTags(this.worktables.iterator(bmk_table)); - } - - public TreeSet<String> getFolders(final String bmk_user, String root) throws IOException { - final TreeSet<String> folders = new TreeSet<String>(); - final StringBuilder path = new StringBuilder(BUFFER_LENGTH); - final String r = root + YMarkUtil.FOLDERS_SEPARATOR; - final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); - - // if exists, try the index first - if(this.worktables.hasIndex(bmk_table, YMarkEntry.BOOKMARK.FOLDERS.key())) { - TablesColumnIndex index; - try { - index = this.worktables.getIndex(bmk_table); - final Iterator<String> fiter = index.keySet(YMarkEntry.BOOKMARK.FOLDERS.key()).iterator(); - while(fiter.hasNext()) { - final String folder = fiter.next(); - if(folder.startsWith(r)) { - path.setLength(0); - path.append(folder); - while(path.length() > 0 && !path.toString().equals(root)){ - final String p = path.toString(); - if(folders.isEmpty() || !p.equals(folders.floor(p))) { - folders.add(p); - } - path.setLength(path.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR)); - } - } - } - if (!root.equals(YMarkTables.FOLDERS_ROOT)) { folders.add(root); } - return folders; - } catch (final Exception e) { - ConcurrentLog.logException(e); - } - } - - // by default iterate all bookmarks and extract folder information - final Iterator<Tables.Row> bit = this.worktables.iterator(bmk_table); - Tables.Row bmk_row = null; - while(bit.hasNext()) { - bmk_row = bit.next(); - if(bmk_row.containsKey(YMarkEntry.BOOKMARK.FOLDERS.key())) { - final String[] folderArray = YMarkUtil.TAGS_SEPARATOR_PATTERN.split(new String(bmk_row.get(YMarkEntry.BOOKMARK.FOLDERS.key()),"UTF8")); - for (final String folder : folderArray) { - if(folder.length() > root.length() && folder.substring(0, root.length()+1).equals(r)) { - if(!folders.contains(folder)) { - path.setLength(0); - path.append(folder); - //TODO: get rid of .toString.equals() - while(path.length() > 0 && !path.toString().equals(root)){ - folders.add(path.toString()); - path.setLength(path.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR)); - } - } - } - } - } - } - if (!root.equals(YMarkTables.FOLDERS_ROOT)) { folders.add(root); } - return folders; - } - - public int getSize(final String bmk_user) throws IOException { - final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); - return this.worktables.size(bmk_table); - } - - public Iterator<Tables.Row> getBookmarksByFolder(final String bmk_user, final String foldersString) { - final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); - return this.worktables.getByIndex(bmk_table, YMarkEntry.BOOKMARK.FOLDERS.key(), YMarkEntry.BOOKMARK.FOLDERS.seperator(), foldersString); - } - - public Iterator<Tables.Row> getBookmarksByTag(final String bmk_user, final String tagsString) { - final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); - return this.worktables.getByIndex(bmk_table, YMarkEntry.BOOKMARK.TAGS.key(), YMarkEntry.BOOKMARK.TAGS.seperator(), tagsString); - } - - public List<Row> orderBookmarksBy(final Iterator<Row> rowIterator, final String sortname, final String sortorder) { - final List<Row> sortList = new ArrayList<Row>(); - Row row; - while (rowIterator.hasNext()) { - row = rowIterator.next(); - if(row != null) - sortList.add(row); - } - Collections.sort(sortList, new TablesRowComparator(sortname, sortorder)); - return sortList; - } - - public void addTags(final String bmk_user, final String url, final String tagString, final boolean merge) throws IOException { - if(!tagString.isEmpty()) { - // do not set defaults as we only want to update tags - final YMarkEntry bmk = new YMarkEntry(false); - bmk.put(YMarkEntry.BOOKMARK.URL.key(), url); - bmk.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkUtil.cleanTagsString(tagString)); - addBookmark(bmk_user, bmk, merge, true); - } - this.dirty = true; - } - - public void replaceTags(final Iterator<Row> rowIterator, final String bmk_user, final String tagString, final String replaceString) throws IOException { - final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); - final HashSet<String> remove = YMarkUtil.keysStringToSet(YMarkUtil.cleanTagsString(tagString.toLowerCase())); - final StringBuilder t = new StringBuilder(200); - HashSet<String> tags; - Row row; - while (rowIterator.hasNext()) { - row = rowIterator.next(); - if(row != null) { - tags = YMarkUtil.keysStringToSet(row.get(YMarkEntry.BOOKMARK.TAGS.key(), YMarkEntry.BOOKMARK.TAGS.deflt()).toLowerCase()); - tags.removeAll(remove); - t.append(YMarkUtil.keySetToString(tags)); - } - t.append(YMarkUtil.TAGS_SEPARATOR); - t.append(replaceString); - row.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkUtil.cleanTagsString(t.toString())); - this.worktables.update(bmk_table, row); - if(this.worktables.hasIndex(bmk_table)) { - try { - this.worktables.getIndex(bmk_table).update(YMarkEntry.BOOKMARK.TAGS.key(), YMarkEntry.BOOKMARK.TAGS.seperator(), row); - } catch (final Exception e) { - // nothing to do - } - } - } - this.dirty = true; - } - - public void addFolder(final String bmk_user, final String url, final String folder) throws IOException { - if(!folder.isEmpty()) { - // do not set defaults as we only want to add a folder - final YMarkEntry bmk = new YMarkEntry(false); - bmk.put(YMarkEntry.BOOKMARK.URL.key(), url); - bmk.put(YMarkEntry.BOOKMARK.FOLDERS.key(), folder); - addBookmark(bmk_user, bmk, true, true); - } - } - - public void visited(final String bmk_user, final String url) throws IOException { - // do not set defaults - final YMarkEntry bmk = new YMarkEntry(false); - bmk.put(YMarkEntry.BOOKMARK.URL.key(), url); - bmk.put(YMarkEntry.BOOKMARK.DATE_VISITED.key(), (new YMarkDate()).toString()); - addBookmark(bmk_user, bmk, true, true); - } - - public void createBookmark(final LoaderDispatcher loader, final String url, final ClientIdentification.Agent agent, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure { - createBookmark(loader, new DigestURL(url), agent, bmk_user, autotag, tagsString, foldersString); - } - - public void createBookmark(final LoaderDispatcher loader, final DigestURL url, final ClientIdentification.Agent agent, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure { - - final YMarkEntry bmk_entry = new YMarkEntry(false); - final YMarkMetadata meta = new YMarkMetadata(url); - final Document document = meta.loadDocument(loader, agent); - final EnumMap<YMarkMetadata.METADATA, String> metadata = meta.loadMetadata(); - final String urls = url.toNormalform(true); - bmk_entry.put(YMarkEntry.BOOKMARK.URL.key(), urls); - if(!this.worktables.has(YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user), YMarkUtil.getBookmarkId(urls))) { - bmk_entry.put(YMarkEntry.BOOKMARK.PUBLIC.key(), "false"); - bmk_entry.put(YMarkEntry.BOOKMARK.TITLE.key(), metadata.get(YMarkMetadata.METADATA.TITLE)); - bmk_entry.put(YMarkEntry.BOOKMARK.DESC.key(), metadata.get(YMarkMetadata.METADATA.DESCRIPTION)); - } - final String fs = YMarkUtil.cleanFoldersString(foldersString); - if(fs.isEmpty()) - bmk_entry.put(YMarkEntry.BOOKMARK.FOLDERS.key(), YMarkEntry.BOOKMARK.FOLDERS.deflt()); - else - bmk_entry.put(YMarkEntry.BOOKMARK.FOLDERS.key(), fs); - final StringBuilder strb = new StringBuilder(); - if(autotag) { - final String autotags = YMarkAutoTagger.autoTag(document, 3, this.worktables.bookmarks.getTags(bmk_user)); - strb.append(autotags); - } - if(!tagsString.isEmpty()) { - strb.append(YMarkUtil.TAGS_SEPARATOR); - strb.append(tagsString); - } - bmk_entry.put(YMarkEntry.BOOKMARK.TAGS.key(),YMarkUtil.cleanTagsString(strb.toString())); - this.worktables.bookmarks.addBookmark(bmk_user, bmk_entry, true, true); - } - - /** - * Create YMark bookmark from a loaded document - * this is identical to {@link #createBookmark(net.yacy.repository.LoaderDispatcher, net.yacy.cora.document.id.DigestURL, net.yacy.cora.protocol.ClientIdentification.Agent, java.lang.String, boolean, java.lang.String, java.lang.String) } - * without try calling a loader - * - * @param document to create the YMark bookmark for - * @param bmk_user - * @param autotag - * @param tagsString - * @param foldersString - * @throws IOException - * @throws net.yacy.document.Parser.Failure - */ - public void createBookmark(final Document document, final String bmk_user, final boolean autotag, final String tagsString, final String foldersString) throws IOException, Failure { - - if (document != null) { - final YMarkEntry bmk_entry = new YMarkEntry(false); - final YMarkMetadata meta = new YMarkMetadata(document); - - final EnumMap<YMarkMetadata.METADATA, String> metadata = meta.loadMetadata(); - final String urls = document.dc_identifier(); - bmk_entry.put(YMarkEntry.BOOKMARK.URL.key(), urls); - if (!this.worktables.has(YMarkTables.TABLES.BOOKMARKS.tablename(bmk_user), YMarkUtil.getBookmarkId(urls))) { - bmk_entry.put(YMarkEntry.BOOKMARK.PUBLIC.key(), "false"); - bmk_entry.put(YMarkEntry.BOOKMARK.TITLE.key(), metadata.get(YMarkMetadata.METADATA.TITLE)); - bmk_entry.put(YMarkEntry.BOOKMARK.DESC.key(), metadata.get(YMarkMetadata.METADATA.DESCRIPTION)); - } - final String fs = YMarkUtil.cleanFoldersString(foldersString); - if (fs.isEmpty()) - bmk_entry.put(YMarkEntry.BOOKMARK.FOLDERS.key(), YMarkEntry.BOOKMARK.FOLDERS.deflt()); - else - bmk_entry.put(YMarkEntry.BOOKMARK.FOLDERS.key(), fs); - final StringBuilder strb = new StringBuilder(); - if (autotag) { - final String autotags = YMarkAutoTagger.autoTag(document, 3, this.worktables.bookmarks.getTags(bmk_user)); - strb.append(autotags); - } - if (!tagsString.isEmpty()) { - strb.append(YMarkUtil.TAGS_SEPARATOR); - strb.append(tagsString); - } - bmk_entry.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkUtil.cleanTagsString(strb.toString())); - this.worktables.bookmarks.addBookmark(bmk_user, bmk_entry, true, true); - } - } - - public boolean hasBookmark(final String bmk_user, final String urlhash) { - final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); - try { - return this.worktables.has(bmk_table, ASCII.getBytes(urlhash)); - } catch (final IOException e) { - return false; - } - } - - public void addBookmark(final String bmk_user, final YMarkEntry bmk, final boolean mergeTags, final boolean mergeFolders) throws IOException { - final String bmk_table = TABLES.BOOKMARKS.tablename(bmk_user); - final String date = String.valueOf(System.currentTimeMillis()); - byte[] urlHash = null; - try { - urlHash = YMarkUtil.getBookmarkId(bmk.get(YMarkEntry.BOOKMARK.URL.key())); - } catch (final MalformedURLException e) { - ConcurrentLog.info("BOOKMARKIMPORT", "invalid url: "+bmk.get(YMarkEntry.BOOKMARK.URL.key())); - } - Tables.Row bmk_row = null; - - if (urlHash != null) { - try { - bmk_row = this.worktables.select(bmk_table, urlHash); - } catch (final Exception e) { - - } - if (bmk_row == null) { - // create and insert new entry - if(!bmk.containsKey(YMarkEntry.BOOKMARK.DATE_ADDED.key())) { - bmk.put(YMarkEntry.BOOKMARK.DATE_ADDED.key(), date); - bmk.put(YMarkEntry.BOOKMARK.DATE_MODIFIED.key(), date); - } - this.worktables.insert(bmk_table, urlHash, bmk.getData()); - try { - if(this.worktables.hasIndex(bmk_table)) - this.worktables.getIndex(bmk_table).add(YMarkEntry.BOOKMARK.indexColumns(), bmk, urlHash); - } catch (final Exception e) { - // nothing to do - } - } else { - // modify and update existing entry - HashSet<String> oldSet; - HashSet<String> newSet; - - for (final YMarkEntry.BOOKMARK b : YMarkEntry.BOOKMARK.values()) { - switch(b) { - case DATE_ADDED: - if(!bmk_row.containsKey(b.key())) - bmk_row.put(b.key(), date); - break; - case DATE_MODIFIED: - bmk_row.put(b.key(), date); - break; - case TAGS: - oldSet = YMarkUtil.keysStringToSet(bmk_row.get(b.key(),b.deflt())); - if(bmk.containsKey(b.key())) { - newSet = YMarkUtil.keysStringToSet(bmk.get(b.key())); - if(mergeTags) { - newSet.addAll(oldSet); - if(newSet.size() > 1 && newSet.contains(YMarkEntry.BOOKMARK.TAGS.deflt())) - newSet.remove(YMarkEntry.BOOKMARK.TAGS.deflt()); - bmk_row.put(b.key(), YMarkUtil.keySetToString(newSet)); - } else { - bmk_row.put(b.key(), bmk.get(b.key())); - } - } else { - bmk_row.put(b.key(), bmk_row.get(b.key(), b.deflt())); - } - break; - case FOLDERS: - oldSet = YMarkUtil.keysStringToSet(bmk_row.get(b.key(),b.deflt())); - if(bmk.containsKey(b.key())) { - newSet = YMarkUtil.keysStringToSet(bmk.get(b.key())); - if(mergeFolders) { - newSet.addAll(oldSet); - if(newSet.size() > 1 && newSet.contains(YMarkEntry.BOOKMARK.FOLDERS.deflt())) - newSet.remove(YMarkEntry.BOOKMARK.FOLDERS.deflt()); - bmk_row.put(b.key(), YMarkUtil.keySetToString(newSet)); - } else { - bmk_row.put(b.key(), bmk.get(b.key())); - } - } else { - bmk_row.put(b.key(), bmk_row.get(b.key(), b.deflt())); - } - break; - default: - if(bmk.containsKey(b.key())) { - bmk_row.put(b.key(), bmk.get(b.key())); - } else { - bmk_row.put(b.key(), bmk_row.get(b.key(), b.deflt())); - } - } - } - // update bmk_table - this.worktables.update(bmk_table, bmk_row); - try { - if(this.worktables.hasIndex(bmk_table)) - this.worktables.getIndex(bmk_table).update(YMarkEntry.BOOKMARK.indexColumns(), bmk_row); - } catch (final Exception e) { - // nothing to do - } - } - - this.dirty = true; - } - } -} - diff --git a/source/net/yacy/data/ymark/YMarkTag.java b/source/net/yacy/data/ymark/YMarkTag.java deleted file mode 100644 index 13a198878..000000000 --- a/source/net/yacy/data/ymark/YMarkTag.java +++ /dev/null @@ -1,47 +0,0 @@ -package net.yacy.data.ymark; - -public class YMarkTag implements Comparable<YMarkTag>{ - private String name; - private int size; - - public YMarkTag(final String tag) { - this.name = tag.toLowerCase(); - this.size = 1; - } - - public YMarkTag(final String tag, final int size) { - this.name = tag.toLowerCase(); - this.size = size; - } - - public int inc() { - return this.size++; - } - - public int dec() { - if(this.size > 0) - this.size--; - return this.size; - } - - public String name() { - return this.name; - } - - public int size() { - return this.size; - } - - @Override - public int compareTo(YMarkTag tag) { - if(this.name.equals(tag.name())) - return 0; - if(tag.size() < this.size) - return -1; - else if(tag.size() > this.size) - return 1; - else - return this.name.compareTo(tag.name()); - } -} - diff --git a/source/net/yacy/data/ymark/YMarkUtil.java b/source/net/yacy/data/ymark/YMarkUtil.java deleted file mode 100644 index 2d506f1d1..000000000 --- a/source/net/yacy/data/ymark/YMarkUtil.java +++ /dev/null @@ -1,148 +0,0 @@ -// YMarkUtil.java -// (C) 2011 by Stefan Foerster, sof@gmx.de, Norderstedt, Germany -// first published 2011 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate: 2011-03-09 13:50:39 +0100 (Mi, 09 Mrz 2011) $ -// $LastChangedRevision: 7574 $ -// $LastChangedBy: apfelmaennchen $ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.data.ymark; - -import java.net.MalformedURLException; -import java.util.HashSet; -import java.util.Iterator; -import java.util.regex.Pattern; - -import net.yacy.cora.document.encoding.UTF8; -import net.yacy.cora.document.id.DigestURL; -import net.yacy.kelondro.data.word.Word; - -public class YMarkUtil { - public final static String TAGS_SEPARATOR = ","; - public final static String FOLDERS_SEPARATOR = "/"; - public final static String SPACE = " "; - public final static String EMPTY_STRING = new String(); - - public final static Pattern TAGS_SEPARATOR_PATTERN = Pattern.compile(TAGS_SEPARATOR); - public final static Pattern FOLDERS_SEPARATOR_PATTERN = Pattern.compile(FOLDERS_SEPARATOR); - - /** - * convenience function to generate url hashes for YMark bookmarks - * @param url a string representation of a valid url - * @return a byte[] hash for the input URL string - * @throws MalformedURLException - * @see {@link DigestURL#hash()} - */ - public final static byte[] getBookmarkId(String url) throws MalformedURLException { - return (new DigestURL(url)).hash(); - } - - /** - * convenience function to generate word hashes for YMark tags and folders - * @param key a tag or folder name - * @return a byte[] hash for the input string - * @see net.yacy.kelondro.data.word.Word#word2hash(String) - */ - public final static byte[] getKeyId(final String key) { - return Word.word2hash(key.toLowerCase()); - } - - public final static byte[] keySetToBytes(final HashSet<String> urlSet) { - return UTF8.getBytes(keySetToString(urlSet)); - } - - public final static String keySetToString(final HashSet<String> urlSet) { - final Iterator<String> urlIter = urlSet.iterator(); - final - StringBuilder urls = new StringBuilder(urlSet.size()*20); - while(urlIter.hasNext()) { - urls.append(TAGS_SEPARATOR); - urls.append(urlIter.next()); - } - urls.deleteCharAt(0); - return urls.toString(); - } - - public final static HashSet<String> keysStringToSet(final String keysString) { - HashSet<String> keySet = new HashSet<String>(); - final String[] keyArray = keysString.split(TAGS_SEPARATOR); - for (final String key : keyArray) { - keySet.add(key); - } - return keySet; - } - - public final static String cleanTagsString(final String tagsString) { - return cleanTagsString(tagsString, YMarkUtil.EMPTY_STRING); - } - - public final static String cleanTagsString(final String tagsString, final String dflt) { - StringBuilder ts = new StringBuilder(tagsString); - if(ts.length() == 0) - return dflt; - // get rid of double commas and space characters following a comma - for (int i = 0; i < ts.length()-1; i++) { - if (ts.charAt(i) == TAGS_SEPARATOR.charAt(0)) { - if (ts.charAt(i+1) == TAGS_SEPARATOR.charAt(0) || ts.charAt(i+1) == ' ') { - ts.deleteCharAt(i+1); - i--; - } - } - } - // get rid of heading and trailing comma - if (ts.charAt(0) == TAGS_SEPARATOR.charAt(0)) - ts.deleteCharAt(0); - if (ts.length()>0 && ts.charAt(ts.length()-1) == TAGS_SEPARATOR.charAt(0)) - ts.deleteCharAt(ts.length()-1); - return new String(ts); - } - - public final static String cleanFoldersString(final String foldersString) { - return cleanFoldersString(foldersString, YMarkUtil.EMPTY_STRING); - } - - public final static String cleanFoldersString(final String foldersString, final String dflt) { - if(foldersString.isEmpty()) { - return dflt; - } - return cleanFoldersString(new StringBuilder(cleanTagsString(foldersString))); - } - - public final static String cleanFoldersString(final StringBuilder fs) { - if(fs.length() == 0) - return YMarkEntry.BOOKMARK.FOLDERS.deflt(); - for (int i = 0; i < fs.length()-1; i++) { - if (fs.charAt(i) == FOLDERS_SEPARATOR.charAt(0)) { - if (fs.charAt(i+1) == TAGS_SEPARATOR.charAt(0) || fs.charAt(i+1) == FOLDERS_SEPARATOR.charAt(0)) { - fs.deleteCharAt(i); - i--; - } else if (fs.charAt(i+1) == ' ') { - fs.deleteCharAt(i+1); - i--; - } - } - } - if (fs.charAt(fs.length()-1) == FOLDERS_SEPARATOR.charAt(0)) { - fs.deleteCharAt(fs.length()-1); - } - return new String(fs); - } -} diff --git a/source/net/yacy/data/ymark/YMarkXBELImporter.java b/source/net/yacy/data/ymark/YMarkXBELImporter.java deleted file mode 100644 index 1f3125e4e..000000000 --- a/source/net/yacy/data/ymark/YMarkXBELImporter.java +++ /dev/null @@ -1,298 +0,0 @@ -// YMarkXBELImporter.java -// (C) 2011 by Stefan Foerster, sof@gmx.de, Norderstedt, Germany -// first published 2010 on http://yacy.net -// -// This is a part of YaCy, a peer-to-peer based web search engine -// -// $LastChangedDate$ -// $LastChangedRevision$ -// $LastChangedBy$ -// -// LICENSE -// -// This program is free software; you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation; either version 2 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program; if not, write to the Free Software -// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -package net.yacy.data.ymark; - -import java.text.ParseException; -import java.util.HashMap; -import java.util.HashSet; - -import net.yacy.cora.util.ConcurrentLog; - -import org.xml.sax.Attributes; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; -import org.xml.sax.helpers.DefaultHandler; -import org.xml.sax.helpers.XMLReaderFactory; - -public class YMarkXBELImporter extends YMarkImporter { - - // Statics - public static String IMPORTER = "XBEL"; - public static enum XBEL { - NOTHING (""), - XBEL ("<xbel"), - TITLE ("<title"), - DESC ("<desc"), - BOOKMARK ("<bookmark"), - FOLDER ("<folder"), - SEPARATOR ("<separator"), - ALIAS ("<alias"), - INFO ("<info"), - METADATA ("<metadata"); - - private static StringBuilder buffer = new StringBuilder(25); - private String tag; - - private XBEL(String t) { - this.tag = t; - } - public String tag() { - return this.toString().toLowerCase(); - } - public String endTag(boolean empty) { - buffer.setLength(0); - buffer.append(tag); - if(empty) { - buffer.append('/'); - } else { - buffer.insert(1, '/'); - } - buffer.append('>'); - return buffer.toString(); - } - public String startTag(boolean att) { - buffer.setLength(0); - buffer.append(tag); - if(!att) - buffer.append('>'); - return buffer.toString(); - } - } - - // Importer Variables - private final XMLReader xmlReader; - - public YMarkXBELImporter (final MonitoredReader bmk_file, final int queueSize, final String targetFolder, final String sourceFolder) throws SAXException { - super(bmk_file, queueSize, targetFolder, sourceFolder); - setImporter(IMPORTER); - this.xmlReader = XMLReaderFactory.createXMLReader(); - this.xmlReader.setFeature(XML_NAMESPACE_PREFIXES, false); - this.xmlReader.setFeature(XML_NAMESPACES, false); - this.xmlReader.setFeature(XML_VALIDATION, false); - this.xmlReader.setContentHandler(new XBELParser()); - } - - public YMarkXBELImporter (final MonitoredReader bmk_file, final int queueSize, final String targetFolder) throws SAXException { - this(bmk_file, queueSize, "", targetFolder); - } - - @Override - public void parse() throws Exception { - xmlReader.parse(new InputSource(bmk_file)); - } - - public class XBELParser extends DefaultHandler { - - // Parser Variables - private final StringBuilder folderstring; - private final HashMap<String,YMarkEntry> bmkRef; - private final HashSet<YMarkEntry> aliasRef; - private final StringBuilder buffer; - private final StringBuilder folder; - - private YMarkEntry bmk; - private YMarkEntry ref; - private XBEL outer_state; // BOOKMARK, FOLDER, NOTHING - private XBEL inner_state; // DESC, TITLE, INFO, ALIAS, (METADATA), NOTHING - private boolean parse_value; - - - public XBELParser() { - this.folderstring = new StringBuilder(YMarkTables.BUFFER_LENGTH); - this.folderstring.append(targetFolder); - this.bmk = new YMarkEntry(); - this.bmkRef = new HashMap<String,YMarkEntry>(); - this.aliasRef = new HashSet<YMarkEntry>(); - this.buffer = new StringBuilder(); - this.folder = new StringBuilder(YMarkTables.BUFFER_LENGTH); - this.folder.append(targetFolder); - } - - @Override - public void endDocument() throws SAXException { - // put alias references in the bookmark queue to ensure that folders get updated - // we do that at endDocument to ensure all referenced bookmarks already exist - bookmarks.addAll(this.aliasRef); - this.aliasRef.clear(); - this.bmkRef.clear(); - } - - @Override - public void startElement(final String uri, final String name, String tag, final Attributes atts) throws SAXException { - YMarkDate date = new YMarkDate(); - if (tag == null) return; - tag = tag.toLowerCase(); - if (XBEL.BOOKMARK.tag().equals(tag)) { - this.bmk = new YMarkEntry(); - this.bmk.put(YMarkEntry.BOOKMARK.URL.key(), atts.getValue(uri, YMarkEntry.BOOKMARK.URL.xbel_attrb())); - //TODO: include a dynamic loop over all annotation tags - this.bmk.put(YMarkEntry.BOOKMARK.TAGS.key(), atts.getValue(uri, YMarkEntry.BOOKMARK.TAGS.xbel_attrb())); - this.bmk.put(YMarkEntry.BOOKMARK.PUBLIC.key(), atts.getValue(uri, YMarkEntry.BOOKMARK.PUBLIC.xbel_attrb())); - this.bmk.put(YMarkEntry.BOOKMARK.VISITS.key(), atts.getValue(uri, YMarkEntry.BOOKMARK.VISITS.xbel_attrb())); - try { - date.parseISO8601(atts.getValue(uri, YMarkEntry.BOOKMARK.DATE_ADDED.xbel_attrb())); - } catch (final ParseException e) { - // TODO: exception handling - } - this.bmk.put(YMarkEntry.BOOKMARK.DATE_ADDED.key(), date.toString()); - try { - date.parseISO8601(atts.getValue(uri, YMarkEntry.BOOKMARK.DATE_VISITED.xbel_attrb())); - } catch (final ParseException e) { - // TODO: exception handling - } - this.bmk.put(YMarkEntry.BOOKMARK.DATE_VISITED.key(), date.toString()); - try { - date.parseISO8601(atts.getValue(uri, YMarkEntry.BOOKMARK.DATE_MODIFIED.xbel_attrb())); - } catch (final ParseException e) { - // TODO: exception handling - } - this.bmk.put(YMarkEntry.BOOKMARK.DATE_MODIFIED.key(), date.toString()); - UpdateBmkRef(atts.getValue(uri, YMarkEntry.BOOKMARKS_ID), true); - this.outer_state = XBEL.BOOKMARK; - this.inner_state = XBEL.NOTHING; - this.parse_value = false; - } else if(XBEL.FOLDER.tag().equals(tag)) { - this.outer_state = XBEL.FOLDER; - this.inner_state = XBEL.NOTHING; - } else if (XBEL.DESC.tag().equals(tag)) { - this.inner_state = XBEL.DESC; - this.parse_value = true; - } else if (XBEL.TITLE.tag().equals(tag)) { - this.inner_state = XBEL.TITLE; - this.parse_value = true; - } else if (XBEL.INFO.tag().equals(tag)) { - this.inner_state = XBEL.INFO; - this.parse_value = false; - } else if (XBEL.METADATA.tag().equals(tag)) { - // Support for old YaCy BookmarksDB XBEL Metadata (non valid XBEL) - if(this.outer_state == XBEL.BOOKMARK) { - final boolean isMozillaShortcutURL = atts.getValue(uri, "owner").equals("Mozilla") && !atts.getValue(uri, "ShortcutURL").isEmpty(); - final boolean isYacyPublic = atts.getValue(uri, "owner").equals("YaCy") && !atts.getValue(uri, "public").isEmpty(); - if(isMozillaShortcutURL) - this.bmk.put(YMarkEntry.BOOKMARK.TAGS.key(), YMarkUtil.cleanTagsString(atts.getValue(uri, "ShortcutURL"))); - if(isYacyPublic) - this.bmk.put(YMarkEntry.BOOKMARK.PUBLIC.key(), atts.getValue(uri, "public")); - } - } else if (XBEL.ALIAS.tag().equals(tag)) { - final String r = atts.getValue(uri, YMarkEntry.BOOKMARKS_REF); - UpdateBmkRef(r, false); - this.aliasRef.add(this.bmkRef.get(r)); - } - else { - this.outer_state = XBEL.NOTHING; - this.inner_state = XBEL.NOTHING; - this.parse_value = false; - } - } - - @Override - public void endElement(final String uri, final String name, String tag) { - if (tag == null) return; - tag = tag.toLowerCase(); - if(XBEL.BOOKMARK.tag().equals(tag)) { - // write bookmark - if (!this.bmk.isEmpty()) { - this.bmk.put(YMarkEntry.BOOKMARK.FOLDERS.key(), this.folder.toString()); - try { - bookmarks.put(this.bmk); - bmk = new YMarkEntry(); - } catch (final InterruptedException e) { - ConcurrentLog.logException(e); - } - } - this.outer_state = XBEL.FOLDER; - } else if (XBEL.FOLDER.tag().equals(tag)) { - // go up one folder - //TODO: get rid of .toString.equals() - if(!this.folder.toString().equals(targetFolder)) { - folder.setLength(folder.lastIndexOf(YMarkUtil.FOLDERS_SEPARATOR)); - } - this.outer_state = XBEL.FOLDER; - } else if (XBEL.INFO.tag().equals(tag)) { - this.inner_state = XBEL.NOTHING; - } else if (XBEL.METADATA.tag().equals(tag)) { - this.inner_state = XBEL.INFO; - } - } - - @Override - public void characters(final char ch[], final int start, final int length) { - // TODO move string processing to endElement as characters() could be called more than once per tag - if (parse_value) { - buffer.append(ch, start, length); - switch(outer_state) { - case BOOKMARK: - switch(inner_state) { - case DESC: - this.bmk.put(YMarkEntry.BOOKMARK.DESC.key(), buffer.toString().trim()); - break; - case TITLE: - this.bmk.put(YMarkEntry.BOOKMARK.TITLE.key(), buffer.toString().trim()); - break; - default: - break; - } - break; - case FOLDER: - switch(inner_state) { - case DESC: - break; - case TITLE: - this.folder.append(YMarkUtil.FOLDERS_SEPARATOR); - this.folder.append(this.buffer); - break; - default: - break; - } - break; - default: - break; - } - this.buffer.setLength(0); - this.parse_value = false; - } - } - - private void UpdateBmkRef(final String id, final boolean url) { - this.folderstring.setLength(0); - - if(this.bmkRef.containsKey(id)) { - this.folderstring.append(this.bmkRef.get(id).get(YMarkEntry.BOOKMARK.FOLDERS.key())); - this.folderstring.append(','); - this.ref = this.bmkRef.get(id); - } else { - this.ref = new YMarkEntry(); - } - this.folderstring.append(this.folder); - if(url) - this.ref.put(YMarkEntry.BOOKMARK.URL.key(), this.bmk.get(YMarkEntry.BOOKMARK.URL.key())); - this.ref.put(YMarkEntry.BOOKMARK.FOLDERS.key(), this.folderstring.toString()); - this.bmkRef.put(id, ref); - } - } -} diff --git a/source/net/yacy/kelondro/blob/Tables.java b/source/net/yacy/kelondro/blob/Tables.java index dc807fb13..37f7a7672 100644 --- a/source/net/yacy/kelondro/blob/Tables.java +++ b/source/net/yacy/kelondro/blob/Tables.java @@ -52,7 +52,7 @@ import net.yacy.cora.util.ByteArray; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.LookAheadIterator; import net.yacy.cora.util.SpaceExceededException; -import net.yacy.data.ymark.YMarkUtil; +import net.yacy.kelondro.data.word.Word; import net.yacy.kelondro.util.FileUtils; @@ -166,7 +166,7 @@ public class Tables implements Iterable<String> { return tci.hasIndex(columnName); } try { - if(this.has(tableName+CIDX, YMarkUtil.getKeyId(columnName))) { + if(this.has(tableName+CIDX, Word.word2hash(columnName))) { return true; } } catch (final IOException e) { @@ -240,7 +240,7 @@ public class Tables implements Iterable<String> { public Iterator<String> iterator() { return getTablenames().iterator(); } - + public Set<String> getTablenames() { // we did a lazy initialization, but here we must discover all actually existing tables String tablename; @@ -274,7 +274,7 @@ public class Tables implements Iterable<String> { for (final BEncodedHeap heap: this.tables.values()) heap.close(); this.tables.clear(); } - + public void clear() { Set<String> tablenames = this.getTablenames(); for (String tablename: tablenames) this.clear(tablename); @@ -443,7 +443,7 @@ public class Tables implements Iterable<String> { final BEncodedHeap heap = getHeap(table); return heap.keys(); } - + public Iterator<byte[]> keys(final String table, final boolean up, final boolean rotating) throws IOException { final BEncodedHeap heap = getHeap(table); return heap.keys(up, rotating); @@ -480,7 +480,7 @@ public class Tables implements Iterable<String> { public Iterator<Row> iterator(final String table, final Pattern wherePattern, final boolean up) throws IOException { return new OrderedRowIterator(table, wherePattern, up); } - + /** * @param rowIterator * an open iterator on a table. Must not be null. @@ -492,7 +492,7 @@ public class Tables implements Iterable<String> { public static Collection<Row> orderBy(final Iterator<Row> rowIterator, final String sortColumn) { return orderByString(rowIterator, sortColumn, "", SortDirection.ASC); } - + /** * @param rowIterator * an open iterator on a table. Must not be null. @@ -518,7 +518,7 @@ public class Tables implements Iterable<String> { } return orderBy(rowIterator, comparator); } - + /** * @param rowIterator * an open iterator on a table. Must not be null. @@ -571,7 +571,7 @@ public class Tables implements Iterable<String> { } return orderBy(rowIterator, comparator); } - + /** * @param rowIterator * an open iterator on a table. Must not be null. @@ -598,7 +598,7 @@ public class Tables implements Iterable<String> { } return orderBy(rowIterator, comparator); } - + /** * @param rowIterator * an open iterator on a table. Must not be null. @@ -619,7 +619,7 @@ public class Tables implements Iterable<String> { final BEncodedHeap heap = getHeap(table); return heap.columns(); } - + /** Sort direction for ordering rows */ public enum SortDirection { /** Ascending order */ @@ -743,7 +743,7 @@ public class Tables implements Iterable<String> { this.whereValue = null; this.wherePattern = null; this.heap = getHeap(table); - this.i = heap.keys(up, false); + this.i = this.heap.keys(up, false); } /** @@ -761,7 +761,7 @@ public class Tables implements Iterable<String> { this.whereValue = whereValue; this.wherePattern = null; this.heap = getHeap(table); - this.i = heap.keys(up, false); + this.i = this.heap.keys(up, false); } /** @@ -778,7 +778,7 @@ public class Tables implements Iterable<String> { this.whereValue = null; this.wherePattern = wherePattern == null || wherePattern.toString().isEmpty() ? null : wherePattern; this.heap = getHeap(table); - this.i = heap.keys(up, false); + this.i = this.heap.keys(up, false); } /** @@ -794,7 +794,7 @@ public class Tables implements Iterable<String> { this.whereValue = null; this.wherePattern = wherePattern == null || wherePattern.toString().isEmpty() ? null : wherePattern; this.heap = getHeap(table); - this.i = heap.keys(up, false); + this.i = this.heap.keys(up, false); } @Override @@ -831,7 +831,7 @@ public class Tables implements Iterable<String> { } } - + public static class Data extends LinkedHashMap<String, byte[]> { private static final long serialVersionUID = 978426054043749337L; diff --git a/source/net/yacy/kelondro/blob/TablesColumnBLOBIndex.java b/source/net/yacy/kelondro/blob/TablesColumnBLOBIndex.java index c8203b884..8c4e7c8e6 100644 --- a/source/net/yacy/kelondro/blob/TablesColumnBLOBIndex.java +++ b/source/net/yacy/kelondro/blob/TablesColumnBLOBIndex.java @@ -34,7 +34,7 @@ import java.util.concurrent.ConcurrentHashMap; import net.yacy.cora.util.ByteBuffer; import net.yacy.cora.util.ConcurrentLog; import net.yacy.cora.util.SpaceExceededException; -import net.yacy.data.ymark.YMarkUtil; +import net.yacy.kelondro.data.word.Word; public class TablesColumnBLOBIndex extends TablesColumnIndex{ @@ -66,7 +66,7 @@ public class TablesColumnBLOBIndex extends TablesColumnIndex{ @Override public void deleteIndex(final String columnName) { - final byte[] column = YMarkUtil.getKeyId(columnName); + final byte[] column = Word.word2hash(columnName); try { this.index.remove(column); } catch (final IOException e) { @@ -80,7 +80,7 @@ public class TablesColumnBLOBIndex extends TablesColumnIndex{ protected void insertPK(final String columnName, final String columnValue, final byte[] pk) { Map<String, byte[]> valueIdxMap; Collection<byte[]> PKset; - final byte[] column = YMarkUtil.getKeyId(columnName); + final byte[] column = Word.word2hash(columnName); try { valueIdxMap = this.index.get(column); if(valueIdxMap != null) { @@ -147,7 +147,7 @@ public class TablesColumnBLOBIndex extends TablesColumnIndex{ @Override public Set<String> keySet(final String columnName) { - final byte[] column = YMarkUtil.getKeyId(columnName); + final byte[] column = Word.word2hash(columnName); // a TreeSet is used to get sorted set of keys (e.g. folders) if(this.index.containsKey(column)) { try { @@ -163,7 +163,7 @@ public class TablesColumnBLOBIndex extends TablesColumnIndex{ @Override public boolean containsKey(final String columnName, final String key) { - final byte[] column = YMarkUtil.getKeyId(columnName); + final byte[] column = Word.word2hash(columnName); if(this.index.containsKey(column)) { try { return this.index.get(column).containsKey(key); @@ -178,13 +178,13 @@ public class TablesColumnBLOBIndex extends TablesColumnIndex{ @Override public boolean hasIndex(final String columnName) { - final byte[] column = YMarkUtil.getKeyId(columnName); + final byte[] column = Word.word2hash(columnName); return this.index.containsKey(column); } @Override public Collection<byte[]> get(final String columnName, final String key) { - final byte[] column = YMarkUtil.getKeyId(columnName); + final byte[] column = Word.word2hash(columnName.toLowerCase()); // deserialize try { return byteToCollection(this.index.get(column).get(key)); @@ -198,7 +198,7 @@ public class TablesColumnBLOBIndex extends TablesColumnIndex{ @Override public int size(final String columnName) { - final byte[] column = YMarkUtil.getKeyId(columnName); + final byte[] column = Word.word2hash(columnName.toLowerCase()); if(this.index.containsKey(column)) { try { return this.index.get(column).size(); diff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.java index 39d6aed41..75ddf416c 100644 --- a/source/net/yacy/search/Switchboard.java +++ b/source/net/yacy/search/Switchboard.java @@ -180,12 +180,10 @@ import net.yacy.data.WorkTables; import net.yacy.data.wiki.WikiBoard; import net.yacy.data.wiki.WikiCode; import net.yacy.data.wiki.WikiParser; -import net.yacy.data.ymark.YMarkTables; import net.yacy.document.Condenser; import net.yacy.document.Document; import net.yacy.document.LibraryProvider; import net.yacy.document.Parser; -import net.yacy.document.Parser.Failure; import net.yacy.document.ProbabilisticClassifier; import net.yacy.document.TextParser; import net.yacy.document.Tokenizer; @@ -3910,16 +3908,6 @@ public final class Switchboard extends serverSwitch { this.bookmarksDB.saveBookmark(bookmark); } - // do the same for ymarks - // TODO: could a non admin user add crawls? - try { - this.tables.bookmarks.createBookmark(scraper, YMarkTables.USER_ADMIN, true, "crawlStart", "/Crawl Start"); - } catch (final IOException e) { - ConcurrentLog.logException(e); - } catch (final Failure e) { - ConcurrentLog.logException(e); - } - // that was ok return null; } |