Support for indexing of RSS feeds!

- added a scanning in html parser for rss feeds
- storage of rss feed addresses, can be viewed with http://localhost:8080/Tables_p.html?table=rss
- rss items retrieved by http://localhost:8080/Load_RSS_p.html (in Index Creation menu) can be selected and indexed
- a rss feed retrieved in http://localhost:8080/Load_RSS_p.html can now be fully indexed
- indexing of rss feeds can be placed in scheduler

git-svn-id: https://svn.berlios.de/svnroot/repos/yacy/trunk@7073 6c8d7289-2bf4-0310-a012-ef5d649a1542

5 files changed, 146 insertions, 58 deletions

diff --git a/htroot/Load_RSS_p.html b/htroot/Load_RSS_p.html
index 4776b39a3..9a7b95937 100644
--- a/htroot/Load_RSS_p.html
+++ b/htroot/Load_RSS_p.html
@@ -36,11 +36,32 @@
         <dl>
           <dt><b>URL of the RSS feed</b></dt>
           <dd><input type="text" name="url" value="#[url]#" size="60" maxlength="256"/></dd>
-          <dt>Simulation Mode</dt>
+          <dt>Preview</dt>
           <dd><input type="submit" name="showrss" value="Show RSS Items" /></dd>
-          <dt>Indexing Mode</dt>
-          <dd>#(showload)#Available after successful loading of rss feed in simulation mode::
-          <!--<input type="submit" name="loadrss" value="Index RSS Items" />-->not yet implemented <b>THIS INTERFACE IS A STUB - DEVELOPMENT IS ONGOING</b>
+          <dt>Indexing</dt>
+          <dd>#(showload)#Available after successful loading of rss feed in preview::
+          <input type="hidden" name="url" value="#[rss]#" />
+          <input type="submit" name="indexAllItemContent" value="Add All Items to Index (full content of url)" />
+          <dl>
+            <dt>once<input type="radio" name="repeat" value="off" checked="checked"/></dt>
+            <dd>load this feed once now</dd>
+			<dt>scheduled<input type="radio" name="repeat" value="on"/></dt>
+			<dd>repeat the feed loading every<br/>
+			<select name="repeat_time">
+              <option value="1">1</option><option value="2">2</option><option value="3">3</option>
+              <option value="4">4</option><option value="5">5</option><option value="6">6</option>
+              <option value="7" selected="selected">7</option>
+              <option value="8">8</option><option value="9">9</option><option value="10">10</option>
+              <option value="12">12</option><option value="14">14</option><option value="21">21</option>
+              <option value="28">28</option><option value="30">30</option>
+			</select>
+			<select name="repeat_unit">
+              <option value="selminutes">minutes</option>
+              <option value="selhours">hours</option>
+              <option value="seldays" selected="selected">days</option>
+            </select> automatically.
+			</dd>
+          </dl>
           #(/showload)#</dd>
 	      
         </dl>
@@ -49,7 +70,7 @@
 
 	#(showitems)#::
 	<form name="rssfeed"><fieldset>
-	  <legend><label for="table">RSS Feed</label></legend>
+	  <legend><label for="table">RSS Feed of #[rss]#</label></legend>
       <dl>
 		<dt>Title</dt><dd>#[title]#</dd>
 		<dt>Author</dt><dd>#[author]#</dd>
@@ -62,6 +83,7 @@
       <table class="sortable" border="0" cellpadding="2" cellspacing="1">
         <tr class="TableHeader" valign="bottom">
           <td><input type="checkbox" name="allswitch" onclick="setall(this.form.name, this.value)" /></td>
+          <td>State</td>
           <td>Title</td>
           <td>URL</td>
           <td>Author</td>
@@ -71,7 +93,8 @@
         </tr>
         #{item}#
         <tr class="TableCellLight">
-          <td align="left"><input type="checkbox" name="item_#[count]#" value="mark_#[hash]#" /></td>
+          <td align="left"><input type="checkbox" name="item_#[count]#" value="mark_#[guid]#" /></td>
+          <td>#(state)#new::enqueued::indexed#(/state)#</td>
           <td><a href="#[link]#">#[title]#</a></td>
           <td><a href="#[link]#">#[link]#</a></td>
           <td>#[author]#</td>
@@ -81,7 +104,11 @@
         </tr>
         #{/item}#
       </table>
+      <p>
       <input type="hidden" name="num" value="#[num]#" />
+      <input type="hidden" name="url" value="#[rss]#" />
+      <input type="submit" name="indexSelectedItemContent" value="Add Selected Items to Index (full content of url)" /></dt>
+      </p>
     </fieldset></form>
 	#(/showitems)#
 
diff --git a/htroot/Load_RSS_p.java b/htroot/Load_RSS_p.java
index 1f02d8f30..57b7d2950 100644
--- a/htroot/Load_RSS_p.java
+++ b/htroot/Load_RSS_p.java
@@ -21,23 +21,33 @@
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.text.DateFormat;
+import java.util.Date;
+import java.util.Map;
 
 import net.yacy.cora.document.Hit;
 import net.yacy.cora.document.RSSFeed;
 import net.yacy.cora.document.RSSMessage;
 import net.yacy.cora.document.RSSReader;
 import net.yacy.cora.protocol.RequestHeader;
+import net.yacy.cora.storage.ARC;
+import net.yacy.cora.storage.ComparableARC;
+import net.yacy.document.Parser.Failure;
 import net.yacy.kelondro.data.meta.DigestURI;
 import net.yacy.kelondro.logging.Log;
+import net.yacy.kelondro.order.Base64Order;
 
 import de.anomic.crawler.CrawlProfile;
 import de.anomic.crawler.retrieval.Response;
+import de.anomic.data.WorkTables;
+import de.anomic.search.Segments;
 import de.anomic.search.Switchboard;
 import de.anomic.server.serverObjects;
 import de.anomic.server.serverSwitch;
 
 public class Load_RSS_p {
 
+    private static final ARC<byte[], Date> indexTriggered = new ComparableARC<byte[], Date>(1000, Base64Order.enhancedCoder);
+    
     public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {
 
         final serverObjects prop = new serverObjects();
@@ -51,11 +61,17 @@ public class Load_RSS_p {
 
         prop.put("url", post.get("url", ""));
         
+        int repeat_time = Integer.parseInt(post.get("repeat_time", "-1"));
+        final String repeat_unit = post.get("repeat_unit", "seldays"); // selminutes, selhours, seldays
+        if (!post.get("repeat", "off").equals("on") && repeat_time > 0) repeat_time = -1;
+        
+        boolean record_api = false;
+        
         DigestURI url = null;
         try {
             url = post.containsKey("url") ? new DigestURI(post.get("url", ""), null) : null;
         } catch (MalformedURLException e) {
-            Log.logException(e);
+            Log.logWarning("Load_RSS_p", "url not well-formed: '" + post.get("url", "") + "'");
         }
         
         // if we have an url then try to load the rss
@@ -69,6 +85,54 @@ public class Load_RSS_p {
             Log.logException(e);
         }
 
+        // index all selected items: description only
+        if (rss != null && post.containsKey("indexSelectedItemContent")) {
+            RSSFeed feed = rss.getFeed();
+            loop: for (Map.Entry<String, String> entry: post.entrySet()) {
+                if (entry.getValue().startsWith("mark_")) try {
+                    RSSMessage message = feed.getMessage(entry.getValue().substring(5));
+                    DigestURI messageurl = new DigestURI(message.getLink());
+                    if (indexTriggered.containsKey(messageurl.hash())) continue loop;
+                    if (sb.urlExists(Segments.Process.LOCALCRAWLING, messageurl.hash()) != null) continue loop;
+                    sb.addToIndex(messageurl, null, null);
+                    indexTriggered.put(messageurl.hash(), new Date());
+                } catch (IOException e) {
+                    Log.logException(e);
+                } catch (Failure e) {
+                    Log.logException(e);
+                }
+            }
+        }
+        if (rss != null && post.containsKey("indexAllItemContent")) {
+            record_api = true;
+            RSSFeed feed = rss.getFeed();
+            loop: for (RSSMessage message: feed) {
+                try {
+                    DigestURI messageurl = new DigestURI(message.getLink());
+                    if (indexTriggered.containsKey(messageurl.hash()) && post.containsKey("indexSelectedItemContent")) continue loop;
+                    if (sb.urlExists(Segments.Process.LOCALCRAWLING, messageurl.hash()) != null) continue loop;
+                    sb.addToIndex(messageurl, null, null);
+                    indexTriggered.put(messageurl.hash(), new Date());
+                } catch (IOException e) {
+                    Log.logException(e);
+                } catch (Failure e) {
+                    Log.logException(e);
+                }
+            }
+        }
+        
+        if (record_api) {
+            // record API action
+            if (repeat_time > 0) {
+                // store as scheduled api call
+                sb.tables.recordAPICall(post, "Load_RSS_p.html", WorkTables.TABLE_API_TYPE_CRAWLER, "import feed " + url.toNormalform(true, false), repeat_time, repeat_unit.substring(3));
+            } else {
+                // store just a protocol
+                sb.tables.recordAPICall(post, "Load_RSS_p.html", WorkTables.TABLE_API_TYPE_CRAWLER, "import feed " + url.toNormalform(true, false));
+            }
+        }
+        
+        // show items from rss
         if (rss != null) {
             prop.put("showitems", 1);
             RSSFeed feed = rss.getFeed();
@@ -76,27 +140,30 @@ public class Load_RSS_p {
             prop.putHTML("showitems_title", channel.getTitle());
             String author = channel.getAuthor();
             if (author == null || author.length() == 0) author = channel.getCopyright();
+            Date pubDate = channel.getPubDate();
             prop.putHTML("showitems_author", author == null ? "" : author);
             prop.putHTML("showitems_description", channel.getDescription());
             prop.putHTML("showitems_language", channel.getLanguage());
-            prop.putHTML("showitems_date", DateFormat.getDateTimeInstance().format(channel.getPubDate()));
+            prop.putHTML("showitems_date", (pubDate == null) ? "" : DateFormat.getDateTimeInstance().format(pubDate));
             prop.putHTML("showitems_ttl", channel.getTTL());
             prop.putHTML("showitems_docs", channel.getDocs());
             
             int i = 0;
             for (final Hit item: feed) {
                 try {
-                    url = new DigestURI(item.getLink(), null);
+                    DigestURI messageurl = new DigestURI(item.getLink(), null);
                     author = item.getAuthor();
                     if (author == null) author = item.getCopyright();
+                    pubDate = item.getPubDate();
                     prop.put("showitems_item_" + i + "_count", i);
-                    prop.putHTML("showitems_item_" + i + "_hash", new String(url.hash()));
+                    prop.put("showitems_item_" + i + "_state", sb.urlExists(Segments.Process.LOCALCRAWLING, messageurl.hash()) != null ? 2 : indexTriggered.containsKey(messageurl.hash()) ? 1 : 0);
+                    prop.putHTML("showitems_item_" + i + "_guid", item.getGuid());
                     prop.putHTML("showitems_item_" + i + "_author", author == null ? "" : author);
                     prop.putHTML("showitems_item_" + i + "_title", item.getTitle());
-                    prop.putHTML("showitems_item_" + i + "_link", url.toNormalform(false, false));
+                    prop.putHTML("showitems_item_" + i + "_link", messageurl.toNormalform(false, false));
                     prop.putHTML("showitems_item_" + i + "_description", item.getDescription());
                     prop.putHTML("showitems_item_" + i + "_language", item.getLanguage());
-                    prop.putHTML("showitems_item_" + i + "_date", DateFormat.getDateTimeInstance().format(item.getPubDate()));
+                    prop.putHTML("showitems_item_" + i + "_date", (pubDate == null) ? "" : DateFormat.getDateTimeInstance().format(pubDate));
                     i++;
                 } catch (MalformedURLException e) {
                     Log.logException(e);
@@ -105,7 +172,11 @@ public class Load_RSS_p {
             }
             prop.put("showitems_item", i);
             prop.put("showitems_num", i);
-            if (i > 0) prop.put("showload", 1);
+            prop.putHTML("showitems_rss", url.toNormalform(true, false));
+            if (i > 0) {
+                prop.put("showload", 1);
+                prop.put("showload_rss", url.toNormalform(true, false));
+            }
         }
         
         return prop;
diff --git a/htroot/Tables_p.html b/htroot/Tables_p.html
index f8794c16f..a2ae0c488 100644
--- a/htroot/Tables_p.html
+++ b/htroot/Tables_p.html
@@ -28,7 +28,7 @@
   <body id="Tables">

     #%env/templates/header.template%#

     #%env/templates/submenuConfig.template%#

-    

+    #(showselection)#::

     <h2>Table Administration</h2>

     <form action="Tables_p.html" method="get">

       <fieldset><legend>Table Selection</legend>

@@ -60,6 +60,7 @@
       </dl>

     </fieldset>

     </form>

+    #(/showselection)#

     #(showtable)#::

     <form action="Tables_p.html" method="post" enctype="multipart/form-data" accept-charset="UTF-8" name="tablelisting">

       <fieldset>

diff --git a/htroot/Tables_p.java b/htroot/Tables_p.java
index bd9d47edb..33bbe0333 100644
--- a/htroot/Tables_p.java
+++ b/htroot/Tables_p.java
@@ -39,52 +39,35 @@ public class Tables_p {
     public static serverObjects respond(final RequestHeader header, final serverObjects post, final serverSwitch env) {

         final Switchboard sb = (Switchboard) env;

         final serverObjects prop = new serverObjects();

+

+        prop.put("showtable", 0);

+        prop.put("showedit", 0);

+        prop.put("showselection", 0);

         

-        if (post == null) {

-            prop.put("pattern", "");

-            

-            // show table selection

-            int count = 0;

-            Iterator<String> ti = sb.tables.tables();

-            String tablename;

-            while (ti.hasNext()) {

-                tablename = ti.next();

-                prop.put("tables_" + count + "_name", tablename);

-                prop.put("tables_" + count + "_selected", 0);

-                count++;

-            }

-            prop.put("tables", count);

-            

-            // generate table

-            prop.put("showtable", 0);

-            prop.put("showedit", 0);

-            

-            // the peer address

-            prop.put("address", sb.peers.mySeed().getPublicAddress());

-            

-            // return rewrite properties

-            return prop;

-        }

-        

-        String table = post.get("table", null);

+        String table = (post == null) ? null : post.get("table", null);

         if (table != null && !sb.tables.hasHeap(table)) table = null;

-        String counts = post.get("count", null);

-        int maxcount = (counts == null || counts.equals("all")) ? Integer.MAX_VALUE : Integer.parseInt(counts);

-        String pattern = post.get("search", "");

-        Pattern matcher = (pattern.length() == 0 || pattern.equals(".*")) ? null : Pattern.compile(".*" + pattern + ".*");

-        prop.put("pattern", pattern);

         

         // show table selection

         int count = 0;

         Iterator<String> ti = sb.tables.tables();

         String tablename;

+        prop.put("showselection", 1);

         while (ti.hasNext()) {

             tablename = ti.next();

-            prop.put("tables_" + count + "_name", tablename);

-            prop.put("tables_" + count + "_selected", (table != null && table.equals(tablename)) ? 1 : 0);

+            prop.put("showselection_tables_" + count + "_name", tablename);

+            prop.put("showselection_tables_" + count + "_selected", (table != null && table.equals(tablename)) ? 1 : 0);

             count++;

         }

-        prop.put("tables", count);

+        prop.put("showselection_tables", count);

+        prop.put("showselection_pattern", "");

+

+        if (post == null) return prop; // return rewrite properties

+        

+        String counts = post.get("count", null);

+        int maxcount = (counts == null || counts.equals("all")) ? Integer.MAX_VALUE : Integer.parseInt(counts);

+        String pattern = post.get("search", "");

+        Pattern matcher = (pattern.length() == 0 || pattern.equals(".*")) ? null : Pattern.compile(".*" + pattern + ".*");

+        prop.put("pattern", pattern);

         

         List<String> columns = null;

         if (table != null) try {

diff --git a/htroot/api/table_p.java b/htroot/api/table_p.java
index 4de9e5d3d..48dff143d 100644
--- a/htroot/api/table_p.java
+++ b/htroot/api/table_p.java
@@ -36,13 +36,15 @@ public class table_p {
         final serverObjects prop = new serverObjects();

         String table = (post == null) ? null : post.get("table", null);

         if (table != null && !sb.tables.hasHeap(table)) table = null;

-        if (table == null) {

-            prop.put("showtable", 0);

-            return prop;

-        }

+        prop.put("showtable", 0);

+        

+        if (table == null) return prop;

         

         boolean showpk = post.containsKey("pk");

         

+        String selectKey = post.containsKey("selectKey") ? post.get("selectKey") : null;

+        String selectValue = (selectKey != null && post.containsKey("selectValue")) ? post.get("selectValue") : null;

+        

         ArrayList<String> columns = null;

         try {

             columns = sb.tables.columns(table);

@@ -86,8 +88,8 @@ public class table_p {
             final Iterator<Tables.Row> mapIterator = sb.tables.orderByPK(plainIterator, maxCount).iterator();

             Tables.Row trow;

             boolean dark = true;

-            byte[] cell;

-            while ((mapIterator.hasNext()) && (count < maxCount)) {

+            String cellName, cellValue;

+            rowloop: while ((mapIterator.hasNext()) && (count < maxCount)) {

                 trow = mapIterator.next();

                 if (row == null) continue;

                 prop.put("showtable_list_" + count + "_dark", ((dark) ? 1 : 0) ); dark=!dark;

@@ -95,9 +97,13 @@ public class table_p {
                 prop.put("showtable_list_" + count + "_showpk_pk", new String(trow.getPK()));

                 prop.put("showtable_list_" + count + "_count", count);

                 for (int i = 0; i < columns.size(); i++) {

-                    cell = trow.get(columns.get(i));

-                    prop.putHTML("showtable_list_" + count + "_columns_" + i + "_column", columns.get(i));

-                    prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cell == null ? "" : new String(cell));

+                    cellName = columns.get(i);

+                    cellValue = new String(trow.get(cellName));

+                    if (selectKey != null && cellName.equals(selectKey) && !cellValue.matches(selectValue)) {

+                        continue rowloop;

+                    }

+                    prop.putHTML("showtable_list_" + count + "_columns_" + i + "_column", cellName);

+                    prop.putHTML("showtable_list_" + count + "_columns_" + i + "_cell", cellValue);

                 }

                 prop.put("showtable_list_" + count + "_columns", columns.size());