added a crawl option to obey html-meta-robots-noindex. This is on by

default.

6 files changed, 29 insertions, 19 deletions

diff --git a/htroot/CrawlStartExpert_p.html b/htroot/CrawlStartExpert_p.html
index cc82de4c4..f3b562e72 100644
--- a/htroot/CrawlStartExpert_p.html
+++ b/htroot/CrawlStartExpert_p.html
@@ -109,14 +109,17 @@
             <input name="crawlingDomMaxPages" id="crawlingDomMaxPages" type="text" size="6" maxlength="6" value="#[crawlingDomMaxPages]#" />
           </dd>
           
-          <dt><label for="crawlingQ">Accept URLs with '?' / dynamic URLs</label></dt>
+          <dt><label for="Constraints">misc. Constraints</label></dt>
           <dd>
             <span class="info" style="float:right"><img src="/env/grafics/i16.gif" width="16" height="16" alt="info"/><span style="right:0px;">
             A questionmark is usually a hint for a dynamic page. URLs pointing to dynamic content should usually not be crawled.
             However, there are sometimes web pages with static content that
             is accessed with URLs containing question marks. If you are unsure, do not check this to avoid crawl loops.
+            Following frames is NOT done by Gxxg1e, but we do by default to have a richer content. 'nofollow' in robots metadata can be overridden; this does not affect obeying of the robots.txt which is never ignored.
             </span></span>
-            <input type="checkbox" name="crawlingQ" id="crawlingQ" #(crawlingQChecked)#::checked="checked"#(/crawlingQChecked)# />
+            Accept URLs with query-part ('?'): <input type="checkbox" name="crawlingQ" id="crawlingQ" #(crawlingQChecked)#::checked="checked"#(/crawlingQChecked)# />&nbsp;&nbsp;
+            Obey html-robots-noindex: <input type="checkbox" name="obeyHtmlRobotsNoindex" id="obeyHtmlRobotsNoindex" #(obeyHtmlRobotsNoindexChecked)#::checked="checked"#(/obeyHtmlRobotsNoindexChecked)# /><!--&nbsp;&nbsp;
+            Follow Frames: <input type="checkbox" name="followFrames" id="followFrames" #(followFramesChecked)#::checked="checked"#(/followFramesChecked)# />&nbsp;&nbsp;-->
           </dd>
 	        <dt>Load Filter on URLs</dt>
 	        <dd><span class="info" style="float:right"><img src="/env/grafics/i16.gif" width="16" height="16" alt="info"/><span style="right:0px;">
diff --git a/htroot/CrawlStartExpert_p.java b/htroot/CrawlStartExpert_p.java
index 3f9cfd0df..514302541 100644
--- a/htroot/CrawlStartExpert_p.java
+++ b/htroot/CrawlStartExpert_p.java
@@ -62,6 +62,8 @@ public class CrawlStartExpert_p {
         prop.put("crawlingDomMaxCheck", (crawlingDomMaxPages == -1) ? "0" : "1");
         prop.put("crawlingDomMaxPages", (crawlingDomMaxPages == -1) ? 10000 : crawlingDomMaxPages);
         prop.put("crawlingQChecked", env.getConfigBool("crawlingQ", true) ? "1" : "0");
+        prop.put("followFramesChecked", env.getConfigBool("followFrames", true) ? "1" : "0");
+        prop.put("obeyHtmlRobotsNoindexChecked", env.getConfigBool("obeyHtmlRobotsNoindex", true) ? "1" : "0");
         prop.put("storeHTCacheChecked", env.getConfigBool("storeHTCache", true) ? "1" : "0");
         prop.put("indexingTextChecked", env.getConfigBool("indexText", true) ? "1" : "0");
         prop.put("indexingMediaChecked", env.getConfigBool("indexMedia", true) ? "1" : "0");
diff --git a/htroot/CrawlStartSite_p.html b/htroot/CrawlStartSite_p.html
index 357e58ef6..cfa384317 100644
--- a/htroot/CrawlStartSite_p.html
+++ b/htroot/CrawlStartSite_p.html
@@ -39,7 +39,7 @@
         <dd>

           <table border="0" cellpadding="0" cellspacing="0"><tr valign="top">

             <td valign="top"><input type="radio" name="crawlingMode" id="url" value="url" checked="checked"

-            onmousedown="document.getElementById('rangeDomain').disabled=false;document.getElementById('rangeSubpath').disabled=false;document.getElementById('crawlingDomMaxCheck').disabled=false;document.getElementById('crawlingDomMaxPages').disabled=false;document.getElementById('crawlingQ').disabled=false;"/>Start URL&nbsp;(must start with<br/>http:// https:// ftp:// smb:// file://)</td>

+            onmousedown="document.getElementById('rangeDomain').disabled=false;document.getElementById('rangeSubpath').disabled=false;document.getElementById('crawlingDomMaxCheck').disabled=false;document.getElementById('crawlingDomMaxPages').disabled=false;"/>Start URL&nbsp;(must start with<br/>http:// https:// ftp:// smb:// file://)</td>

             <td valign="top">

             <input name="crawlingURL" id="crawlingURL" type="text" size="50" maxlength="256" value="#[starturl]#" onkeypress="changed()" onfocus="check('url')" style="font-size:16px"/><br/>

             <input name="bookmarkTitle" id="bookmarkTitle" type="text" size="50" maxlength="256" value="" readonly="readonly" style="background:transparent; border:0px"/>

@@ -53,7 +53,7 @@
             <td><div id="sitelistURLs"></div></td>

           </tr><tr>

             <td><input type="radio" name="crawlingMode" id="sitemap" value="sitemap" disabled="disabled"

-            onmousedown="document.getElementById('rangeDomain').disabled=true;document.getElementById('rangeSubpath').disabled=true;document.getElementById('crawlingDomMaxCheck').disabled=true;document.getElementById('crawlingDomMaxPages').disabled=true;document.getElementById('crawlingQ').disabled=true;"/>Sitemap URL</td>

+            onmousedown="document.getElementById('rangeDomain').disabled=true;document.getElementById('rangeSubpath').disabled=true;document.getElementById('crawlingDomMaxCheck').disabled=true;document.getElementById('crawlingDomMaxPages').disabled=true;"/>Sitemap URL</td>

             <td><input name="sitemapURL" type="text" size="41" maxlength="256" value="" readonly="readonly" style="background:transparent; border:0px"/></td>

           </tr>

           </table><br/>

@@ -75,10 +75,6 @@
           <td valign="top">documents</td>

           </tr></table>

         </dd>

-        <dt><label>Dynamic URLs</label></dt>

-        <dd>

-		  <input type="checkbox" name="crawlingQ" id="crawlingQ" #(crawlingQChecked)#::checked="checked"#(/crawlingQChecked)# /> allow <a href="http://en.wikipedia.org/wiki/Query_string">query-strings</a> (urls with a '?' in the path)

-        </dd>

         <dt><label>Collection</label></dt>

         <dd>

           <input name="collection" id="collection" type="text" size="60" maxlength="100" value="#[collection]#" #(collectionEnabled)#disabled="disabled"::#(/collectionEnabled)# />

@@ -92,6 +88,9 @@
 		  <input type="hidden" name="deleteold" id="deleteold" value="on" />

 		  <input type="hidden" name="storeHTCache" id="storeHTCache" value="on" />

           <input type="hidden" name="cachePolicy" id="cachePolicy" value="iffresh" />

+          <input type="hidden" name="crawlingQ" id="crawlingQ" value="on" />

+          <input type="hidden" name="followFrames" id="followFrames" value="on" />

+          <input type="hidden" name="obeyHtmlRobotsNoindex" id="obeyHtmlRobotsNoindex" value="on" />

           <input type="hidden" name="indexText" id="indexText" value="on" />

           <input type="hidden" name="indexMedia" id="indexMedia" value="on" />

           <input type="hidden" name="intention" id="intention" value="" />

diff --git a/htroot/Crawler_p.java b/htroot/Crawler_p.java
index f0a112eca..3c368a95a 100644
--- a/htroot/Crawler_p.java
+++ b/htroot/Crawler_p.java
@@ -246,13 +246,19 @@ public class Crawler_p {
                 final int crawlingDomMaxPages = (crawlingDomMaxCheck) ? post.getInt("crawlingDomMaxPages", -1) : -1;
                 env.setConfig("crawlingDomMaxPages", Integer.toString(crawlingDomMaxPages));
 
-                boolean crawlingQ = "on".equals(post.get("crawlingQ", "off"));
+                boolean crawlingQ = "on".equals(post.get("crawlingQ", "on"));
                 env.setConfig("crawlingQ", crawlingQ);
+                
+                boolean followFrames = "on".equals(post.get("followFrames", "on"));
+                env.setConfig("followFrames", followFrames);
+                
+                boolean obeyHtmlRobotsNoindex = "on".equals(post.get("obeyHtmlRobotsNoindex", "on"));
+                env.setConfig("obeyHtmlRobotsNoindex", obeyHtmlRobotsNoindex);
 
-                final boolean indexText = "on".equals(post.get("indexText", "off"));
+                final boolean indexText = "on".equals(post.get("indexText", "on"));
                 env.setConfig("indexText", indexText);
 
-                final boolean indexMedia = "on".equals(post.get("indexMedia", "off"));
+                final boolean indexMedia = "on".equals(post.get("indexMedia", "on"));
                 env.setConfig("indexMedia", indexMedia);
 
                 env.setConfig("storeHTCache", storeHTCache);
@@ -361,7 +367,7 @@ public class Crawler_p {
                         directDocByURL,
                         crawlingIfOlder,
                         crawlingDomMaxPages,
-                        crawlingQ,
+                        crawlingQ, followFrames, obeyHtmlRobotsNoindex,
                         indexText,
                         indexMedia,
                         storeHTCache,
diff --git a/htroot/QuickCrawlLink_p.html b/htroot/QuickCrawlLink_p.html
index 9d9a9d1b9..b2cfa73d3 100644
--- a/htroot/QuickCrawlLink_p.html
+++ b/htroot/QuickCrawlLink_p.html
@@ -15,7 +15,7 @@
       If you click on it while browsing, the currently viewed website will be inserted into the YaCy crawling queue for indexing.

     </p>

     <p>

-      <a class="BookmarkLink" href="javascript:w = window.open('http://#[host]#:#[port]#/QuickCrawlLink_p.html?indexText=on&amp;indexMedia=on&amp;crawlingQ=on&amp;xdstopw=on&amp;title='+escape(document.title)+'&amp;url='+escape(location.href),'_blank','height=150,width=500,resizable=yes,scrollbar=no,directory=no,menubar=no,location=no');w.focus();">Crawl with YaCy</a>

+      <a class="BookmarkLink" href="javascript:w = window.open('http://#[host]#:#[port]#/QuickCrawlLink_p.html?indexText=on&amp;indexMedia=on&amp;crawlingQ=on&amp;followFrames=on&amp;obeyHtmlRobotsNoindex=on&amp;xdstopw=on&amp;title='+escape(document.title)+'&amp;url='+escape(location.href),'_blank','height=150,width=500,resizable=yes,scrollbar=no,directory=no,menubar=no,location=no');w.focus();">Crawl with YaCy</a>

     </p>

     

     ::<!-- 1 -->

diff --git a/htroot/QuickCrawlLink_p.java b/htroot/QuickCrawlLink_p.java
index 11e81c9df..d21f2e48e 100644
--- a/htroot/QuickCrawlLink_p.java
+++ b/htroot/QuickCrawlLink_p.java
@@ -97,7 +97,9 @@ public class QuickCrawlLink_p {
         final String crawlingMustMatch  = post.get("mustmatch", CrawlProfile.MATCH_ALL_STRING);

         final String crawlingMustNotMatch  = post.get("mustnotmatch", CrawlProfile.MATCH_NEVER_STRING);

         final int CrawlingDepth      = post.getInt("crawlingDepth", 0);

-        final boolean crawlDynamic   = post.get("crawlingQ", "").equals("on");

+        final boolean crawlingQ      = post.get("crawlingQ", "").equals("on");

+        final boolean followFrames   = post.get("followFrames", "").equals("on");

+        final boolean obeyHtmlRobotsNoindex = post.get("obeyHtmlRobotsNoindex", "").equals("on");

         final boolean indexText      = post.get("indexText", "off").equals("on");

         final boolean indexMedia     = post.get("indexMedia", "off").equals("on");

         final boolean storeHTCache   = post.get("storeHTCache", "").equals("on");

@@ -145,11 +147,9 @@ public class QuickCrawlLink_p {
                         true,

                         60 * 24 * 30, // recrawlIfOlder (minutes); here: one month

                         -1, // domMaxPages, if negative: no count restriction

-                        crawlDynamic,

-                        indexText,

-                        indexMedia,

-                        storeHTCache,

-                        remoteIndexing,

+                        crawlingQ, followFrames, obeyHtmlRobotsNoindex,

+                        indexText, indexMedia,

+                        storeHTCache, remoteIndexing,

                         CacheStrategy.IFFRESH,

                         collection);

                 sb.crawler.putActive(pe.handle().getBytes(), pe);