added canonical filter

attention: this is on by default!
(it should do the right thing)

2 files changed, 48 insertions, 44 deletions

diff --git a/htroot/CrawlProfileEditor_p.xml b/htroot/CrawlProfileEditor_p.xml
index 0b880ac3f..58036b48e 100644
--- a/htroot/CrawlProfileEditor_p.xml
+++ b/htroot/CrawlProfileEditor_p.xml
@@ -1,48 +1,49 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <crawlProfiles>
 #{crawlProfiles}# 
-	<crawlProfile>
-		<handle>#[handle]#</handle>
-		<name>#[name]#</name>
-		<collections>#[collections]#</collections>
-		<agentName>#[agentName]#</agentName>
-		<userAgent>#[userAgent]#</userAgent>
-		<depth>#[depth]#</depth>
-		<directDocByURL>#(directDocByURL)#false::true#(/directDocByURL)#</directDocByURL>
-		<recrawlIfOlder>#[recrawlIfOlder]#</recrawlIfOlder>
-		<domMaxPages>#[domMaxPages]#</domMaxPages>
-		<crawlingQ>#(crawlingQ)#false::true#(/crawlingQ)#</crawlingQ>
-		<followFrames>#(followFrames)#false::true#(/followFrames)#</followFrames>
-		<obeyHtmlRobotsNoindex>#(obeyHtmlRobotsNoindex)#false::true#(/obeyHtmlRobotsNoindex)#</obeyHtmlRobotsNoindex>
-		<obeyHtmlRobotsNofollow>#(obeyHtmlRobotsNofollow)#false::true#(/obeyHtmlRobotsNofollow)#</obeyHtmlRobotsNofollow>
-		<indexText>#(indexText)#false::true#(/indexText)#</indexText>
-		<indexMedia>#(indexMedia)#false::true#(/indexMedia)#</indexMedia>
-		<storeHTCache>#(storeHTCache)#false::true#(/storeHTCache)#</storeHTCache>
-		<remoteIndexing>#(remoteIndexing)#false::true#(/remoteIndexing)#</remoteIndexing>
-		<cacheStrategy>#[cacheStrategy]#</cacheStrategy>
-		<crawlerAlwaysCheckMediaType>#(crawlerAlwaysCheckMediaType)#false::true#(/crawlerAlwaysCheckMediaType)#</crawlerAlwaysCheckMediaType>
-		<crawlerURLMustMatch>#[crawlerURLMustMatch]#</crawlerURLMustMatch>
-		<crawlerURLMustNotMatch>#[crawlerURLMustNotMatch]#</crawlerURLMustNotMatch>
-		<crawlerOriginURLMustMatch>#[crawlerOriginURLMustMatch]#</crawlerOriginURLMustMatch>
-		<crawlerOriginURLMustNotMatch>#[crawlerOriginURLMustNotMatch]#</crawlerOriginURLMustNotMatch>
-		<crawlerIPMustMatch>#[crawlerIPMustMatch]#</crawlerIPMustMatch>
-		<crawlerIPMustNotMatch>#[crawlerIPMustNotMatch]#</crawlerIPMustNotMatch>
-		<crawlerCountryMustMatch>#[crawlerCountryMustMatch]#</crawlerCountryMustMatch>
-		<crawlerNoLimitURLMustMatch>#[crawlerNoLimitURLMustMatch]#</crawlerNoLimitURLMustMatch>
-		<indexURLMustMatch>#[indexURLMustMatch]#</indexURLMustMatch>
-		<indexURLMustNotMatch>#[indexURLMustNotMatch]#</indexURLMustNotMatch>
-		<indexContentMustMatch>#[indexContentMustMatch]#</indexContentMustMatch>
-		<indexContentMustNotMatch>#[indexContentMustNotMatch]#</indexContentMustNotMatch>
-		<indexMediaTypeMustMatch>#[indexMediaTypeMustMatch]#</indexMediaTypeMustMatch>
-		<indexMediaTypeMustNotMatch>#[indexMediaTypeMustNotMatch]#</indexMediaTypeMustNotMatch>
-		<indexSolrQueryMustMatch>#[indexSolrQueryMustMatch]#</indexSolrQueryMustMatch>
-		<indexSolrQueryMustNotMatch>#[indexSolrQueryMustNotMatch]#</indexSolrQueryMustNotMatch>
-		<status>#(status)#terminated::active::system#(/status)#</status>
-		<crawlingDomFilterContent>
-		#{crawlingDomFilterContent}#
-			<item>#[item]#</item>
-		#{/crawlingDomFilterContent}#
-		</crawlingDomFilterContent>
-	</crawlProfile>
+    <crawlProfile>
+        <handle>#[handle]#</handle>
+        <name>#[name]#</name>
+        <collections>#[collections]#</collections>
+        <agentName>#[agentName]#</agentName>
+        <userAgent>#[userAgent]#</userAgent>
+        <depth>#[depth]#</depth>
+        <directDocByURL>#(directDocByURL)#false::true#(/directDocByURL)#</directDocByURL>
+        <recrawlIfOlder>#[recrawlIfOlder]#</recrawlIfOlder>
+        <domMaxPages>#[domMaxPages]#</domMaxPages>
+        <crawlingQ>#(crawlingQ)#false::true#(/crawlingQ)#</crawlingQ>
+        <followFrames>#(followFrames)#false::true#(/followFrames)#</followFrames>
+        <obeyHtmlRobotsNoindex>#(obeyHtmlRobotsNoindex)#false::true#(/obeyHtmlRobotsNoindex)#</obeyHtmlRobotsNoindex>
+        <obeyHtmlRobotsNofollow>#(obeyHtmlRobotsNofollow)#false::true#(/obeyHtmlRobotsNofollow)#</obeyHtmlRobotsNofollow>
+        <indexText>#(indexText)#false::true#(/indexText)#</indexText>
+        <indexMedia>#(indexMedia)#false::true#(/indexMedia)#</indexMedia>
+        <storeHTCache>#(storeHTCache)#false::true#(/storeHTCache)#</storeHTCache>
+        <remoteIndexing>#(remoteIndexing)#false::true#(/remoteIndexing)#</remoteIndexing>
+        <cacheStrategy>#[cacheStrategy]#</cacheStrategy>
+        <crawlerAlwaysCheckMediaType>#(crawlerAlwaysCheckMediaType)#false::true#(/crawlerAlwaysCheckMediaType)#</crawlerAlwaysCheckMediaType>
+        <crawlerURLMustMatch>#[crawlerURLMustMatch]#</crawlerURLMustMatch>
+        <crawlerURLMustNotMatch>#[crawlerURLMustNotMatch]#</crawlerURLMustNotMatch>
+        <crawlerOriginURLMustMatch>#[crawlerOriginURLMustMatch]#</crawlerOriginURLMustMatch>
+        <crawlerOriginURLMustNotMatch>#[crawlerOriginURLMustNotMatch]#</crawlerOriginURLMustNotMatch>
+        <crawlerIPMustMatch>#[crawlerIPMustMatch]#</crawlerIPMustMatch>
+        <crawlerIPMustNotMatch>#[crawlerIPMustNotMatch]#</crawlerIPMustNotMatch>
+        <crawlerCountryMustMatch>#[crawlerCountryMustMatch]#</crawlerCountryMustMatch>
+        <crawlerNoLimitURLMustMatch>#[crawlerNoLimitURLMustMatch]#</crawlerNoLimitURLMustMatch>
+        <indexURLMustMatch>#[indexURLMustMatch]#</indexURLMustMatch>
+        <indexURLMustNotMatch>#[indexURLMustNotMatch]#</indexURLMustNotMatch>
+        <indexContentMustMatch>#[indexContentMustMatch]#</indexContentMustMatch>
+        <indexContentMustNotMatch>#[indexContentMustNotMatch]#</indexContentMustNotMatch>
+        <indexMediaTypeMustMatch>#[indexMediaTypeMustMatch]#</indexMediaTypeMustMatch>
+        <indexMediaTypeMustNotMatch>#[indexMediaTypeMustNotMatch]#</indexMediaTypeMustNotMatch>
+        <indexSolrQueryMustMatch>#[indexSolrQueryMustMatch]#</indexSolrQueryMustMatch>
+        <indexSolrQueryMustNotMatch>#[indexSolrQueryMustNotMatch]#</indexSolrQueryMustNotMatch>
+        <noindexWhenCanonicalUnequalURL>#(noindexWhenCanonicalUnequalURL)#false::true#(/noindexWhenCanonicalUnequalURL)#</noindexWhenCanonicalUnequalURL>
+        <status>#(status)#terminated::active::system#(/status)#</status>
+        <crawlingDomFilterContent>
+        #{crawlingDomFilterContent}#
+            <item>#[item]#</item>
+        #{/crawlingDomFilterContent}#
+        </crawlingDomFilterContent>
+    </crawlProfile>
 #{/crawlProfiles}# 
 </crawlProfiles>
diff --git a/htroot/CrawlStartExpert.html b/htroot/CrawlStartExpert.html
index c3e60c1b8..9917cd7e1 100644
--- a/htroot/CrawlStartExpert.html
+++ b/htroot/CrawlStartExpert.html
@@ -412,6 +412,9 @@
             <table style="border-width: 0px">
             <tr><td style="width:110px"><img src="env/grafics/plus.gif" alt=""> must-match</td><td><input name="indexmustmatch" id="indexmustmatch" type="text" size="55" maxlength="100000" value="#[indexmustmatch]#" onblur="if (this.value=='') this.value='.*';"/> (must not be empty)</td></tr>
             <tr><td><img src="env/grafics/minus.gif" alt=""> must-not-match</td><td><input name="indexmustnotmatch" id="indexmustnotmatch" type="text" size="55" maxlength="100000" value="#[indexmustnotmatch]#" /></td></tr>
+            <tr>
+              <td colspan="2"><input type="checkbox" name="noindexWhenCanonicalUnequalURL" id="noindexWhenCanonicalUnequalURL" #(noindexWhenCanonicalUnequalURLChecked)#::checked="checked"#(/noindexWhenCanonicalUnequalURLChecked)#/> No Indexing when Canonical present and Canonical != URL</td>
+            </tr>
             </table>
             </dd>
             <dt>Filter on Content of Document<br/>(all visible text, including camel-case-tokenized url and title)</dt>
@@ -470,7 +473,7 @@
                     <tr>
                         <td style="width:110px"><img src="env/grafics/minus.gif" alt=""> must-not-match</td>
                         <td>
-                            <input name="indexSolrQueryMustNotMatch" id="indexSolrQueryMustNotMatch" type="text" size="55" maxlength="100000" value="#[indexSolrQueryMustNotMatch]#" aria-describedby="indexSolrQueryInfo" />
+                            <input name="indexSolrQueryMustNotMatch" id="indexSolrQueryMustNotMatch" type="text" size="55" maxlength="100000" value="#[indexSolrQueryMustNotMatch]#" aria-describedby="indexSolrQueryInfo" enabled="false"/>
                         </td>
                     </tr>
                     #(/embeddedSolrConnected)#